diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..14bc64b --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +* @kaanyagci + +/bootstrap/ @kaanyagci +/config/ @kaanyagci +/.github/ @kaanyagci diff --git a/.github/workflows/amiary-ci.yml b/.github/workflows/amiary-ci.yml new file mode 100644 index 0000000..5d714e0 --- /dev/null +++ b/.github/workflows/amiary-ci.yml @@ -0,0 +1,50 @@ +name: Amiary PostgreSQL CI + +on: + pull_request: + push: + branches: [main] + paths: + - ".github/workflows/amiary-ci.yml" + - ".github/workflows/manual-deploy.yml" + - "bootstrap/amiary-apps.sql" + - "config/runtrace-pg_hba.conf" + - "envs/production/.env.db" + - "envs/production/compose.yml" + - "scripts/run-runtrace-backup.sh" + - "scripts/run-runtrace-backup-loop.sh" + - "scripts/test-amiary-bootstrap.sh" + - "scripts/test-amiary-shared-backup.sh" + - "scripts/test-shared-restore.sh" + - "scripts/preflight-postgres-major.sh" + - "scripts/validate-amiary-config.sh" + - "scripts/validate-postgres-config.sh" + - "scripts/verify-runtrace-restore.sh" + - "README.md" + +permissions: + contents: read + +concurrency: + group: amiary-postgres-ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + policy-and-integration: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Validate repository contracts + run: scripts/validate-postgres-config.sh + - name: Check shell scripts + run: shellcheck scripts/*.sh + - name: Exercise PostgreSQL 18 role isolation + run: scripts/test-amiary-bootstrap.sh + - name: Exercise five-database backup contract + run: scripts/test-amiary-shared-backup.sh + - name: Reject unsafe restore targets + run: python3 scripts/test-amiary-restore-guards.py + - name: Exercise ownership and ACL restore contract + run: scripts/test-shared-restore.sh diff --git a/README.md b/README.md index 79b2db9..c6eb85e 100644 --- a/README.md +++ b/README.md @@ -912,6 +912,143 @@ fixtures also use the production candidate. These fixtures do not replace a fresh verified backup of every production database or a scoped rollout and rollback plan for the shared service. +## Amiary database isolation + +Amiary provisions separate production, canary and identity databases and roles. This PR preserves the current production PostgreSQL image; a major upgrade requires its own restored-data release. The five-database backup/restore helpers are named `run-amiary-shared-backup.sh` and `verify-amiary-shared-restore.sh` and do not replace the active Runtrace schedule. Configure and restore-test the new job before enabling it. + +For Amiary, use separate production and canary login roles for migrations, the +API, and the graph worker. Shared `NOLOGIN` roles carry object privileges while +the environment-specific logins carry credentials and database access: + +| Environment | Database | Purpose | Login role | Membership / ownership | +| --- | --- | --- | --- | --- | +| Production | `amiary` | Migrations | `amiary_migrator` | Database owner; may `SET ROLE amiary_security_definer` | +| Production | `amiary` | API | `amiary_api_prod` | Inherits `amiary_api`; direct `CONNECT` only | +| Production | `amiary` | Graph worker | `amiary_worker_prod` | Inherits `amiary_worker`; direct `CONNECT` only | +| Canary | `amiary_canary` | Migrations | `amiary_canary_migrator` | Database owner; may `SET ROLE amiary_security_definer` | +| Canary | `amiary_canary` | API | `amiary_api_canary` | Inherits `amiary_api`; direct `CONNECT` only | +| Canary | `amiary_canary` | Graph worker | `amiary_worker_canary` | Inherits `amiary_worker`; direct `CONNECT` only | +| Identity | `keycloak_amiary` | Keycloak | `keycloak_amiary_app` | Database owner; no Amiary capability membership | +| Shared | Five protected databases | Backup | `makepad_backup` | `NOINHERIT`; may `SET ROLE makepad_backup_reader`; HBA-denied elsewhere | + +`amiary_api` and `amiary_worker` are constrained `NOLOGIN NOBYPASSRLS` roles. +Application migrations grant them only the schema, table, sequence, and +function privileges each service needs. `amiary_security_definer` is a +`NOLOGIN BYPASSRLS` function-owner role with no database `CONNECT`; it is not +granted to API, worker, or Keycloak logins. Only the two migrators receive a +non-inherited, non-admin membership that permits an explicit `SET ROLE` while +installing reviewed security-definer functions. + +Generate eight independent passwords and provision the three databases plus +the shared read-only backup login: + +```bash +: "${POSTGRES_ADMIN_URL:?set POSTGRES_ADMIN_URL to a PostgreSQL superuser connection URI}" +: "${AMIARY_MIGRATOR_DB_PASSWORD:?set AMIARY_MIGRATOR_DB_PASSWORD to a generated password}" +: "${AMIARY_API_DB_PASSWORD:?set AMIARY_API_DB_PASSWORD to a generated password}" +: "${AMIARY_WORKER_DB_PASSWORD:?set AMIARY_WORKER_DB_PASSWORD to a generated password}" +: "${AMIARY_CANARY_MIGRATOR_DB_PASSWORD:?set AMIARY_CANARY_MIGRATOR_DB_PASSWORD to a generated password}" +: "${AMIARY_CANARY_API_DB_PASSWORD:?set AMIARY_CANARY_API_DB_PASSWORD to a generated password}" +: "${AMIARY_CANARY_WORKER_DB_PASSWORD:?set AMIARY_CANARY_WORKER_DB_PASSWORD to a generated password}" +: "${KEYCLOAK_AMIARY_DB_PASSWORD:?set KEYCLOAK_AMIARY_DB_PASSWORD to a generated password}" +: "${MAKEPAD_BACKUP_DB_PASSWORD:?set MAKEPAD_BACKUP_DB_PASSWORD to a generated password}" + +psql "${POSTGRES_ADMIN_URL}" \ + -v amiary_migrator_password="${AMIARY_MIGRATOR_DB_PASSWORD}" \ + -v amiary_api_prod_password="${AMIARY_API_DB_PASSWORD}" \ + -v amiary_worker_prod_password="${AMIARY_WORKER_DB_PASSWORD}" \ + -v amiary_canary_migrator_password="${AMIARY_CANARY_MIGRATOR_DB_PASSWORD}" \ + -v amiary_api_canary_password="${AMIARY_CANARY_API_DB_PASSWORD}" \ + -v amiary_worker_canary_password="${AMIARY_CANARY_WORKER_DB_PASSWORD}" \ + -v keycloak_amiary_app_password="${KEYCLOAK_AMIARY_DB_PASSWORD}" \ + -v makepad_backup_password="${MAKEPAD_BACKUP_DB_PASSWORD}" \ + -f bootstrap/amiary-apps.sql +``` + +The bootstrap removes public database privileges, repairs role and ownership +drift on every run, and grants API/worker logins only `CONNECT` to their own +environment. Application schema migrations remain in `Makepad-fr/amiary`. + +Use the purpose-specific DSN; never run the API or worker through a migrator +connection. The CA path below is the backend container contract: + +```text +postgres://amiary_migrator:@:5432/amiary?sslmode=verify-full&sslrootcert=/run/secrets/database_ca +postgres://amiary_api_prod:@:5432/amiary?sslmode=verify-full&sslrootcert=/run/secrets/database_ca +postgres://amiary_worker_prod:@:5432/amiary?sslmode=verify-full&sslrootcert=/run/secrets/database_ca +postgres://amiary_canary_migrator:@:5432/amiary_canary?sslmode=verify-full&sslrootcert=/run/secrets/database_ca +postgres://amiary_api_canary:@:5432/amiary_canary?sslmode=verify-full&sslrootcert=/run/secrets/database_ca +postgres://amiary_worker_canary:@:5432/amiary_canary?sslmode=verify-full&sslrootcert=/run/secrets/database_ca +postgres://keycloak_amiary_app:@:5432/keycloak_amiary?sslmode=verify-full&sslrootcert=/run/secrets/database_ca +postgres://makepad_backup:@:5432/amiary?sslmode=verify-full&sslrootcert=/run/secrets/database_ca +``` + +```bash +: "${POSTGRES_ADMIN_URL:?set POSTGRES_ADMIN_URL to a PostgreSQL superuser connection URI}" +: "${KEYCLOAK_VIF_DB_PASSWORD:?set KEYCLOAK_VIF_DB_PASSWORD to a generated password}" +: "${KEYCLOAK_MAKEPAD_DB_PASSWORD:?set KEYCLOAK_MAKEPAD_DB_PASSWORD to a generated password}" +: "${KEYCLOAK_VESTIAIRE_DB_PASSWORD:?set KEYCLOAK_VESTIAIRE_DB_PASSWORD to a generated password}" +: "${KEYCLOAK_RUNTRACE_DB_PASSWORD:?set KEYCLOAK_RUNTRACE_DB_PASSWORD to a generated password}" +: "${RUNTRACE_DB_PASSWORD:?set RUNTRACE_DB_PASSWORD to a generated password}" +: "${OPENPANEL_DB_PASSWORD:?set OPENPANEL_DB_PASSWORD to a generated password}" + +psql "$POSTGRES_ADMIN_URL" \ + -v keycloak_vif_app_password="$KEYCLOAK_VIF_DB_PASSWORD" \ + -v keycloak_makepad_app_password="$KEYCLOAK_MAKEPAD_DB_PASSWORD" \ + -v keycloak_vestiaire_app_password="$KEYCLOAK_VESTIAIRE_DB_PASSWORD" \ + -v keycloak_runtrace_app_password="$KEYCLOAK_RUNTRACE_DB_PASSWORD" \ + -f bootstrap/keycloak-new-instances.sql + +psql "$POSTGRES_ADMIN_URL" \ + -v runtrace_app_password="$RUNTRACE_DB_PASSWORD" \ + -f bootstrap/runtrace-app.sql + +# Use this targeted bootstrap when the other Keycloak databases already exist. +psql "$POSTGRES_ADMIN_URL" \ + -v keycloak_runtrace_app_password="$KEYCLOAK_RUNTRACE_DB_PASSWORD" \ + -f bootstrap/keycloak-runtrace-app.sql + +psql "$POSTGRES_ADMIN_URL" \ + -v openpanel_app_password="$OPENPANEL_DB_PASSWORD" \ + -f bootstrap/openpanel-app.sql +``` + +The current production Keycloak environments connect with the DB VM host: + +```text +postgres://keycloak_vif_app:@:5432/keycloak_vif?sslmode=disable +postgres://keycloak_makepad_app:@:5432/keycloak_makepad?sslmode=disable +postgres://keycloak_vestiaire_app:@:5432/keycloak_vestiaire?sslmode=disable +postgres://keycloak_runtrace_app:@:5432/keycloak_runtrace?sslmode=verify-full&sslrootcert=/etc/makepad/tls/postgres/ca.crt +postgres://runtrace_app:@:5432/runtrace?sslmode=verify-full&sslrootcert=/etc/runtrace/postgres/ca.crt +postgres://openpanel_app:@:5432/openpanel?schema=public&sslmode=disable +``` + +Stacks deployed through this repository's shared overlay network should use the `makepad-postgres` alias instead: + +```text +postgres://keycloak_vif_app:@makepad-postgres:5432/keycloak_vif?sslmode=disable +postgres://keycloak_makepad_app:@makepad-postgres:5432/keycloak_makepad?sslmode=disable +postgres://keycloak_vestiaire_app:@makepad-postgres:5432/keycloak_vestiaire?sslmode=disable +postgres://keycloak_runtrace_app:@makepad-postgres:5432/keycloak_runtrace?sslmode=verify-full&sslrootcert=/etc/makepad/tls/postgres/ca.crt +postgres://runtrace_app:@makepad-postgres:5432/runtrace?sslmode=verify-full&sslrootcert=/etc/runtrace/postgres/ca.crt +postgres://openpanel_app:@makepad-postgres:5432/openpanel?schema=public&sslmode=disable +``` + +Le Petit Coin uses the app-specific overlay alias: + +```text +postgres://le_petit_coin_canary_app:@makepad-postgres-le-petit-coin:5432/le_petit_coin_canary?sslmode=disable +postgres://le_petit_coin_app:@makepad-postgres-le-petit-coin:5432/le_petit_coin?sslmode=disable +``` + +The production VIF app uses its app-specific overlay alias and deploy-time provisioned database: + +```text +postgres://vif:@makepad-postgres-vif:5432/vif?sslmode=disable +``` + +If production overrides `DEPLOY_VIF_DB_NAME` or `DEPLOY_VIF_DB_USER`, use those values in the connection URI. ## Fashion catalog databases The application bootstraps are scoped to Scraping, the Iceberg catalog and the Vestiaire developer platform. They do not redeploy or change the shared PostgreSQL runtime. Use certificate-verified TLS and protected application credentials. Existing credentials and ownership must be preserved; drift requires separate review. diff --git a/bootstrap/amiary-apps.sql b/bootstrap/amiary-apps.sql new file mode 100644 index 0000000..ae8968d --- /dev/null +++ b/bootstrap/amiary-apps.sql @@ -0,0 +1,328 @@ +\set ON_ERROR_STOP on + +-- Run with a PostgreSQL superuser connection. This idempotently provisions +-- Amiary production, canary, and identity databases without embedding secrets. +-- Application migrations own schema/object grants; this bootstrap owns only +-- cluster roles, memberships, database ownership, and database CONNECT policy. + +\if :{?amiary_migrator_password} +\else + \echo 'missing required psql variable: amiary_migrator_password' + \quit 1 +\endif +\if :{?amiary_api_prod_password} +\else + \echo 'missing required psql variable: amiary_api_prod_password' + \quit 1 +\endif +\if :{?amiary_worker_prod_password} +\else + \echo 'missing required psql variable: amiary_worker_prod_password' + \quit 1 +\endif +\if :{?amiary_canary_migrator_password} +\else + \echo 'missing required psql variable: amiary_canary_migrator_password' + \quit 1 +\endif +\if :{?amiary_api_canary_password} +\else + \echo 'missing required psql variable: amiary_api_canary_password' + \quit 1 +\endif +\if :{?amiary_worker_canary_password} +\else + \echo 'missing required psql variable: amiary_worker_canary_password' + \quit 1 +\endif +\if :{?keycloak_amiary_app_password} +\else + \echo 'missing required psql variable: keycloak_amiary_app_password' + \quit 1 +\endif +\if :{?makepad_backup_password} +\else + \echo 'missing required psql variable: makepad_backup_password' + \quit 1 +\endif + +SELECT CASE WHEN NULLIF(btrim(:'amiary_migrator_password'), '') IS NULL THEN 'false' ELSE 'true' END AS amiary_migrator_password_is_nonempty \gset +SELECT CASE WHEN NULLIF(btrim(:'amiary_api_prod_password'), '') IS NULL THEN 'false' ELSE 'true' END AS amiary_api_prod_password_is_nonempty \gset +SELECT CASE WHEN NULLIF(btrim(:'amiary_worker_prod_password'), '') IS NULL THEN 'false' ELSE 'true' END AS amiary_worker_prod_password_is_nonempty \gset +SELECT CASE WHEN NULLIF(btrim(:'amiary_canary_migrator_password'), '') IS NULL THEN 'false' ELSE 'true' END AS amiary_canary_migrator_password_is_nonempty \gset +SELECT CASE WHEN NULLIF(btrim(:'amiary_api_canary_password'), '') IS NULL THEN 'false' ELSE 'true' END AS amiary_api_canary_password_is_nonempty \gset +SELECT CASE WHEN NULLIF(btrim(:'amiary_worker_canary_password'), '') IS NULL THEN 'false' ELSE 'true' END AS amiary_worker_canary_password_is_nonempty \gset +SELECT CASE WHEN NULLIF(btrim(:'keycloak_amiary_app_password'), '') IS NULL THEN 'false' ELSE 'true' END AS keycloak_amiary_app_password_is_nonempty \gset +SELECT CASE WHEN NULLIF(btrim(:'makepad_backup_password'), '') IS NULL THEN 'false' ELSE 'true' END AS makepad_backup_password_is_nonempty \gset + +\if :amiary_migrator_password_is_nonempty +\else + \echo 'empty required psql variable: amiary_migrator_password' + \quit 1 +\endif +\if :amiary_api_prod_password_is_nonempty +\else + \echo 'empty required psql variable: amiary_api_prod_password' + \quit 1 +\endif +\if :amiary_worker_prod_password_is_nonempty +\else + \echo 'empty required psql variable: amiary_worker_prod_password' + \quit 1 +\endif +\if :amiary_canary_migrator_password_is_nonempty +\else + \echo 'empty required psql variable: amiary_canary_migrator_password' + \quit 1 +\endif +\if :amiary_api_canary_password_is_nonempty +\else + \echo 'empty required psql variable: amiary_api_canary_password' + \quit 1 +\endif +\if :amiary_worker_canary_password_is_nonempty +\else + \echo 'empty required psql variable: amiary_worker_canary_password' + \quit 1 +\endif +\if :keycloak_amiary_app_password_is_nonempty +\else + \echo 'empty required psql variable: keycloak_amiary_app_password' + \quit 1 +\endif +\if :makepad_backup_password_is_nonempty +\else + \echo 'empty required psql variable: makepad_backup_password' + \quit 1 +\endif + +SELECT pg_advisory_lock(hashtext('makepad-postgres'), hashtext('amiary-bootstrap')); +CREATE TEMP TABLE amiary_existing_logins AS SELECT rolname FROM pg_roles; + +-- Shared capability roles never authenticate. Only the definer role may bypass +-- RLS, and it receives no database CONNECT privilege from this bootstrap. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'amiary_api') THEN + CREATE ROLE amiary_api NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'amiary_worker') THEN + CREATE ROLE amiary_worker NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'amiary_security_definer') THEN + CREATE ROLE amiary_security_definer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION BYPASSRLS; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'makepad_backup') THEN + CREATE ROLE makepad_backup LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'makepad_backup_reader') THEN + CREATE ROLE makepad_backup_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION BYPASSRLS; + END IF; +END; +$$; +ALTER ROLE amiary_api NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; +ALTER ROLE amiary_worker NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; +ALTER ROLE amiary_security_definer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION BYPASSRLS; +ALTER ROLE makepad_backup LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; +SELECT format('ALTER ROLE %I PASSWORD %L', 'makepad_backup', :'makepad_backup_password') +WHERE NOT EXISTS (SELECT 1 FROM amiary_existing_logins WHERE rolname='makepad_backup') \gexec +ALTER ROLE makepad_backup_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION BYPASSRLS; +ALTER ROLE makepad_backup SET statement_timeout = '30min'; +ALTER ROLE makepad_backup SET idle_in_transaction_session_timeout = '30s'; + +-- Production login roles. The migrator owns the database; runtime roles only +-- inherit their matching capability role and receive direct CONNECT. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'amiary_migrator') THEN + CREATE ROLE amiary_migrator LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'amiary_api_prod') THEN + CREATE ROLE amiary_api_prod LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION NOBYPASSRLS; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'amiary_worker_prod') THEN + CREATE ROLE amiary_worker_prod LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION NOBYPASSRLS; + END IF; +END; +$$; +ALTER ROLE amiary_migrator LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; +SELECT format('ALTER ROLE %I PASSWORD %L', 'amiary_migrator', :'amiary_migrator_password') +WHERE NOT EXISTS (SELECT 1 FROM amiary_existing_logins WHERE rolname='amiary_migrator') \gexec +ALTER ROLE amiary_api_prod LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION NOBYPASSRLS; +SELECT format('ALTER ROLE %I PASSWORD %L', 'amiary_api_prod', :'amiary_api_prod_password') +WHERE NOT EXISTS (SELECT 1 FROM amiary_existing_logins WHERE rolname='amiary_api_prod') \gexec +ALTER ROLE amiary_worker_prod LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION NOBYPASSRLS; +SELECT format('ALTER ROLE %I PASSWORD %L', 'amiary_worker_prod', :'amiary_worker_prod_password') +WHERE NOT EXISTS (SELECT 1 FROM amiary_existing_logins WHERE rolname='amiary_worker_prod') \gexec + +ALTER ROLE amiary_api_prod SET statement_timeout = '30s'; +ALTER ROLE amiary_api_prod SET idle_in_transaction_session_timeout = '30s'; +ALTER ROLE amiary_worker_prod SET statement_timeout = '5min'; +ALTER ROLE amiary_worker_prod SET idle_in_transaction_session_timeout = '30s'; + +-- Canary login roles are distinct credentials and cannot connect to production. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'amiary_canary_migrator') THEN + CREATE ROLE amiary_canary_migrator LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'amiary_api_canary') THEN + CREATE ROLE amiary_api_canary LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION NOBYPASSRLS; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'amiary_worker_canary') THEN + CREATE ROLE amiary_worker_canary LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION NOBYPASSRLS; + END IF; +END; +$$; +ALTER ROLE amiary_canary_migrator LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; +SELECT format('ALTER ROLE %I PASSWORD %L', 'amiary_canary_migrator', :'amiary_canary_migrator_password') +WHERE NOT EXISTS (SELECT 1 FROM amiary_existing_logins WHERE rolname='amiary_canary_migrator') \gexec +ALTER ROLE amiary_api_canary LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION NOBYPASSRLS; +SELECT format('ALTER ROLE %I PASSWORD %L', 'amiary_api_canary', :'amiary_api_canary_password') +WHERE NOT EXISTS (SELECT 1 FROM amiary_existing_logins WHERE rolname='amiary_api_canary') \gexec +ALTER ROLE amiary_worker_canary LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION NOBYPASSRLS; +SELECT format('ALTER ROLE %I PASSWORD %L', 'amiary_worker_canary', :'amiary_worker_canary_password') +WHERE NOT EXISTS (SELECT 1 FROM amiary_existing_logins WHERE rolname='amiary_worker_canary') \gexec + +ALTER ROLE amiary_api_canary SET statement_timeout = '30s'; +ALTER ROLE amiary_api_canary SET idle_in_transaction_session_timeout = '30s'; +ALTER ROLE amiary_worker_canary SET statement_timeout = '5min'; +ALTER ROLE amiary_worker_canary SET idle_in_transaction_session_timeout = '30s'; + +-- Membership options are explicit: runtime privileges are inherited without +-- SET ROLE; only migrators may deliberately SET ROLE to the definer owner. +WITH desired(granted_role, member_role, inherit_membership, set_membership) AS ( + VALUES + ('amiary_api', 'amiary_api_prod', true, false), + ('amiary_api', 'amiary_api_canary', true, false), + ('amiary_worker', 'amiary_worker_prod', true, false), + ('amiary_worker', 'amiary_worker_canary', true, false), + ('amiary_security_definer', 'amiary_migrator', false, true), + ('amiary_security_definer', 'amiary_canary_migrator', false, true), + ('pg_read_all_data', 'makepad_backup_reader', true, false), + ('makepad_backup_reader', 'makepad_backup', false, true) +) +SELECT format( + 'GRANT %I TO %I WITH ADMIN FALSE, INHERIT %s, SET %s', + desired.granted_role, + desired.member_role, + upper(desired.inherit_membership::text), + upper(desired.set_membership::text) +) +FROM desired +WHERE NOT EXISTS ( + SELECT 1 + FROM pg_auth_members membership + JOIN pg_roles granted ON granted.oid = membership.roleid + JOIN pg_roles member ON member.oid = membership.member + WHERE granted.rolname = desired.granted_role + AND member.rolname = desired.member_role + AND NOT membership.admin_option + AND membership.inherit_option = desired.inherit_membership + AND membership.set_option = desired.set_membership +) \gexec + +-- Remove every undeclared membership held by an Amiary login. This repairs +-- pre-existing cross-application role drift, not only drift involving the +-- three Amiary capability roles. Conditional generation keeps reruns quiet. +WITH allowed(granted_role, member_role) AS ( + VALUES + ('amiary_api', 'amiary_api_prod'), + ('amiary_api', 'amiary_api_canary'), + ('amiary_worker', 'amiary_worker_prod'), + ('amiary_worker', 'amiary_worker_canary'), + ('amiary_security_definer', 'amiary_migrator'), + ('amiary_security_definer', 'amiary_canary_migrator'), + ('pg_read_all_data', 'makepad_backup_reader'), + ('makepad_backup_reader', 'makepad_backup') +) +SELECT format('REVOKE %I FROM %I', granted.rolname, member.rolname) +FROM pg_auth_members membership +JOIN pg_roles granted ON granted.oid = membership.roleid +JOIN pg_roles member ON member.oid = membership.member +WHERE member.rolname IN ( + 'amiary_migrator', + 'amiary_api_prod', + 'amiary_worker_prod', + 'amiary_canary_migrator', + 'amiary_api_canary', + 'amiary_worker_canary', + 'keycloak_amiary_app', + 'makepad_backup', + 'makepad_backup_reader' + ) + AND NOT EXISTS ( + SELECT 1 + FROM allowed + WHERE allowed.granted_role = granted.rolname + AND allowed.member_role = member.rolname + ) +ORDER BY granted.rolname, member.rolname +\gexec + +SELECT 'CREATE DATABASE amiary OWNER amiary_migrator' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'amiary') \gexec +SELECT 'ALTER DATABASE amiary OWNER TO amiary_migrator' +WHERE EXISTS ( + SELECT 1 FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'amiary' AND r.rolname <> 'amiary_migrator' +) \gexec +REVOKE ALL PRIVILEGES ON DATABASE amiary FROM PUBLIC; +REVOKE ALL PRIVILEGES ON DATABASE amiary FROM amiary_api, amiary_worker, amiary_security_definer, amiary_api_prod, amiary_worker_prod, amiary_canary_migrator, amiary_api_canary, amiary_worker_canary; +GRANT CONNECT ON DATABASE amiary TO amiary_migrator, amiary_api_prod, amiary_worker_prod; +ALTER DATABASE amiary SET timezone TO 'UTC'; + +SELECT 'CREATE DATABASE amiary_canary OWNER amiary_canary_migrator' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'amiary_canary') \gexec +SELECT 'ALTER DATABASE amiary_canary OWNER TO amiary_canary_migrator' +WHERE EXISTS ( + SELECT 1 FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'amiary_canary' AND r.rolname <> 'amiary_canary_migrator' +) \gexec +REVOKE ALL PRIVILEGES ON DATABASE amiary_canary FROM PUBLIC; +REVOKE ALL PRIVILEGES ON DATABASE amiary_canary FROM amiary_api, amiary_worker, amiary_security_definer, amiary_migrator, amiary_api_prod, amiary_worker_prod, amiary_api_canary, amiary_worker_canary; +GRANT CONNECT ON DATABASE amiary_canary TO amiary_canary_migrator, amiary_api_canary, amiary_worker_canary; +ALTER DATABASE amiary_canary SET timezone TO 'UTC'; + +-- Preserve the dedicated Keycloak role/database pair. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'keycloak_amiary_app') THEN + CREATE ROLE keycloak_amiary_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; + END IF; +END; +$$; +ALTER ROLE keycloak_amiary_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; +SELECT format('ALTER ROLE %I PASSWORD %L', 'keycloak_amiary_app', :'keycloak_amiary_app_password') +WHERE NOT EXISTS (SELECT 1 FROM amiary_existing_logins WHERE rolname='keycloak_amiary_app') \gexec +REVOKE ALL PRIVILEGES ON DATABASE amiary FROM keycloak_amiary_app; +REVOKE ALL PRIVILEGES ON DATABASE amiary_canary FROM keycloak_amiary_app; +SELECT 'CREATE DATABASE keycloak_amiary OWNER keycloak_amiary_app' +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'keycloak_amiary') \gexec +SELECT 'ALTER DATABASE keycloak_amiary OWNER TO keycloak_amiary_app' +WHERE EXISTS ( + SELECT 1 FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba + WHERE d.datname = 'keycloak_amiary' AND r.rolname <> 'keycloak_amiary_app' +) \gexec +REVOKE ALL PRIVILEGES ON DATABASE keycloak_amiary FROM PUBLIC; +REVOKE ALL PRIVILEGES ON DATABASE keycloak_amiary FROM amiary_api, amiary_worker, amiary_security_definer, amiary_migrator, amiary_api_prod, amiary_worker_prod, amiary_canary_migrator, amiary_api_canary, amiary_worker_canary; +GRANT CONNECT ON DATABASE keycloak_amiary TO keycloak_amiary_app; +ALTER DATABASE keycloak_amiary SET timezone TO 'UTC'; + +-- The scheduled backup login never receives superuser or write privileges. It +-- connects only to the five protected databases and pg_dump explicitly uses +-- SET ROLE makepad_backup_reader. That NOLOGIN role has read-only data access +-- and BYPASSRLS (required for a complete forced-RLS backup); the login is +-- NOINHERIT and has neither capability at all other times. +SELECT format('REVOKE ALL PRIVILEGES ON DATABASE %I FROM makepad_backup', database.datname) +FROM pg_database database +WHERE database.datname IN ('runtrace', 'keycloak_runtrace', 'amiary', 'amiary_canary', 'keycloak_amiary') +ORDER BY database.datname +\gexec +SELECT format('GRANT CONNECT ON DATABASE %I TO makepad_backup', database.datname) +FROM pg_database database +WHERE database.datname IN ('runtrace', 'keycloak_runtrace', 'amiary', 'amiary_canary', 'keycloak_amiary') +ORDER BY database.datname +\gexec + +SELECT pg_advisory_unlock(hashtext('makepad-postgres'), hashtext('amiary-bootstrap')); diff --git a/config/runtrace-pg_hba.conf b/config/runtrace-pg_hba.conf index 022cf69..bcf8ee3 100644 --- a/config/runtrace-pg_hba.conf +++ b/config/runtrace-pg_hba.conf @@ -2,6 +2,14 @@ # Jotwink and BetaCrew are source-restricted to the app WireGuard peer and Keycloak host. # Other shared databases retain their current SCRAM policy during migration. local all all trust +hostnossl amiary all all reject +hostnossl amiary_canary all all reject +hostnossl keycloak_amiary all all reject +hostssl amiary all all scram-sha-256 +hostssl amiary_canary all all scram-sha-256 +hostssl keycloak_amiary all all scram-sha-256 +hostssl runtrace,keycloak_runtrace makepad_backup all scram-sha-256 +host all amiary_migrator,amiary_api_prod,amiary_worker_prod,amiary_canary_migrator,amiary_api_canary,amiary_worker_canary,keycloak_amiary_app,makepad_backup all reject hostnossl runtrace all all reject hostnossl keycloak_runtrace all all reject hostnossl brio_staging all all reject diff --git a/docs/postgresql-18-upgrade.md b/docs/postgresql-18-upgrade.md new file mode 100644 index 0000000..5d8be5c --- /dev/null +++ b/docs/postgresql-18-upgrade.md @@ -0,0 +1,39 @@ +# PostgreSQL 18 major-version upgrade + +The repository is pinned to PostgreSQL 18. `scripts/preflight-postgres-major.sh` +refuses to start that image against a data directory initialized by an older +major version. A normal deploy is not an upgrade mechanism. + +## Required gate + +Schedule a maintenance window and assign an operator plus reviewer. Before +stopping writers, require all of the following: + +- a fresh certificate-verified logical backup of every database, including + `amiary`, `amiary_canary`, and `keycloak_amiary`; +- verified checksums and `pg_restore --list` for every custom-format archive; +- a successful isolated restore drill using the PostgreSQL 18 client/server; +- a separately protected globals export and an inventory of extensions; +- measured rollback time within the four-hour RTO; and +- a filesystem-level snapshot of the PostgreSQL 16 data directory retained + unchanged until the PostgreSQL 18 validation window closes. + +## Cutover outline + +1. Disable application writes and stop all database clients and backup jobs. +2. Take and verify a final logical backup and globals export over + certificate-verified TLS. Treat globals and dumps as secrets. +3. Move the PostgreSQL 16 data directory to a narrow, timestamped rollback + path. Never overwrite or delete it during the cutover. +4. Create a new empty directory at the configured data path with the existing + PostgreSQL uid/gid and mode, then run the major-version preflight. +5. Start the pinned PostgreSQL 18 image, restore reviewed globals and all + database archives, and re-run every idempotent application bootstrap. +6. Run ownership, grants, RLS isolation, migration checksum, TLS/HBA, backup, + application smoke, and latency checks before enabling writers. +7. Re-enable clients gradually. Roll back to the untouched PostgreSQL 16 + directory if any acceptance gate fails. + +Record image digests, backup identifiers, checksums, commands, timings, +reviewer approval, and restore evidence. Do not delete the PostgreSQL 16 +rollback snapshot until the retention decision is explicitly reviewed. diff --git a/scripts/preflight-postgres-major.sh b/scripts/preflight-postgres-major.sh new file mode 100755 index 0000000..3b24812 --- /dev/null +++ b/scripts/preflight-postgres-major.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +data_path=${1:?Usage: preflight-postgres-major.sh [expected-major]} +expected_major=${2:-18} + +if [[ "${data_path}" != /* || "${data_path}" == / || "${data_path}" == /var || "${data_path}" == /var/lib ]]; then + echo "PostgreSQL data path must be a narrow absolute path." >&2 + exit 1 +fi +if [[ ! "${expected_major}" =~ ^[0-9]+$ ]]; then + echo "Expected PostgreSQL major must be numeric." >&2 + exit 1 +fi +if [[ -L "${data_path}" ]]; then + echo "PostgreSQL data path must not be a symlink: ${data_path}" >&2 + exit 1 +fi +if [[ -e "${data_path}" && ! -d "${data_path}" ]]; then + echo "PostgreSQL data path is not a directory: ${data_path}" >&2 + exit 1 +fi + +version_file=${data_path}/PG_VERSION +if [[ ! -e "${version_file}" ]]; then + # A missing directory or genuinely empty/new data directory is initialized by + # the pinned image. Non-empty unrecognized directories are never accepted. + if [[ -d "${data_path}" && -n "$(find "${data_path}" -mindepth 1 -maxdepth 1 -print -quit)" ]]; then + echo "Non-empty PostgreSQL data path has no PG_VERSION marker: ${data_path}" >&2 + exit 1 + fi + exit 0 +fi +if [[ ! -f "${version_file}" || -L "${version_file}" ]]; then + echo "PostgreSQL PG_VERSION must be a regular non-symlink file." >&2 + exit 1 +fi + +actual_major=$(tr -d '[:space:]' < "${version_file}") +if [[ "${actual_major}" != "${expected_major}" ]]; then + echo "Refusing PostgreSQL ${expected_major} against data initialized by PostgreSQL ${actual_major}. Complete docs/postgresql-18-upgrade.md first." >&2 + exit 1 +fi diff --git a/scripts/run-amiary-shared-backup.sh b/scripts/run-amiary-shared-backup.sh new file mode 100755 index 0000000..88f9bc9 --- /dev/null +++ b/scripts/run-amiary-shared-backup.sh @@ -0,0 +1,113 @@ +#!/bin/sh +set -eu + +umask 077 + +backup_root=${RUNTRACE_BACKUP_ROOT:-/backups} +password_file=${POSTGRES_BACKUP_PASSWORD_FILE:-/run/secrets/postgres_backup_password} +retention_days=${RUNTRACE_BACKUP_RETENTION_DAYS:-35} +pg_host=${PGHOST:-makepad-postgres} +pg_port=${PGPORT:-5432} +pg_user=${PGUSER:-makepad_backup} + +case "${retention_days}" in + ''|*[!0-9]*) + echo "RUNTRACE_BACKUP_RETENTION_DAYS must be a positive integer." >&2 + exit 1 + ;; +esac +if [ "${retention_days}" -lt 1 ]; then + echo "RUNTRACE_BACKUP_RETENTION_DAYS must be at least 1." >&2 + exit 1 +fi +if [ ! -s "${password_file}" ]; then + echo "PostgreSQL backup credential file is missing or empty." >&2 + exit 1 +fi + +mkdir -p "${backup_root}" +chmod 700 "${backup_root}" +lock_dir=${backup_root}/.runtrace-backup.lock +if ! mkdir "${lock_dir}" 2>/dev/null; then + echo "Another Runtrace PostgreSQL backup is already running." >&2 + exit 1 +fi + +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +partial_dir=${backup_root}/.${timestamp}.partial +final_dir=${backup_root}/${timestamp} +pgpass=${backup_root}/.pgpass.${timestamp} +if [ -e "${final_dir}" ] || [ -L "${final_dir}" ]; then + echo "Backup destination already exists for ${timestamp}." >&2 + exit 1 +fi + +cleanup() { + if [ -d "${partial_dir}" ]; then + find "${partial_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${partial_dir}" 2>/dev/null || true + fi + if [ -f "${pgpass}" ]; then + find "${pgpass}" -delete 2>/dev/null || true + fi + rmdir "${lock_dir}" 2>/dev/null || true +} +trap cleanup EXIT HUP INT TERM + +password=$(tr -d '\r\n' < "${password_file}") +if [ -z "${password}" ]; then + echo "PostgreSQL backup credential file contains no usable value." >&2 + exit 1 +fi +escaped_password=$(printf '%s' "${password}" | sed 's/\\/\\\\/g; s/:/\\:/g') +printf '%s:%s:*:%s:%s\n' "${pg_host}" "${pg_port}" "${pg_user}" "${escaped_password}" > "${pgpass}" +unset password escaped_password +chmod 600 "${pgpass}" +export PGPASSFILE="${pgpass}" +export PGSSLMODE="${PGSSLMODE:-verify-full}" +export PGSSLROOTCERT="${PGSSLROOTCERT:-/etc/postgresql/ca.crt}" + +mkdir "${partial_dir}" +databases='runtrace keycloak_runtrace amiary amiary_canary keycloak_amiary' +for database in ${databases}; do + dump_path=${partial_dir}/${database}.dump + pg_dump \ + --host="${pg_host}" \ + --port="${pg_port}" \ + --username="${pg_user}" \ + --role=makepad_backup_reader \ + --dbname="${database}" \ + --format=custom \ + --compress=9 \ + --file="${dump_path}" + pg_restore --list "${dump_path}" >/dev/null +done + +( + cd "${partial_dir}" + sha256sum runtrace.dump keycloak_runtrace.dump amiary.dump amiary_canary.dump keycloak_amiary.dump > SHA256SUMS +) +cat > "${partial_dir}/metadata.json" <&2 + exit 1 +fi +ln -sfn "${timestamp}" "${backup_root}/latest" + +status_tmp=${backup_root}/.last-success.json.tmp +printf '{"createdAt":"%s","backup":"%s","databases":["runtrace","keycloak_runtrace","amiary","amiary_canary","keycloak_amiary"]}\n' "${timestamp}" "${timestamp}" > "${status_tmp}" +chmod 600 "${status_tmp}" +mv "${status_tmp}" "${backup_root}/last-success.json" + +find "${backup_root}" -mindepth 1 -maxdepth 1 -type d -name '20??????T??????Z' -mtime "+${retention_days}" -exec sh -c ' + for directory do + find "$directory" -mindepth 1 -delete + rmdir "$directory" + done +' sh {} + + +echo "Shared PostgreSQL backup completed: ${final_dir}" diff --git a/scripts/run-betacrew-backup-loop.sh b/scripts/run-betacrew-backup-loop.sh index fc351dd..08594fa 100755 --- a/scripts/run-betacrew-backup-loop.sh +++ b/scripts/run-betacrew-backup-loop.sh @@ -8,7 +8,10 @@ backup_root=${BETACREW_BACKUP_ROOT:-/backups} case "${interval_seconds}:${retry_seconds}" in *[!0-9:]*) echo "BetaCrew backup timing values must be positive integers." >&2; exit 1 ;; esac -[ "${interval_seconds}" -ge 300 ] && [ "${retry_seconds}" -ge 30 ] || { echo "BetaCrew backup interval or retry delay is too short." >&2; exit 1; } +if ! [ "${interval_seconds}" -ge 300 ] || ! [ "${retry_seconds}" -ge 30 ]; then + echo "BetaCrew backup interval or retry delay is too short." >&2 + exit 1 +fi if [ "${1:-}" = healthcheck ]; then status_file=${backup_root}/last-success.json diff --git a/scripts/run-betacrew-backup.sh b/scripts/run-betacrew-backup.sh index c7442f0..d4a11a7 100755 --- a/scripts/run-betacrew-backup.sh +++ b/scripts/run-betacrew-backup.sh @@ -39,7 +39,10 @@ cleanup() { } trap cleanup EXIT HUP INT TERM -[ ! -e "${final_dir}" ] && [ ! -L "${final_dir}" ] || { echo "Backup destination already exists." >&2; exit 1; } +if [ -e "${final_dir}" ] || [ -L "${final_dir}" ]; then + echo "Backup destination already exists." >&2 + exit 1 +fi password=$(tr -d '\r\n' < "${password_file}") [ -n "${password}" ] || { echo "PostgreSQL backup credential contains no usable value." >&2; exit 1; } escaped_password=$(printf '%s' "${password}" | sed 's/\\/\\\\/g; s/:/\\:/g') diff --git a/scripts/run-jotwink-backup.sh b/scripts/run-jotwink-backup.sh index 263fc8d..1618316 100755 --- a/scripts/run-jotwink-backup.sh +++ b/scripts/run-jotwink-backup.sh @@ -48,7 +48,10 @@ cleanup() { } trap cleanup EXIT HUP INT TERM -[ ! -e "${final_dir}" ] && [ ! -L "${final_dir}" ] || { echo "Backup destination already exists for ${timestamp}." >&2; exit 1; } +if [ -e "${final_dir}" ] || [ -L "${final_dir}" ]; then + echo "Backup destination already exists for ${timestamp}." >&2 + exit 1 +fi password=$(tr -d '\r\n' < "${password_file}") [ -n "${password}" ] || { echo "PostgreSQL backup credential file contains no usable value." >&2; exit 1; } escaped_password=$(printf '%s' "${password}" | sed 's/\\/\\\\/g; s/:/\\:/g') diff --git a/scripts/test-amiary-bootstrap.sh b/scripts/test-amiary-bootstrap.sh new file mode 100755 index 0000000..73f5673 --- /dev/null +++ b/scripts/test-amiary-bootstrap.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +set -euo pipefail + +command -v docker >/dev/null 2>&1 || { echo "docker is required" >&2; exit 1; } +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "${script_dir}/.." && pwd) +container="amiary-postgres-bootstrap-test-$$" + +cleanup() { + docker rm -f "${container}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +docker run -d --rm --name "${container}" \ + -e POSTGRES_PASSWORD=test-superuser-password \ + -v "${repo_root}/bootstrap/amiary-apps.sql:/bootstrap/amiary-apps.sql:ro" \ + postgres:18@sha256:4ef4dbc939d61acea57712655ddb4b4ab27419c913f94cca0cd57cb3ea3c2280 >/dev/null + +for _ in $(seq 1 30); do + if docker exec "${container}" pg_isready -U postgres -d postgres >/dev/null 2>&1; then + break + fi + sleep 1 +done +docker exec "${container}" pg_isready -U postgres -d postgres >/dev/null +docker exec "${container}" createdb -U postgres runtrace +docker exec "${container}" createdb -U postgres keycloak_runtrace + +bootstrap() { + docker exec "${container}" psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -v amiary_migrator_password=test-prod-migrator-password \ + -v amiary_api_prod_password=test-prod-api-password \ + -v amiary_worker_prod_password=test-prod-worker-password \ + -v amiary_canary_migrator_password=test-canary-migrator-password \ + -v amiary_api_canary_password=test-canary-api-password \ + -v amiary_worker_canary_password=test-canary-worker-password \ + -v keycloak_amiary_app_password=test-keycloak-password \ + -v makepad_backup_password=test-backup-password \ + -f /bootstrap/amiary-apps.sql >/dev/null +} + +bootstrap +password_fingerprint=$(docker exec "${container}" psql -At -U postgres -c "SELECT md5(string_agg(rolpassword, ',' ORDER BY rolname)) FROM pg_authid WHERE rolname LIKE 'amiary_%' OR rolname IN ('keycloak_amiary_app','makepad_backup')") + +# Introduce representative ownership, membership, and role-attribute drift. A +# second run must converge back to the declared least-privilege state. +docker exec "${container}" psql -v ON_ERROR_STOP=1 -U postgres -d postgres >/dev/null <<'SQL' +ALTER ROLE amiary_api LOGIN SUPERUSER CREATEDB CREATEROLE INHERIT REPLICATION BYPASSRLS; +ALTER ROLE amiary_api_prod SUPERUSER CREATEDB CREATEROLE NOINHERIT REPLICATION BYPASSRLS; +ALTER ROLE amiary_security_definer LOGIN SUPERUSER CREATEDB CREATEROLE INHERIT REPLICATION; +ALTER DATABASE amiary OWNER TO amiary_api_prod; +GRANT ALL PRIVILEGES ON DATABASE amiary_canary TO amiary_api_prod; +GRANT amiary_security_definer TO amiary_api_prod WITH ADMIN TRUE, INHERIT TRUE, SET TRUE; +CREATE ROLE unrelated_application_role NOLOGIN; +GRANT unrelated_application_role TO amiary_api_prod; +GRANT unrelated_application_role TO makepad_backup; +GRANT unrelated_application_role TO makepad_backup_reader; +ALTER ROLE makepad_backup SUPERUSER CREATEDB CREATEROLE INHERIT REPLICATION BYPASSRLS; +SQL + +bootstrap +after_fingerprint=$(docker exec "${container}" psql -At -U postgres -c "SELECT md5(string_agg(rolpassword, ',' ORDER BY rolname)) FROM pg_authid WHERE rolname LIKE 'amiary_%' OR rolname IN ('keycloak_amiary_app','makepad_backup')") +[[ "$password_fingerprint" == "$after_fingerprint" ]] || { echo "Bootstrap reset existing credentials" >&2; exit 1; } + +roles=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "SELECT rolname, rolcanlogin, rolsuper, rolcreatedb, rolcreaterole, rolinherit, rolreplication, rolbypassrls FROM pg_roles WHERE rolname IN ('amiary_api','amiary_worker','amiary_security_definer','amiary_migrator','amiary_api_prod','amiary_worker_prod','amiary_canary_migrator','amiary_api_canary','amiary_worker_canary','keycloak_amiary_app','makepad_backup','makepad_backup_reader') ORDER BY rolname") +expected_roles=$'amiary_api|f|f|f|f|f|f|f\namiary_api_canary|t|f|f|f|t|f|f\namiary_api_prod|t|f|f|f|t|f|f\namiary_canary_migrator|t|f|f|f|f|f|f\namiary_migrator|t|f|f|f|f|f|f\namiary_security_definer|f|f|f|f|f|f|t\namiary_worker|f|f|f|f|f|f|f\namiary_worker_canary|t|f|f|f|t|f|f\namiary_worker_prod|t|f|f|f|t|f|f\nkeycloak_amiary_app|t|f|f|f|f|f|f\nmakepad_backup|t|f|f|f|f|f|f\nmakepad_backup_reader|f|f|f|f|t|f|t' +test "${roles}" = "${expected_roles}" || { printf 'unexpected role policy:\n%s\n' "${roles}" >&2; exit 1; } + +memberships=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "SELECT granted.rolname, member.rolname, membership.admin_option, membership.inherit_option, membership.set_option FROM pg_auth_members membership JOIN pg_roles granted ON granted.oid = membership.roleid JOIN pg_roles member ON member.oid = membership.member WHERE granted.rolname IN ('amiary_api','amiary_worker','amiary_security_definer','pg_read_all_data','makepad_backup_reader') AND member.rolname IN ('amiary_api_prod','amiary_api_canary','amiary_worker_prod','amiary_worker_canary','amiary_migrator','amiary_canary_migrator','makepad_backup','makepad_backup_reader') ORDER BY granted.rolname, member.rolname") +expected_memberships=$'amiary_api|amiary_api_canary|f|t|f\namiary_api|amiary_api_prod|f|t|f\namiary_security_definer|amiary_canary_migrator|f|f|t\namiary_security_definer|amiary_migrator|f|f|t\namiary_worker|amiary_worker_canary|f|t|f\namiary_worker|amiary_worker_prod|f|t|f\nmakepad_backup_reader|makepad_backup|f|f|t\npg_read_all_data|makepad_backup_reader|f|t|f' +test "${memberships}" = "${expected_memberships}" || { printf 'unexpected role membership policy:\n%s\n' "${memberships}" >&2; exit 1; } +undeclared_memberships=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "SELECT count(*) FROM pg_auth_members membership JOIN pg_roles member ON member.oid = membership.member WHERE member.rolname IN ('amiary_migrator','amiary_api_prod','amiary_worker_prod','amiary_canary_migrator','amiary_api_canary','amiary_worker_canary','keycloak_amiary_app','makepad_backup','makepad_backup_reader') AND NOT EXISTS (SELECT 1 FROM (VALUES ('amiary_api','amiary_api_prod'),('amiary_api','amiary_api_canary'),('amiary_worker','amiary_worker_prod'),('amiary_worker','amiary_worker_canary'),('amiary_security_definer','amiary_migrator'),('amiary_security_definer','amiary_canary_migrator'),('pg_read_all_data','makepad_backup_reader'),('makepad_backup_reader','makepad_backup')) allowed(granted_role,member_role) JOIN pg_roles granted ON granted.rolname = allowed.granted_role WHERE granted.oid = membership.roleid AND allowed.member_role = member.rolname)") +test "${undeclared_memberships}" = 0 || { echo "Amiary login retains an undeclared role membership" >&2; exit 1; } + +databases=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "SELECT database.datname, owner.rolname FROM pg_database database JOIN pg_roles owner ON owner.oid = database.datdba WHERE database.datname IN ('amiary','amiary_canary','keycloak_amiary') ORDER BY database.datname") +expected_databases=$'amiary|amiary_migrator\namiary_canary|amiary_canary_migrator\nkeycloak_amiary|keycloak_amiary_app' +test "${databases}" = "${expected_databases}" || { printf 'unexpected database ownership:\n%s\n' "${databases}" >&2; exit 1; } + +runtime_database_privileges=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "WITH expected(role_name, database_name, may_connect) AS (VALUES ('amiary_api_prod','amiary',true), ('amiary_api_prod','amiary_canary',false), ('amiary_worker_prod','amiary',true), ('amiary_worker_prod','amiary_canary',false), ('amiary_api_canary','amiary',false), ('amiary_api_canary','amiary_canary',true), ('amiary_worker_canary','amiary',false), ('amiary_worker_canary','amiary_canary',true)) SELECT count(*) FROM expected WHERE has_database_privilege(role_name, database_name, 'CONNECT') IS DISTINCT FROM may_connect OR has_database_privilege(role_name, database_name, 'CREATE') OR has_database_privilege(role_name, database_name, 'TEMPORARY')") +test "${runtime_database_privileges}" = 0 || { echo "runtime database privileges are too broad" >&2; exit 1; } + +shared_role_connect=$(docker exec "${container}" psql -At -U postgres -d postgres -c \ + "SELECT count(*) FROM (VALUES ('amiary_api'), ('amiary_worker'), ('amiary_security_definer')) AS roles(role_name) CROSS JOIN (VALUES ('amiary'), ('amiary_canary'), ('keycloak_amiary')) AS databases(database_name) WHERE has_database_privilege(role_name, database_name, 'CONNECT')") +test "${shared_role_connect}" = 0 || { echo "NOLOGIN capability role unexpectedly has database CONNECT" >&2; exit 1; } + +connect_as() { + local role=$1 + local password=$2 + local database=$3 + docker exec -e PGPASSWORD="${password}" "${container}" \ + psql -h 127.0.0.1 -At -v ON_ERROR_STOP=1 -U "${role}" -d "${database}" -c 'SELECT current_user' 2>/dev/null +} + +test "$(connect_as amiary_api_prod test-prod-api-password amiary)" = amiary_api_prod +test "$(connect_as amiary_worker_prod test-prod-worker-password amiary)" = amiary_worker_prod +test "$(connect_as amiary_api_canary test-canary-api-password amiary_canary)" = amiary_api_canary +test "$(connect_as amiary_worker_canary test-canary-worker-password amiary_canary)" = amiary_worker_canary +test "$(connect_as keycloak_amiary_app test-keycloak-password keycloak_amiary)" = keycloak_amiary_app +test "$(connect_as makepad_backup test-backup-password amiary)" = makepad_backup + +docker exec -e PGPASSWORD=test-backup-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U makepad_backup -d amiary \ + -c 'SET ROLE makepad_backup_reader; SELECT current_role' >/dev/null +if docker exec -e PGPASSWORD=test-backup-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U makepad_backup -d amiary \ + -c 'CREATE TABLE public.backup_privilege_escape (id bigint)' >/dev/null 2>&1; then + echo "backup login can create schema objects" >&2 + exit 1 +fi +if docker exec -e PGPASSWORD=test-backup-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U makepad_backup -d amiary \ + -c 'ALTER ROLE makepad_backup SUPERUSER' >/dev/null 2>&1; then + echo "backup login can alter roles" >&2 + exit 1 +fi + +if connect_as amiary_api_prod test-prod-api-password amiary_canary >/dev/null; then + echo "production API login connected to canary" >&2 + exit 1 +fi +if connect_as amiary_worker_canary test-canary-worker-password amiary >/dev/null; then + echo "canary worker login connected to production" >&2 + exit 1 +fi +if connect_as keycloak_amiary_app test-keycloak-password amiary >/dev/null; then + echo "Keycloak login connected to the Amiary application database" >&2 + exit 1 +fi + +if docker exec -e PGPASSWORD=test-prod-api-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U amiary_api_prod -d amiary \ + -c 'SET ROLE amiary_security_definer' >/dev/null 2>&1; then + echo "API login can SET ROLE to the RLS-bypass owner" >&2 + exit 1 +fi + +docker exec -e PGPASSWORD=test-prod-migrator-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U amiary_migrator -d amiary \ + -c 'SET ROLE amiary_security_definer' >/dev/null + +if docker exec -e PGPASSWORD=test-prod-api-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U amiary_api_prod -d amiary \ + -c 'CREATE TABLE public.privilege_escape_test (id bigint)' >/dev/null 2>&1; then + echo "API login can create schema objects" >&2 + exit 1 +fi + +docker exec -e PGPASSWORD=test-prod-migrator-password "${container}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U amiary_migrator -d amiary \ + -c 'CREATE TABLE public.migration_permission_test (id bigint); DROP TABLE public.migration_permission_test' >/dev/null + +echo "Amiary PostgreSQL bootstrap is idempotent, drift-repairing, and least-privilege." diff --git a/scripts/test-amiary-restore-guards.py b/scripts/test-amiary-restore-guards.py new file mode 100644 index 0000000..fe5bca5 --- /dev/null +++ b/scripts/test-amiary-restore-guards.py @@ -0,0 +1,25 @@ +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +ROOT=Path(__file__).resolve().parent +class RestoreGuards(unittest.TestCase): + def test_refuses_unsafe_targets_before_mutation(self): + for response, service in [('f','amiary_restore_test'),('error','amiary_restore_test'),('t','amiary_restore_test dbname=amiary')]: + with self.subTest(response=response,service=service),tempfile.TemporaryDirectory() as tmp: + root=Path(tmp); bindir=root/'bin';bindir.mkdir();backup=root/'backup';backup.mkdir() + for name in ('runtrace','keycloak_runtrace','amiary','amiary_canary','keycloak_amiary'):(backup/(name+'.dump')).write_text('fixture') + for name in ('SHA256SUMS','metadata.json'):(backup/name).write_text('fixture') + (root/'services').write_text('fixture') + (bindir/'sha256sum').write_text('#!/bin/sh\nexit 0\n') + (bindir/'psql').write_text('#!/bin/sh\ncat >/dev/null\nif [ "$TEST_RESPONSE" = error ]; then exit 1; fi\necho "$TEST_RESPONSE"\n') + (bindir/'pg_restore').write_text('#!/bin/sh\ntouch "$TEST_ROOT/mutation"\nexit 1\n') + for path in bindir.iterdir():path.chmod(0o700) + env=dict(os.environ,PATH=str(bindir)+os.pathsep+os.environ['PATH'],TEST_ROOT=tmp,TEST_RESPONSE=response,PGSERVICEFILE=str(root/'services'),RUNTRACE_RESTORE_CONFIRM='replace-nonproduction-restore-targets') + for name in ('RUNTRACE','KEYCLOAK_RUNTRACE','AMIARY','AMIARY_CANARY','KEYCLOAK_AMIARY'):env[name+'_RESTORE_SERVICE']=name.lower()+'_restore_test' + env['AMIARY_RESTORE_SERVICE']=service + result=subprocess.run(['bash',str(ROOT/'verify-amiary-shared-restore.sh'),str(backup)],env=env,capture_output=True) + self.assertNotEqual(result.returncode,0) + self.assertFalse((root/'mutation').exists(),'Unsafe target reached pg_restore') +if __name__=='__main__':unittest.main() diff --git a/scripts/test-amiary-shared-backup.sh b/scripts/test-amiary-shared-backup.sh new file mode 100755 index 0000000..c7608cb --- /dev/null +++ b/scripts/test-amiary-shared-backup.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) + +cleanup() { + find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${work_dir}" 2>/dev/null || true +} +trap cleanup EXIT + +mkdir -p "${work_dir}/bin" "${work_dir}/backups" +printf 'validation-password\n' > "${work_dir}/password" +chmod 600 "${work_dir}/password" + +cat > "${work_dir}/bin/pg_dump" <<'EOF' +#!/bin/sh +set -eu +output= +database= +role= +for argument in "$@"; do + case "${argument}" in + --file=*) output=${argument#--file=} ;; + --dbname=*) database=${argument#--dbname=} ;; + --role=*) role=${argument#--role=} ;; + esac +done +[ -n "${output}" ] && [ -n "${database}" ] +[ "${role}" = makepad_backup_reader ] +printf 'validated dump for %s\n' "${database}" > "${output}" +EOF +cat > "${work_dir}/bin/pg_restore" <<'EOF' +#!/bin/sh +set -eu +[ "$1" = "--list" ] +[ -s "$2" ] +EOF +chmod 700 "${work_dir}/bin/pg_dump" "${work_dir}/bin/pg_restore" + +PATH="${work_dir}/bin:${PATH}" \ +RUNTRACE_BACKUP_ROOT="${work_dir}/backups" \ +POSTGRES_BACKUP_PASSWORD_FILE="${work_dir}/password" \ +PGUSER=makepad_backup \ +PGSSLROOTCERT="${work_dir}/unused-test-ca.crt" \ +sh "${repo_root}/scripts/run-amiary-shared-backup.sh" + +latest=$(readlink "${work_dir}/backups/latest") +backup_dir=${work_dir}/backups/${latest} +for expected in runtrace.dump keycloak_runtrace.dump amiary.dump amiary_canary.dump keycloak_amiary.dump SHA256SUMS metadata.json; do + test -s "${backup_dir}/${expected}" +done +( + cd "${backup_dir}" + sha256sum -c SHA256SUMS >/dev/null +) +test -s "${work_dir}/backups/last-success.json" +grep -q '"amiary"' "${backup_dir}/metadata.json" +grep -q '"amiary_canary"' "${backup_dir}/metadata.json" +grep -q '"keycloak_amiary"' "${backup_dir}/metadata.json" + +RUNTRACE_BACKUP_ROOT="${work_dir}/backups" \ +RUNTRACE_BACKUP_INTERVAL_SECONDS=300 \ +RUNTRACE_BACKUP_RETRY_SECONDS=30 \ +sh "${repo_root}/scripts/run-runtrace-backup-loop.sh" healthcheck + +echo "Runtrace PostgreSQL backup contract passed." diff --git a/scripts/test-betacrew-backup.sh b/scripts/test-betacrew-backup.sh index 3f0c406..613b4e5 100755 --- a/scripts/test-betacrew-backup.sh +++ b/scripts/test-betacrew-backup.sh @@ -38,7 +38,10 @@ backup_dir=${work_dir}/backups/${latest} for expected in betacrew.dump.cms keycloak_betacrew.dump.cms SHA256SUMS metadata.json; do test -s "${backup_dir}/${expected}"; done (cd "${backup_dir}" && sha256sum --check SHA256SUMS >/dev/null) for database in betacrew keycloak_betacrew; do - ! grep -a -q 'validated dump' "${backup_dir}/${database}.dump.cms" + if grep -a -q 'validated dump' "${backup_dir}/${database}.dump.cms"; then + echo 'Backup leaked plaintext fixture data.' >&2 + exit 1 + fi openssl cms -decrypt -binary -inform DER -in "${backup_dir}/${database}.dump.cms" \ -recip "${work_dir}/recipient.pem" -inkey "${work_dir}/recipient.key" -out "${work_dir}/${database}.decrypted" grep -q "validated dump for ${database}" "${work_dir}/${database}.decrypted" diff --git a/scripts/test-shared-restore.sh b/scripts/test-shared-restore.sh new file mode 100755 index 0000000..3db93ab --- /dev/null +++ b/scripts/test-shared-restore.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +set -euo pipefail + +command -v docker >/dev/null 2>&1 || { echo "docker is required" >&2; exit 1; } +command -v sha256sum >/dev/null 2>&1 || { echo "sha256sum is required" >&2; exit 1; } + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) +suffix=$$ +network="amiary-pg-restore-${suffix}" +server="amiary-pg-restore-${suffix}" +postgres_image='postgres:18-alpine@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2' +password='disposable-restore-password' + +cleanup() { + docker rm -f "${server}" >/dev/null 2>&1 || true + docker network rm "${network}" >/dev/null 2>&1 || true + find "${work_dir}" -mindepth 1 -delete 2>/dev/null || true + rmdir "${work_dir}" 2>/dev/null || true +} +trap cleanup EXIT + +docker network create "${network}" >/dev/null +docker run -d --rm --name "${server}" --network "${network}" --network-alias postgres-restore \ + -e POSTGRES_PASSWORD="${password}" "${postgres_image}" >/dev/null +for _ in $(seq 1 30); do + if docker exec "${server}" pg_isready -U postgres -d postgres >/dev/null 2>&1; then + break + fi + sleep 1 +done +docker exec "${server}" pg_isready -U postgres -d postgres >/dev/null + +docker exec -i "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d postgres >/dev/null <<'SQL' +CREATE ROLE runtrace_app NOLOGIN; +CREATE ROLE keycloak_runtrace_app NOLOGIN; +CREATE ROLE amiary_migrator NOLOGIN; +CREATE ROLE amiary_canary_migrator NOLOGIN; +CREATE ROLE amiary_security_definer NOLOGIN BYPASSRLS; +CREATE ROLE amiary_api NOLOGIN; +CREATE ROLE amiary_worker NOLOGIN; +CREATE ROLE keycloak_amiary_app NOLOGIN; +CREATE ROLE makepad_backup LOGIN PASSWORD 'disposable-backup-password' + NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS; +CREATE ROLE makepad_backup_reader NOLOGIN + NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION BYPASSRLS; +GRANT pg_read_all_data TO makepad_backup_reader WITH ADMIN FALSE, INHERIT TRUE, SET FALSE; +GRANT makepad_backup_reader TO makepad_backup WITH ADMIN FALSE, INHERIT FALSE, SET TRUE; +SQL + +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj /CN=postgres-restore \ + -addext subjectAltName=DNS:postgres-restore -keyout "${work_dir}/server.key" -out "${work_dir}/server.crt" >/dev/null 2>&1 +docker cp "${work_dir}/server.crt" "${server}:/tmp/server.crt" +docker cp "${work_dir}/server.key" "${server}:/tmp/server.key" +docker exec "${server}" chown postgres:postgres /tmp/server.key +docker exec "${server}" chmod 600 /tmp/server.key +docker exec -i "${server}" psql -v ON_ERROR_STOP=1 -U postgres >/dev/null <<'SQL' +ALTER SYSTEM SET ssl='on'; +ALTER SYSTEM SET ssl_cert_file='/tmp/server.crt'; +ALTER SYSTEM SET ssl_key_file='/tmp/server.key'; +SELECT pg_reload_conf(); +SQL + +for database in runtrace keycloak_runtrace amiary amiary_canary keycloak_amiary; do + docker exec "${server}" createdb -U postgres "${database}" +done +for database in runtrace_restore_test keycloak_runtrace_restore_test amiary_restore_test amiary_canary_restore_test keycloak_amiary_restore_test; do + docker exec "${server}" createdb -U postgres "${database}" +done + +docker exec -i "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d runtrace >/dev/null <<'SQL' +CREATE TABLE public.runtrace_state (id bigint PRIMARY KEY); +ALTER TABLE public.runtrace_state OWNER TO runtrace_app; +SQL +docker exec -i "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d keycloak_runtrace >/dev/null <<'SQL' +CREATE TABLE public.realm (id text PRIMARY KEY); +ALTER TABLE public.realm OWNER TO keycloak_runtrace_app; +SQL +docker exec -i "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d keycloak_amiary >/dev/null <<'SQL' +CREATE TABLE public.realm (id text PRIMARY KEY); +ALTER TABLE public.realm OWNER TO keycloak_amiary_app; +SQL + +for database in amiary amiary_canary; do + docker exec -i "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d "${database}" >/dev/null <<'SQL' +CREATE SCHEMA amiary AUTHORIZATION amiary_migrator; +CREATE TABLE amiary.persons (account_id uuid NOT NULL, id uuid PRIMARY KEY); +ALTER TABLE amiary.persons OWNER TO amiary_migrator; +ALTER TABLE amiary.persons ENABLE ROW LEVEL SECURITY; +ALTER TABLE amiary.persons FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON amiary.persons USING (true) WITH CHECK (true); +GRANT USAGE ON SCHEMA amiary TO amiary_api, amiary_worker, amiary_security_definer; +GRANT SELECT, INSERT, UPDATE, DELETE ON amiary.persons TO amiary_api; + +CREATE FUNCTION amiary.resolve_carddav_credential(text) RETURNS void + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, amiary AS 'BEGIN RETURN; END'; +CREATE FUNCTION amiary.claim_graph_events(integer) RETURNS void + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, amiary AS 'BEGIN RETURN; END'; +CREATE FUNCTION amiary.finish_graph_event(bigint, timestamptz, text) RETURNS void + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, amiary AS 'BEGIN RETURN; END'; +CREATE FUNCTION amiary.fetch_graph_projection(bigint, uuid, bigint, text, uuid) RETURNS void + LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, amiary AS 'BEGIN RETURN; END'; +ALTER FUNCTION amiary.resolve_carddav_credential(text) OWNER TO amiary_security_definer; +ALTER FUNCTION amiary.claim_graph_events(integer) OWNER TO amiary_security_definer; +ALTER FUNCTION amiary.finish_graph_event(bigint, timestamptz, text) OWNER TO amiary_security_definer; +ALTER FUNCTION amiary.fetch_graph_projection(bigint, uuid, bigint, text, uuid) OWNER TO amiary_security_definer; +REVOKE ALL ON FUNCTION amiary.resolve_carddav_credential(text) FROM PUBLIC; +REVOKE ALL ON FUNCTION amiary.claim_graph_events(integer) FROM PUBLIC; +REVOKE ALL ON FUNCTION amiary.finish_graph_event(bigint, timestamptz, text) FROM PUBLIC; +REVOKE ALL ON FUNCTION amiary.fetch_graph_projection(bigint, uuid, bigint, text, uuid) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION amiary.resolve_carddav_credential(text) TO amiary_api; +GRANT EXECUTE ON FUNCTION amiary.claim_graph_events(integer) TO amiary_worker; +GRANT EXECUTE ON FUNCTION amiary.finish_graph_event(bigint, timestamptz, text) TO amiary_worker; +GRANT EXECUTE ON FUNCTION amiary.fetch_graph_projection(bigint, uuid, bigint, text, uuid) TO amiary_worker; +SQL +done + +for database in runtrace keycloak_runtrace amiary amiary_canary keycloak_amiary; do + docker exec "${server}" psql -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -c "GRANT CONNECT ON DATABASE ${database} TO makepad_backup" >/dev/null + docker run --rm --user "$(id -u):$(id -g)" --network "${network}" \ + -e PGPASSWORD=disposable-backup-password -v "${work_dir}:/backups" "${postgres_image}" \ + pg_dump -h postgres-restore -U makepad_backup --role=makepad_backup_reader -d "${database}" -Fc -f "/backups/${database}.dump" +done + +if docker exec -e PGPASSWORD=disposable-backup-password "${server}" \ + psql -h 127.0.0.1 -v ON_ERROR_STOP=1 -U makepad_backup -d amiary \ + -c "INSERT INTO amiary.persons(account_id,id) VALUES ('00000000-0000-4000-8000-000000000001','00000000-0000-4000-8000-000000000002')" >/dev/null 2>&1; then + echo "backup login unexpectedly wrote application data" >&2 + exit 1 +fi +( + cd "${work_dir}" + sha256sum runtrace.dump keycloak_runtrace.dump amiary.dump amiary_canary.dump keycloak_amiary.dump > SHA256SUMS +) +printf '{"fixture":"disposable","databases":["runtrace","keycloak_runtrace","amiary","amiary_canary","keycloak_amiary"]}\n' > "${work_dir}/metadata.json" + +cat > "${work_dir}/pg_service.conf" <&2; exit 1; } +done + +common_roles=(amiary_api amiary_worker) +for role in "${common_roles[@]}"; do + grep -Fq "CREATE ROLE ${role} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS" "${sql}" \ + || { echo "missing constrained NOLOGIN capability role: ${role}" >&2; exit 1; } + grep -Fq "ALTER ROLE ${role} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS" "${sql}" \ + || { echo "bootstrap does not repair capability-role drift: ${role}" >&2; exit 1; } +done + +grep -Fq "CREATE ROLE amiary_security_definer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION BYPASSRLS" "${sql}" \ + || { echo "missing constrained RLS-bypass definer role" >&2; exit 1; } +grep -Fq "ALTER ROLE amiary_security_definer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION BYPASSRLS" "${sql}" \ + || { echo "bootstrap does not repair security-definer role drift" >&2; exit 1; } +grep -Fq "CREATE ROLE makepad_backup_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION BYPASSRLS" "${sql}" \ + || { echo "missing constrained backup reader role" >&2; exit 1; } +grep -Fq "ALTER ROLE makepad_backup_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT NOREPLICATION BYPASSRLS" "${sql}" \ + || { echo "bootstrap does not repair backup reader role drift" >&2; exit 1; } + +login_roles=( + amiary_migrator + amiary_api_prod + amiary_worker_prod + amiary_canary_migrator + amiary_api_canary + amiary_worker_canary + keycloak_amiary_app + makepad_backup +) +password_variables=( + amiary_migrator_password + amiary_api_prod_password + amiary_worker_prod_password + amiary_canary_migrator_password + amiary_api_canary_password + amiary_worker_canary_password + keycloak_amiary_app_password + makepad_backup_password +) + +for index in "${!login_roles[@]}"; do + role=${login_roles[${index}]} + variable=${password_variables[${index}]} + grep -Fq "CREATE ROLE ${role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE" "${sql}" \ + || { echo "missing constrained login role: ${role}" >&2; exit 1; } + grep -Fq "ALTER ROLE ${role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE" "${sql}" \ + || { echo "bootstrap does not repair login-role drift: ${role}" >&2; exit 1; } + grep -Fq ":'${variable}'" "${sql}" \ + || { echo "role password is not supplied by psql variable: ${role}" >&2; exit 1; } + grep -Fq "NULLIF(btrim(:'${variable}'), '')" "${sql}" \ + || { echo "bootstrap does not reject an empty password: ${variable}" >&2; exit 1; } +done + +membership_contracts=( + "('amiary_api', 'amiary_api_prod', true, false)" + "('amiary_api', 'amiary_api_canary', true, false)" + "('amiary_worker', 'amiary_worker_prod', true, false)" + "('amiary_worker', 'amiary_worker_canary', true, false)" + "('amiary_security_definer', 'amiary_migrator', false, true)" + "('amiary_security_definer', 'amiary_canary_migrator', false, true)" + "('pg_read_all_data', 'makepad_backup_reader', true, false)" + "('makepad_backup_reader', 'makepad_backup', false, true)" +) +for contract in "${membership_contracts[@]}"; do + grep -Fq "${contract}" "${sql}" || { echo "missing membership contract: ${contract}" >&2; exit 1; } +done + +grep -Fq "'GRANT %I TO %I WITH ADMIN FALSE, INHERIT %s, SET %s'" "${sql}" +grep -Fq "SELECT format('REVOKE %I FROM %I', granted.rolname, member.rolname)" "${sql}" +grep -Fq "WHERE member.rolname IN (" "${sql}" + +for database in amiary amiary_canary keycloak_amiary; do + grep -Eq "^hostnossl[[:space:]]+${database}[[:space:]]+all[[:space:]]+all[[:space:]]+reject$" "${hba}" \ + || { echo "missing plaintext rejection for ${database}" >&2; exit 1; } + grep -Eq "^hostssl[[:space:]]+${database}[[:space:]]+all[[:space:]]+all[[:space:]]+scram-sha-256$" "${hba}" \ + || { echo "missing TLS/SCRAM rule for ${database}" >&2; exit 1; } + grep -Fq "REVOKE ALL PRIVILEGES ON DATABASE ${database} FROM PUBLIC;" "${sql}" \ + || { echo "missing public database privilege revocation for ${database}" >&2; exit 1; } +done +grep -Eq '^host[[:space:]]+all[[:space:]]+amiary_migrator,amiary_api_prod,amiary_worker_prod,amiary_canary_migrator,amiary_api_canary,amiary_worker_canary,keycloak_amiary_app,makepad_backup[[:space:]]+all[[:space:]]+reject$' "${hba}" \ + || { echo "missing cross-database HBA rejection for Amiary login roles" >&2; exit 1; } + +grep -Fq "CREATE DATABASE amiary OWNER amiary_migrator" "${sql}" +grep -Fq "ALTER DATABASE amiary OWNER TO amiary_migrator" "${sql}" +grep -Fq "GRANT CONNECT ON DATABASE amiary TO amiary_migrator, amiary_api_prod, amiary_worker_prod;" "${sql}" +grep -Fq "CREATE DATABASE amiary_canary OWNER amiary_canary_migrator" "${sql}" +grep -Fq "ALTER DATABASE amiary_canary OWNER TO amiary_canary_migrator" "${sql}" +grep -Fq "GRANT CONNECT ON DATABASE amiary_canary TO amiary_canary_migrator, amiary_api_canary, amiary_worker_canary;" "${sql}" +grep -Fq "CREATE DATABASE keycloak_amiary OWNER keycloak_amiary_app" "${sql}" +grep -Fq "ALTER DATABASE keycloak_amiary OWNER TO keycloak_amiary_app" "${sql}" + +if grep -Eq "(CREATE|ALTER) DATABASE (amiary|amiary_canary) OWNER (amiary_api_|amiary_worker_)" "${sql}"; then + echo "API or worker role must not own an Amiary database" >&2 + exit 1 +fi + +grep -Fq "pg_advisory_lock" "${sql}" +grep -Fq "pg_advisory_unlock" "${sql}" +grep -Fq "bootstrap/amiary-apps.sql" "${readme}" + +readme_roles=( + amiary_migrator + amiary_api_prod + amiary_worker_prod + amiary_canary_migrator + amiary_api_canary + amiary_worker_canary + amiary_api + amiary_worker + amiary_security_definer + keycloak_amiary_app + makepad_backup + makepad_backup_reader +) +for role in "${readme_roles[@]}"; do + grep -Fq "${role}" "${readme}" || { echo "README does not document role: ${role}" >&2; exit 1; } +done + +environment_variables=( + AMIARY_MIGRATOR_DB_PASSWORD + AMIARY_API_DB_PASSWORD + AMIARY_WORKER_DB_PASSWORD + AMIARY_CANARY_MIGRATOR_DB_PASSWORD + AMIARY_CANARY_API_DB_PASSWORD + AMIARY_CANARY_WORKER_DB_PASSWORD + KEYCLOAK_AMIARY_DB_PASSWORD + MAKEPAD_BACKUP_DB_PASSWORD +) +for variable in "${environment_variables[@]}"; do + grep -Fq "\${${variable}:?" "${readme}" || { echo "README does not fail fast for ${variable}" >&2; exit 1; } +done + +for role in amiary_migrator amiary_api_prod amiary_worker_prod amiary_canary_migrator amiary_api_canary amiary_worker_canary keycloak_amiary_app; do + grep -Fq "postgres://${role}:@:5432/" "${readme}" \ + || { echo "README is missing certificate-verified DSN for ${role}" >&2; exit 1; } +done +grep -Fq "sslmode=verify-full&sslrootcert=/run/secrets/database_ca" "${readme}" + +if grep -Eq "(password|secret)[[:space:]]*=[[:space:]]*['\"][^:'\"]+['\"]" "${sql}"; then + echo "possible literal secret in Amiary bootstrap" >&2 + exit 1 +fi + +echo "Amiary PostgreSQL provisioning contract is valid." diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 8ce996b..42a2011 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -645,7 +645,7 @@ workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) + sorte require(workflow_paths, "At least one GitHub Actions workflow must exist.") for workflow_path in workflow_paths: workflow_text = read_required_text(workflow_path, f"workflow {workflow_path.name}") - if workflow_path.name == "fashion-bootstrap.yml": + if workflow_path.name in {"fashion-bootstrap.yml", "amiary-ci.yml"}: require(workflow_text.count(" runs-on: ubuntu-24.04") == 1, "Disposable Fashion database tests require the reviewed GitHub-hosted runner.") require("secrets." not in workflow_text, "Disposable Fashion tests must not receive production secrets.") continue diff --git a/scripts/verify-amiary-shared-restore.sh b/scripts/verify-amiary-shared-restore.sh new file mode 100755 index 0000000..ae3aa19 --- /dev/null +++ b/scripts/verify-amiary-shared-restore.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +set -euo pipefail + +backup_dir=${1:?Usage: verify-runtrace-restore.sh } +: "${PGSERVICEFILE:?PGSERVICEFILE must identify a root-owned libpq service file}" +: "${RUNTRACE_RESTORE_SERVICE:?RUNTRACE_RESTORE_SERVICE must name an empty non-production database}" +: "${KEYCLOAK_RUNTRACE_RESTORE_SERVICE:?KEYCLOAK_RUNTRACE_RESTORE_SERVICE must name an empty non-production database}" +: "${AMIARY_RESTORE_SERVICE:?AMIARY_RESTORE_SERVICE must name an empty non-production database}" +: "${AMIARY_CANARY_RESTORE_SERVICE:?AMIARY_CANARY_RESTORE_SERVICE must name an empty non-production database}" +: "${KEYCLOAK_AMIARY_RESTORE_SERVICE:?KEYCLOAK_AMIARY_RESTORE_SERVICE must name an empty non-production database}" +: "${RUNTRACE_RESTORE_CONFIRM:?set RUNTRACE_RESTORE_CONFIRM=replace-nonproduction-restore-targets}" + +if [[ "${RUNTRACE_RESTORE_CONFIRM}" != "replace-nonproduction-restore-targets" ]]; then + echo "Restore verification requires explicit non-production replacement confirmation." >&2 + exit 1 +fi +if [[ ! -f "${PGSERVICEFILE}" || -L "${PGSERVICEFILE}" ]]; then + echo "PGSERVICEFILE must be a regular, non-symlink file." >&2 + exit 1 +fi +if [[ ! -d "${backup_dir}" || -L "${backup_dir}" ]]; then + echo "Backup directory must be a regular directory and not a symlink." >&2 + exit 1 +fi + +for required in runtrace.dump keycloak_runtrace.dump amiary.dump amiary_canary.dump keycloak_amiary.dump SHA256SUMS metadata.json; do + if [[ ! -s "${backup_dir}/${required}" ]]; then + echo "Backup artifact is missing or empty: ${required}" >&2 + exit 1 + fi +done +( + cd "${backup_dir}" + sha256sum -c SHA256SUMS +) + +# Validate every destination before the first destructive operation. +services=("$RUNTRACE_RESTORE_SERVICE" "$KEYCLOAK_RUNTRACE_RESTORE_SERVICE" "$AMIARY_RESTORE_SERVICE" "$AMIARY_CANARY_RESTORE_SERVICE" "$KEYCLOAK_AMIARY_RESTORE_SERVICE") +databases=(runtrace keycloak_runtrace amiary amiary_canary keycloak_amiary) +for i in "${!services[@]}"; do + service=${services[$i]} + [[ "$service" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "Invalid restore service identifier." >&2; exit 1; } + target=$(psql "service=$service sslmode=verify-full" -XAt -v ON_ERROR_STOP=1 -v expected="${databases[$i]}_restore_test" <<'SQL' +SELECT current_database() = :'expected' AND NOT EXISTS ( + SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_toast%' + AND c.relkind IN ('r','p','v','m','S','f')); +SQL + ) + [[ "$target" == t ]] || { echo "Restore requires the exact empty disposable database." >&2; exit 1; } +done + +restore_dump() { + local service=$1 + local dump=$2 + pg_restore \ + --dbname="service=${service} sslmode=verify-full" \ + --clean \ + --if-exists \ + --exit-on-error \ + --single-transaction \ + "${dump}" +} + +restore_dump "${RUNTRACE_RESTORE_SERVICE}" "${backup_dir}/runtrace.dump" +restore_dump "${KEYCLOAK_RUNTRACE_RESTORE_SERVICE}" "${backup_dir}/keycloak_runtrace.dump" +restore_dump "${AMIARY_RESTORE_SERVICE}" "${backup_dir}/amiary.dump" +restore_dump "${AMIARY_CANARY_RESTORE_SERVICE}" "${backup_dir}/amiary_canary.dump" +restore_dump "${KEYCLOAK_AMIARY_RESTORE_SERVICE}" "${backup_dir}/keycloak_amiary.dump" + +runtrace_table=$(psql "service=${RUNTRACE_RESTORE_SERVICE} sslmode=verify-full" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('public.runtrace_state') IS NOT NULL;") +keycloak_table=$(psql "service=${KEYCLOAK_RUNTRACE_RESTORE_SERVICE} sslmode=verify-full" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('public.realm') IS NOT NULL;") +amiary_table=$(psql "service=${AMIARY_RESTORE_SERVICE} sslmode=verify-full" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('amiary.persons') IS NOT NULL;") +amiary_canary_table=$(psql "service=${AMIARY_CANARY_RESTORE_SERVICE} sslmode=verify-full" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('amiary.persons') IS NOT NULL;") +keycloak_amiary_table=$(psql "service=${KEYCLOAK_AMIARY_RESTORE_SERVICE} sslmode=verify-full" -v ON_ERROR_STOP=1 -Atc \ + "SELECT to_regclass('public.realm') IS NOT NULL;") +if [[ "${runtrace_table}" != "t" || "${keycloak_table}" != "t" || \ + "${amiary_table}" != "t" || "${amiary_canary_table}" != "t" || \ + "${keycloak_amiary_table}" != "t" ]]; then + echo "Restored databases are missing required Runtrace, Amiary, or Keycloak durable tables." >&2 + exit 1 +fi + +amiary_security_contract=$(psql "service=${AMIARY_RESTORE_SERVICE} sslmode=verify-full" -v ON_ERROR_STOP=1 -Atc " + SELECT count(*) = 0 + FROM ( + VALUES + ('resolve_carddav_credential', 'amiary_security_definer', true), + ('claim_graph_events', 'amiary_security_definer', true), + ('finish_graph_event', 'amiary_security_definer', true), + ('fetch_graph_projection', 'amiary_security_definer', true) + ) expected(function_name, owner_name, security_definer) + WHERE NOT EXISTS ( + SELECT 1 + FROM pg_proc function + JOIN pg_namespace namespace ON namespace.oid = function.pronamespace + JOIN pg_roles owner ON owner.oid = function.proowner + WHERE namespace.nspname = 'amiary' + AND function.proname = expected.function_name + AND owner.rolname = expected.owner_name + AND function.prosecdef = expected.security_definer + ); + SELECT count(*) = 0 + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = 'amiary' + AND relation.relkind IN ('r', 'p') + AND NOT relation.relispartition + AND ( + relation.relname = 'accounts' + OR EXISTS ( + SELECT 1 FROM pg_attribute attribute + WHERE attribute.attrelid = relation.oid + AND attribute.attname = 'account_id' + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + ) + ) + AND (NOT relation.relrowsecurity OR NOT relation.relforcerowsecurity); + SELECT + has_schema_privilege('amiary_api', 'amiary', 'USAGE') + AND has_table_privilege('amiary_api', 'amiary.persons', 'SELECT,INSERT,UPDATE,DELETE') + AND NOT has_table_privilege('amiary_api', 'amiary.persons', 'TRUNCATE') + AND has_function_privilege('amiary_api', 'amiary.resolve_carddav_credential(text)', 'EXECUTE') + AND NOT has_table_privilege('amiary_worker', 'amiary.persons', 'SELECT') + AND has_function_privilege('amiary_worker', 'amiary.claim_graph_events(integer)', 'EXECUTE') + AND has_function_privilege('amiary_worker', 'amiary.fetch_graph_projection(bigint,uuid,bigint,text,uuid)', 'EXECUTE') + AND NOT has_function_privilege('amiary_api', 'amiary.claim_graph_events(integer)', 'EXECUTE'); +") +if [[ "${amiary_security_contract}" != $'t\nt\nt' ]]; then + echo "Restored Amiary function ownership or forced-RLS contract is invalid." >&2 + exit 1 +fi + +echo "Shared PostgreSQL restore verification completed against non-production targets."