fix(postgres): preserve Scraping credentials and catalog read access

.github/workflows/fashion-bootstrap.yml

@@ -0,0 +1,17 @@
+name: Fashion database bootstrap
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  push:
+    branches: [main]
+permissions:
+  contents: read
+jobs:
+  database:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          persist-credentials: false
+      - run: python3 scripts/test-fashion-bootstrap.py

bootstrap/fashion-catalog-reader.sql

@@ -0,0 +1,37 @@
+\set ON_ERROR_STOP on
+SELECT pg_advisory_lock(hashtext('makepad-postgres'),hashtext('fashion-catalog-reader'));
+DO $$ BEGIN
+ IF NOT EXISTS(SELECT 1 FROM pg_database d JOIN pg_roles r ON d.datdba=r.oid WHERE d.datname='scraping' AND r.rolname='scraping_crawler') THEN
+  RAISE EXCEPTION 'Scoped Scraping database must exist before the reader';
+ END IF;
+ IF EXISTS(SELECT 1 FROM pg_roles r WHERE r.rolname='fashion_catalog_reader' AND
+   (NOT r.rolcanlogin OR r.rolsuper OR r.rolcreatedb OR r.rolcreaterole OR r.rolreplication OR r.rolbypassrls
+    OR EXISTS(SELECT 1 FROM pg_auth_members m WHERE m.member=r.oid OR m.roleid=r.oid))) THEN
+  RAISE EXCEPTION 'Existing reader privileges require review';
+ END IF;
+END $$;
+SELECT NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='fashion_catalog_reader') AS fresh_reader \gset
+\if :fresh_reader
+\if :{?fashion_catalog_reader_password}
+\else
+ DO $$ BEGIN RAISE EXCEPTION 'Reader password required'; END $$;
+\endif
+SELECT length(:'fashion_catalog_reader_password')>=32 AS strong_password \gset
+\if :strong_password
+\else
+ DO $$ BEGIN RAISE EXCEPTION 'Reader password must contain 32 characters'; END $$;
+\endif
+CREATE ROLE fashion_catalog_reader LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'fashion_catalog_reader_password';
+\endif
+GRANT CONNECT ON DATABASE scraping TO fashion_catalog_reader;
+SELECT pg_advisory_unlock(hashtext('makepad-postgres'),hashtext('fashion-catalog-reader'));
+\connect scraping
+DO $$ BEGIN
+ IF EXISTS(SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p')
+   AND has_table_privilege('fashion_catalog_reader',c.oid,'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER')) THEN
+  RAISE EXCEPTION 'Existing catalog reader can mutate tables';
+ END IF;
+END $$;
+GRANT USAGE ON SCHEMA public TO fashion_catalog_reader;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO fashion_catalog_reader;
+ALTER DEFAULT PRIVILEGES FOR ROLE scraping_crawler IN SCHEMA public GRANT SELECT ON TABLES TO fashion_catalog_reader;

bootstrap/scraping-app.sql

@@ -0,0 +1,35 @@
+\set ON_ERROR_STOP on
+SELECT pg_advisory_lock(hashtext('makepad-postgres'),hashtext('scraping-bootstrap'));
+SELECT NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='scraping_crawler') AND NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='scraping') AS fresh_install \gset
+\if :fresh_install
+\if :{?scraping_crawler_password}
+\else
+  \echo 'missing scraping_crawler_password'
+  DO $$ BEGIN RAISE EXCEPTION 'Scraping bootstrap precondition failed'; END $$;
+\endif
+SELECT length(:'scraping_crawler_password') >= 32 AS strong_password \gset
+\if :strong_password
+\else
+  \echo 'scraping_crawler_password must contain at least 32 characters'
+  DO $$ BEGIN RAISE EXCEPTION 'Scraping bootstrap precondition failed'; END $$;
+\endif
+CREATE ROLE scraping_crawler LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'scraping_crawler_password';
+CREATE DATABASE scraping OWNER scraping_crawler;
+REVOKE ALL ON DATABASE scraping FROM PUBLIC;
+GRANT CONNECT ON DATABASE scraping TO scraping_crawler;
+\else
+  SELECT EXISTS(
+    SELECT 1 FROM pg_database d JOIN pg_roles r ON d.datdba=r.oid
+    WHERE d.datname='scraping' AND r.rolname='scraping_crawler'
+      AND r.rolcanlogin AND NOT r.rolsuper AND NOT r.rolcreatedb
+      AND NOT r.rolcreaterole AND NOT r.rolreplication AND NOT r.rolbypassrls
+      AND NOT EXISTS(SELECT 1 FROM pg_auth_members m WHERE m.member=r.oid OR m.roleid=r.oid)
+      AND NOT EXISTS(SELECT 1 FROM aclexplode(COALESCE(d.datacl,acldefault('d',d.datdba))) a WHERE a.grantee<>d.datdba AND a.grantee<>COALESCE((SELECT oid FROM pg_roles WHERE rolname='fashion_catalog_reader'),0))
+  ) AS existing_is_scoped \gset
+  \if :existing_is_scoped
+    \echo 'Existing Scraping ownership and isolation verified; credentials unchanged.'
+  \else
+    DO $$ BEGIN RAISE EXCEPTION 'Existing Scraping installation is incomplete or unsafe'; END $$;
+  \endif
+\endif
+SELECT pg_advisory_unlock(hashtext('makepad-postgres'),hashtext('scraping-bootstrap'));

scripts/test-fashion-bootstrap.py

@@ -0,0 +1,50 @@
+import subprocess
+from pathlib import Path
+import time
+import unittest
+import uuid
+ROOT=Path(__file__).resolve().parents[1]
+class Bootstrap(unittest.TestCase):
+    def test_existing_credentials_and_isolation(self):
+        name='fashion-db-test-'+uuid.uuid4().hex[:10]
+        def cmd(*args,input=None):
+            return subprocess.run(args,input=input,text=True,capture_output=True,check=True).stdout.strip()
+        def sql(text):
+            return cmd('docker','exec','-i',name,'psql','-U','postgres','-XAt','-v','ON_ERROR_STOP=1',input=text)
+        try:
+            image=next(l.split('=',1)[1] for l in (ROOT/'envs/production/.env.db').read_text().splitlines() if l.startswith('POSTGRES_IMAGE='))
+            cmd('docker','run','-d','--network','none','--name',name,'--tmpfs','/var/lib/postgresql/data','-e','POSTGRES_PASSWORD=fixture-password',image)
+            # The initialization server accepts socket connections before restarting.
+            # TCP readiness identifies the final server used by the bootstrap tests.
+            for _ in range(30):
+                if subprocess.run(['docker','exec',name,'pg_isready','-h','127.0.0.1','-U','postgres'],capture_output=True).returncode==0: break
+                time.sleep(1)
+            else:
+                logs = subprocess.run(['docker','logs',name],capture_output=True,text=True)
+                self.fail('PostgreSQL fixture never became TCP-ready: '+logs.stderr[-2000:])
+            for file,role,db,var in [('scraping-app.sql','scraping_crawler','scraping','scraping_crawler_password'),('iceberg-catalog.sql','iceberg_catalog','iceberg_catalog','iceberg_catalog_password'),('vestiaire-developer-platform.sql','vestiaire_developer_app','vestiaire_developer','vestiaire_developer_app_password')]:
+                if not (ROOT/'bootstrap'/file).exists():
+                    continue
+                with self.subTest(file=file):
+                    def provision(password):
+                        return cmd('docker','exec','-i',name,'psql','-U','postgres','-XAt','-v','ON_ERROR_STOP=1','-v',var+'='+password,'-v','fashion_catalog_reader_password=fixture-reader-password-32-characters',input=(ROOT/'bootstrap'/file).read_text())
+                    provision('fixture-app-password-with-32-characters')
+                    before=sql("SELECT rolpassword FROM pg_authid WHERE rolname='"+role+"'")
+                    provision('different-fixture-password-with-32-characters')
+                    self.assertTrue(before==sql("SELECT rolpassword FROM pg_authid WHERE rolname='"+role+"'"),'Bootstrap rotated existing credential')
+                    self.assertEqual(sql("SELECT count(*) FROM pg_database d, LATERAL aclexplode(COALESCE(d.datacl,acldefault('d',d.datdba))) a WHERE d.datname='"+db+"' AND a.grantee=0"),'0')
+                    sql('ALTER ROLE '+role+' SUPERUSER;')
+                    with self.assertRaises(subprocess.CalledProcessError):provision('fixture-password-32-characters-long')
+                    sql('ALTER ROLE '+role+' NOSUPERUSER;')
+            def reader(password):
+                return cmd('docker','exec','-i',name,'psql','-U','postgres','-XAt','-v','ON_ERROR_STOP=1','-v','fashion_catalog_reader_password='+password,input=(ROOT/'bootstrap/fashion-catalog-reader.sql').read_text())
+            reader('fixture-reader-password-with-32-characters')
+            old=sql("SELECT rolpassword FROM pg_authid WHERE rolname='fashion_catalog_reader'")
+            reader('different-reader-password-with-32-characters')
+            self.assertTrue(old==sql("SELECT rolpassword FROM pg_authid WHERE rolname='fashion_catalog_reader'"))
+            cmd('docker','exec',name,'psql','-U','postgres','-d','scraping','-v','ON_ERROR_STOP=1','-c','SET ROLE scraping_crawler; CREATE TABLE fixture(id int); INSERT INTO fixture VALUES(1);')
+            self.assertEqual(cmd('docker','exec',name,'psql','-U','postgres','-d','scraping','-XAt','-v','ON_ERROR_STOP=1','-c','SET ROLE fashion_catalog_reader; SELECT count(*) FROM fixture;').splitlines()[-1],'1')
+            with self.assertRaises(subprocess.CalledProcessError):
+                cmd('docker','exec',name,'psql','-U','postgres','-d','scraping','-v','ON_ERROR_STOP=1','-c','SET ROLE fashion_catalog_reader; INSERT INTO fixture VALUES(2);')
+        finally: subprocess.run(['docker','rm','-f',name],capture_output=True)
+if __name__=='__main__':unittest.main()

scripts/validate-postgres-config.sh

@@ -645,6 +645,10 @@ workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) + sorte
 require(workflow_paths, "At least one GitHub Actions workflow must exist.")
 for workflow_path in workflow_paths:
     workflow_text = read_required_text(workflow_path, f"workflow {workflow_path.name}")
+    if workflow_path.name == "fashion-bootstrap.yml":
+        require(workflow_text.count("    runs-on: ubuntu-24.04") == 1, "Disposable Fashion database tests require the reviewed GitHub-hosted runner.")
+        require("secrets." not in workflow_text, "Disposable Fashion tests must not receive production secrets.")
+        continue
     runs_on_count = len(re.findall(r"(?m)^    runs-on:", workflow_text))
     existing_runner_count = workflow_text.count("    runs-on: [self-hosted, linux, x64, makepad]")
     require(runs_on_count > 0, f"Workflow {workflow_path.name} must define at least one job runner.")