From f567030a32a196677403f7379233cf043f521592 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Tue, 13 Oct 2026 23:12:17 +0200 Subject: [PATCH 1/3] fix(postgres): isolate Scraping bootstrap and catalog reader --- .github/workflows/fashion-bootstrap.yml | 17 ++++++++++ bootstrap/fashion-catalog-reader.sql | 37 ++++++++++++++++++++ bootstrap/scraping-app.sql | 35 +++++++++++++++++++ scripts/test-fashion-bootstrap.py | 45 +++++++++++++++++++++++++ 4 files changed, 134 insertions(+) create mode 100644 .github/workflows/fashion-bootstrap.yml create mode 100644 bootstrap/fashion-catalog-reader.sql create mode 100644 bootstrap/scraping-app.sql create mode 100644 scripts/test-fashion-bootstrap.py diff --git a/.github/workflows/fashion-bootstrap.yml b/.github/workflows/fashion-bootstrap.yml new file mode 100644 index 0000000..2022972 --- /dev/null +++ b/.github/workflows/fashion-bootstrap.yml @@ -0,0 +1,17 @@ +name: Fashion database bootstrap +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main] +permissions: + contents: read +jobs: + database: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - run: python3 scripts/test-fashion-bootstrap.py diff --git a/bootstrap/fashion-catalog-reader.sql b/bootstrap/fashion-catalog-reader.sql new file mode 100644 index 0000000..199365d --- /dev/null +++ b/bootstrap/fashion-catalog-reader.sql @@ -0,0 +1,37 @@ +\set ON_ERROR_STOP on +SELECT pg_advisory_lock(hashtext('makepad-postgres'),hashtext('fashion-catalog-reader')); +DO $$ BEGIN + IF NOT EXISTS(SELECT 1 FROM pg_database d JOIN pg_roles r ON d.datdba=r.oid WHERE d.datname='scraping' AND r.rolname='scraping_crawler') THEN + RAISE EXCEPTION 'Scoped Scraping database must exist before the reader'; + END IF; + IF EXISTS(SELECT 1 FROM pg_roles r WHERE r.rolname='fashion_catalog_reader' AND + (NOT r.rolcanlogin OR r.rolsuper OR r.rolcreatedb OR r.rolcreaterole OR r.rolreplication OR r.rolbypassrls + OR EXISTS(SELECT 1 FROM pg_auth_members m WHERE m.member=r.oid OR m.roleid=r.oid))) THEN + RAISE EXCEPTION 'Existing reader privileges require review'; + END IF; +END $$; +SELECT NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='fashion_catalog_reader') AS fresh_reader \gset +\if :fresh_reader +\if :{?fashion_catalog_reader_password} +\else + DO $$ BEGIN RAISE EXCEPTION 'Reader password required'; END $$; +\endif +SELECT length(:'fashion_catalog_reader_password')>=32 AS strong_password \gset +\if :strong_password +\else + DO $$ BEGIN RAISE EXCEPTION 'Reader password must contain 32 characters'; END $$; +\endif +CREATE ROLE fashion_catalog_reader LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'fashion_catalog_reader_password'; +\endif +GRANT CONNECT ON DATABASE scraping TO fashion_catalog_reader; +SELECT pg_advisory_unlock(hashtext('makepad-postgres'),hashtext('fashion-catalog-reader')); +\connect scraping +DO $$ BEGIN + IF EXISTS(SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p') + AND has_table_privilege('fashion_catalog_reader',c.oid,'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER')) THEN + RAISE EXCEPTION 'Existing catalog reader can mutate tables'; + END IF; +END $$; +GRANT USAGE ON SCHEMA public TO fashion_catalog_reader; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO fashion_catalog_reader; +ALTER DEFAULT PRIVILEGES FOR ROLE scraping_crawler IN SCHEMA public GRANT SELECT ON TABLES TO fashion_catalog_reader; diff --git a/bootstrap/scraping-app.sql b/bootstrap/scraping-app.sql new file mode 100644 index 0000000..dd5b393 --- /dev/null +++ b/bootstrap/scraping-app.sql @@ -0,0 +1,35 @@ +\set ON_ERROR_STOP on +SELECT pg_advisory_lock(hashtext('makepad-postgres'),hashtext('scraping-bootstrap')); +SELECT NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='scraping_crawler') AND NOT EXISTS(SELECT 1 FROM pg_database WHERE datname='scraping') AS fresh_install \gset +\if :fresh_install +\if :{?scraping_crawler_password} +\else + \echo 'missing scraping_crawler_password' + DO $$ BEGIN RAISE EXCEPTION 'Scraping bootstrap precondition failed'; END $$; +\endif +SELECT length(:'scraping_crawler_password') >= 32 AS strong_password \gset +\if :strong_password +\else + \echo 'scraping_crawler_password must contain at least 32 characters' + DO $$ BEGIN RAISE EXCEPTION 'Scraping bootstrap precondition failed'; END $$; +\endif +CREATE ROLE scraping_crawler LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD :'scraping_crawler_password'; +CREATE DATABASE scraping OWNER scraping_crawler; +REVOKE ALL ON DATABASE scraping FROM PUBLIC; +GRANT CONNECT ON DATABASE scraping TO scraping_crawler; +\else + SELECT EXISTS( + SELECT 1 FROM pg_database d JOIN pg_roles r ON d.datdba=r.oid + WHERE d.datname='scraping' AND r.rolname='scraping_crawler' + AND r.rolcanlogin AND NOT r.rolsuper AND NOT r.rolcreatedb + AND NOT r.rolcreaterole AND NOT r.rolreplication AND NOT r.rolbypassrls + AND NOT EXISTS(SELECT 1 FROM pg_auth_members m WHERE m.member=r.oid OR m.roleid=r.oid) + AND NOT EXISTS(SELECT 1 FROM aclexplode(COALESCE(d.datacl,acldefault('d',d.datdba))) a WHERE a.grantee<>d.datdba AND a.grantee<>COALESCE((SELECT oid FROM pg_roles WHERE rolname='fashion_catalog_reader'),0)) + ) AS existing_is_scoped \gset + \if :existing_is_scoped + \echo 'Existing Scraping ownership and isolation verified; credentials unchanged.' + \else + DO $$ BEGIN RAISE EXCEPTION 'Existing Scraping installation is incomplete or unsafe'; END $$; + \endif +\endif +SELECT pg_advisory_unlock(hashtext('makepad-postgres'),hashtext('scraping-bootstrap')); diff --git a/scripts/test-fashion-bootstrap.py b/scripts/test-fashion-bootstrap.py new file mode 100644 index 0000000..e5ac728 --- /dev/null +++ b/scripts/test-fashion-bootstrap.py @@ -0,0 +1,45 @@ +import subprocess +from pathlib import Path +import time +import unittest +import uuid +ROOT=Path(__file__).resolve().parents[1] +class Bootstrap(unittest.TestCase): + def test_existing_credentials_and_isolation(self): + name='fashion-db-test-'+uuid.uuid4().hex[:10] + def cmd(*args,input=None): + return subprocess.run(args,input=input,text=True,capture_output=True,check=True).stdout.strip() + def sql(text): + return cmd('docker','exec','-i',name,'psql','-U','postgres','-XAt','-v','ON_ERROR_STOP=1',input=text) + try: + image=next(l.split('=',1)[1] for l in (ROOT/'envs/production/.env.db').read_text().splitlines() if l.startswith('POSTGRES_IMAGE=')) + cmd('docker','run','-d','--network','none','--name',name,'--tmpfs','/var/lib/postgresql/data','-e','POSTGRES_PASSWORD=fixture-password',image) + for _ in range(30): + if subprocess.run(['docker','exec',name,'pg_isready','-U','postgres'],capture_output=True).returncode==0: break + time.sleep(1) + for file,role,db,var in [('scraping-app.sql','scraping_crawler','scraping','scraping_crawler_password'),('iceberg-catalog.sql','iceberg_catalog','iceberg_catalog','iceberg_catalog_password'),('vestiaire-developer-platform.sql','vestiaire_developer_app','vestiaire_developer','vestiaire_developer_app_password')]: + if not (ROOT/'bootstrap'/file).exists(): + continue + with self.subTest(file=file): + def provision(password): + return cmd('docker','exec','-i',name,'psql','-U','postgres','-XAt','-v','ON_ERROR_STOP=1','-v',var+'='+password,'-v','fashion_catalog_reader_password=fixture-reader-password-32-characters',input=(ROOT/'bootstrap'/file).read_text()) + provision('fixture-app-password-with-32-characters') + before=sql("SELECT rolpassword FROM pg_authid WHERE rolname='"+role+"'") + provision('different-fixture-password-with-32-characters') + self.assertTrue(before==sql("SELECT rolpassword FROM pg_authid WHERE rolname='"+role+"'"),'Bootstrap rotated existing credential') + self.assertEqual(sql("SELECT count(*) FROM pg_database d, LATERAL aclexplode(COALESCE(d.datacl,acldefault('d',d.datdba))) a WHERE d.datname='"+db+"' AND a.grantee=0"),'0') + sql('ALTER ROLE '+role+' SUPERUSER;') + with self.assertRaises(subprocess.CalledProcessError):provision('fixture-password-32-characters-long') + sql('ALTER ROLE '+role+' NOSUPERUSER;') + def reader(password): + return cmd('docker','exec','-i',name,'psql','-U','postgres','-XAt','-v','ON_ERROR_STOP=1','-v','fashion_catalog_reader_password='+password,input=(ROOT/'bootstrap/fashion-catalog-reader.sql').read_text()) + reader('fixture-reader-password-with-32-characters') + old=sql("SELECT rolpassword FROM pg_authid WHERE rolname='fashion_catalog_reader'") + reader('different-reader-password-with-32-characters') + self.assertTrue(old==sql("SELECT rolpassword FROM pg_authid WHERE rolname='fashion_catalog_reader'")) + cmd('docker','exec',name,'psql','-U','postgres','-d','scraping','-v','ON_ERROR_STOP=1','-c','SET ROLE scraping_crawler; CREATE TABLE fixture(id int); INSERT INTO fixture VALUES(1);') + self.assertEqual(cmd('docker','exec',name,'psql','-U','postgres','-d','scraping','-XAt','-v','ON_ERROR_STOP=1','-c','SET ROLE fashion_catalog_reader; SELECT count(*) FROM fixture;').splitlines()[-1],'1') + with self.assertRaises(subprocess.CalledProcessError): + cmd('docker','exec',name,'psql','-U','postgres','-d','scraping','-v','ON_ERROR_STOP=1','-c','SET ROLE fashion_catalog_reader; INSERT INTO fixture VALUES(2);') + finally: subprocess.run(['docker','rm','-f',name],capture_output=True) +if __name__=='__main__':unittest.main() From bc6594720cbbe75e9748d7790fbfafce106638f3 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Tue, 13 Oct 2026 23:20:49 +0200 Subject: [PATCH 2/3] fix(ci): allow isolated Fashion tests on GitHub-hosted Linux --- scripts/validate-postgres-config.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/validate-postgres-config.sh b/scripts/validate-postgres-config.sh index 65a83f9..8ce996b 100755 --- a/scripts/validate-postgres-config.sh +++ b/scripts/validate-postgres-config.sh @@ -645,6 +645,10 @@ workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) + sorte require(workflow_paths, "At least one GitHub Actions workflow must exist.") for workflow_path in workflow_paths: workflow_text = read_required_text(workflow_path, f"workflow {workflow_path.name}") + if workflow_path.name == "fashion-bootstrap.yml": + require(workflow_text.count(" runs-on: ubuntu-24.04") == 1, "Disposable Fashion database tests require the reviewed GitHub-hosted runner.") + require("secrets." not in workflow_text, "Disposable Fashion tests must not receive production secrets.") + continue runs_on_count = len(re.findall(r"(?m)^ runs-on:", workflow_text)) existing_runner_count = workflow_text.count(" runs-on: [self-hosted, linux, x64, makepad]") require(runs_on_count > 0, f"Workflow {workflow_path.name} must define at least one job runner.") From e222e496f6a0a639ab5c7cddf3bb510de54f1987 Mon Sep 17 00:00:00 2001 From: Kaan Yagci Date: Thu, 15 Oct 2026 08:20:46 +0200 Subject: [PATCH 3/3] test(postgres): wait for the final fixture server --- scripts/test-fashion-bootstrap.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/test-fashion-bootstrap.py b/scripts/test-fashion-bootstrap.py index e5ac728..81e37cb 100644 --- a/scripts/test-fashion-bootstrap.py +++ b/scripts/test-fashion-bootstrap.py @@ -14,9 +14,14 @@ def sql(text): try: image=next(l.split('=',1)[1] for l in (ROOT/'envs/production/.env.db').read_text().splitlines() if l.startswith('POSTGRES_IMAGE=')) cmd('docker','run','-d','--network','none','--name',name,'--tmpfs','/var/lib/postgresql/data','-e','POSTGRES_PASSWORD=fixture-password',image) + # The initialization server accepts socket connections before restarting. + # TCP readiness identifies the final server used by the bootstrap tests. for _ in range(30): - if subprocess.run(['docker','exec',name,'pg_isready','-U','postgres'],capture_output=True).returncode==0: break + if subprocess.run(['docker','exec',name,'pg_isready','-h','127.0.0.1','-U','postgres'],capture_output=True).returncode==0: break time.sleep(1) + else: + logs = subprocess.run(['docker','logs',name],capture_output=True,text=True) + self.fail('PostgreSQL fixture never became TCP-ready: '+logs.stderr[-2000:]) for file,role,db,var in [('scraping-app.sql','scraping_crawler','scraping','scraping_crawler_password'),('iceberg-catalog.sql','iceberg_catalog','iceberg_catalog','iceberg_catalog_password'),('vestiaire-developer-platform.sql','vestiaire_developer_app','vestiaire_developer','vestiaire_developer_app_password')]: if not (ROOT/'bootstrap'/file).exists(): continue