RHINENG-26348: add ability to repair system_advisories_0 partition

database_admin/config.go

@@ -19,4 +19,6 @@ var (
 	updateDBConfig = utils.PodConfig.GetBool("update_db_config", false)
 	// Terminate lockUsers sessions after NOLOGIN (for major DDL migrations)
 	terminateDBSessions = utils.PodConfig.GetBool("terminate_db_sessions", false)
+	// One-off: truncate corrupt system_advisories_0 and clear bucket-0 advisory caches
+	repairSystemAdvisories0 = utils.PodConfig.GetBool("repair_system_advisories_0", false)
 )

database_admin/schema/repair_system_advisories_0.sql

@@ -0,0 +1,20 @@
+-- One-off repair for corrupt system_advisories_0 (pg_xact / unreadable heap).
+-- Gated by database_admin flag repair_system_advisories_0=true (default false).
+-- Irreversible: truncates partition data for hash remainder 0 and clears related caches.
+-- Safe to re-run only when intentionally wiping bucket 0 again (e.g. failed cutover).
+-- TRUNCATE drops table segments without scanning the heap (validated on restore DB).
+
+BEGIN;
+
+TRUNCATE TABLE system_advisories_0;
+
+-- Clear denormalized counts for accounts that hash into remainder 0 (do not read old _0).
+DELETE FROM advisory_account_data aad
+ WHERE satisfies_hash_partition(
+         'system_advisories'::regclass, 32, 0, aad.rh_account_id);
+
+DELETE FROM account_advisory aa
+ WHERE satisfies_hash_partition(
+         'system_advisories'::regclass, 32, 0, aa.rh_account_id);
+
+COMMIT;

database_admin/update.go

@@ -184,6 +184,15 @@ func startMigration(conn database.Driver, db *sql.DB, migrationFilesURL string)
 	unblockUsers(db)
 }
 
+func repairSystemAdvisories0Partition(db *sql.DB) {
+	log.Info("Repairing system_advisories_0 (truncate + clear bucket-0 caches)")
+	prepareForMigration(db)
+	execFromFile(db, "./database_admin/schema/repair_system_advisories_0.sql")
+	log.Info("system_advisories_0 repair finished")
+	log.Info("Reverting components privileges after system_advisories_0 repair")
+	unblockUsers(db)
+}
+
 func dbConn() (database.Driver, *sql.DB) {
 	sslModeCert := utils.CoreCfg.DBSslMode
 	if utils.CoreCfg.DBSslRootCert != "" {
@@ -236,6 +245,10 @@ func UpdateDB(migrationFilesURL string) {
 		startMigration(conn, db, migrationFilesURL)
 	}
 
+	if repairSystemAdvisories0 {
+		repairSystemAdvisories0Partition(db)
+	}
+
 	if updateUsers {
 		log.Info("Setting user passwords")
 		// Set specific password for each user. If the users are already created, change the password.

deploy/clowdapp.yaml

@@ -896,6 +896,7 @@ parameters:
 - {name: DATABASE_ADMIN_CONFIG, value: ''} # Set 'schema_version=XXX' if need specific database schema
                                            # 'terminate_db_sessions=true' to kill app DB sessions before DDL
                                            # 'force_schema_version=XXX' to reset the dirty flag to false and force the specific version, it will follow up with the schema upgrade defined by schema_version
+                                           # 'repair_system_advisories_0=true' one-off: truncate corrupt system_advisories_0 and clear bucket-0 AAD/AA (default false; remove after cutover)
 
 # Common parameters
 - {name: IMAGE, value: quay.io/redhat-services-prod/insights-management-tenant/insights-patch/patchman-engine}

docs/md/major-migration-runbook.md

@@ -147,6 +147,18 @@ Config keys are defined in `database_admin/config.go`. ClowdApp comments in `dep
 
 `NOLOGIN` alone does not close existing connections — that is why this flag exists.
 
+### `repair_system_advisories_0`
+
+| | |
+|---|---|
+| **Config key** | `repair_system_advisories_0` (boolean, default `false`) |
+| **Where** | `DATABASE_ADMIN_CONFIG` on the **db-migration Job** only |
+| **Effect** | After migrate CONTINUE/MIGRATE, runs `prepareForMigration`, then `database_admin/schema/repair_system_advisories_0.sql`: `TRUNCATE system_advisories_0`, clear bucket-0 `advisory_account_data` and `account_advisory`. **Destructive** for hash remainder 0. |
+
+**Enable when:** one-off recovery from corrupt/unreadable `system_advisories_0`. Combine with `terminate_db_sessions=true` if truncate is blocked by app sessions.
+
+**Leave off for all normal deploys.** Remove after the cutover succeeds. Do not enable on manager/listener/evaluator pods.
+
 ---
 
 ## During deploy