From 0fc2eb9780232dcfe9d598d1d6ff08b7d5af25bc Mon Sep 17 00:00:00 2001
From: Smana <smainklh@gmail.com>
Date: Tue, 29 Sep 2026 10:52:53 +0200
Subject: [PATCH 01/10] fix(gcp): stage 2 writes the CA, adopts jwt/gcp-0 and
 passes deploy_identity_provider

---
 opentofu/gcp/gke/init/workflows.tm.hcl        | 20 +++++++--
 .../ci/tests/test-gcp-gke-init-workflow.py    | 44 +++++++++++++++++++
 2 files changed, 61 insertions(+), 3 deletions(-)
 create mode 100644 scripts/ci/tests/test-gcp-gke-init-workflow.py

diff --git a/opentofu/gcp/gke/init/workflows.tm.hcl b/opentofu/gcp/gke/init/workflows.tm.hcl
index e59767fb8..f70af6849 100644
--- a/opentofu/gcp/gke/init/workflows.tm.hcl
+++ b/opentofu/gcp/gke/init/workflows.tm.hcl
@@ -105,13 +105,27 @@ script "deploy" {
         ${global.cloud_gate}
         set -euo pipefail
         cd ../configure
+        # ../configure's vault provider needs the CA chain before init, and a
+        # fresh checkout has no .tls/ (gitignored). gke/configure's own deploy
+        # writes it; this inline apply bypasses that script (09-11 bug 3).
+        bash "${terramate.root.path.fs.absolute}/scripts/provision/openbao-config.sh" ca \
+          --cloud gcp --project ogenki-435905 \
+          --root-ca-secret-name openbao-priv-gcp-ca-chain --ca-output-file .tls/ca.pem
         ${global.provisioner} init -lock-timeout=5m
-        ${global.provisioner} apply -auto-approve -var-file=variables.tfvars -var='cilium_version=${global.cilium_version}' -var='gateway_api_version=${global.gateway_api_version}' -var='flux_operator_version=${global.flux_operator_version}' -var='flux_instance_version=${global.flux_instance_version}' $${TF_VAR_flux_git_ref:+-var="flux_git_ref=$${TF_VAR_flux_git_ref}"}
+        # A restored lineage already holds jwt/gcp-0, and creating it again 400s
+        # with "path is already in use". Same call as gke/configure's deploy.
+        bash "${terramate.root.path.fs.absolute}/scripts/provision/openbao-adopt-jwt-mount.sh" \
+          --cluster-name gcp-0 --url https://bao.priv.gcp.ogenki.io:8200 \
+          --root-token-secret-name openbao-priv-gcp-root-token \
+          --ca-file .tls/ca.pem --cloud gcp --project ogenki-435905 \
+          -- -var='cilium_version=${global.cilium_version}' -var='gateway_api_version=${global.gateway_api_version}' -var='flux_operator_version=${global.flux_operator_version}' -var='flux_instance_version=${global.flux_instance_version}' -var='deploy_identity_provider=${global.deploy_identity_provider_gcp}' $${TF_VAR_flux_git_ref:+-var="flux_git_ref=$${TF_VAR_flux_git_ref}"}
+        # deploy_identity_provider here too: without it this apply publishes the
+        # consumed (AWS) identity_provider_url until the standalone configure run.
+        ${global.provisioner} apply -auto-approve -var-file=variables.tfvars -var='cilium_version=${global.cilium_version}' -var='gateway_api_version=${global.gateway_api_version}' -var='flux_operator_version=${global.flux_operator_version}' -var='flux_instance_version=${global.flux_instance_version}' -var='deploy_identity_provider=${global.deploy_identity_provider_gcp}' $${TF_VAR_flux_git_ref:+-var="flux_git_ref=$${TF_VAR_flux_git_ref}"}
         # Forget flux-operator here, in the job that just created it -- NOT only
         # in gke/configure's own `deploy`, which this job bypasses. Left in state,
         # the standalone gke/configure stack plans count=0 against a resource that
-        # IS in state, and that is a destroy: a real `helm uninstall`. The AWS lane
-        # had the identical gap and hit it on 2026-09-16.
+        # IS in state, and that is a destroy: a real `helm uninstall`.
         ${global.provisioner} state rm helm_release.flux_operator 2>/dev/null || true
       BASH
       ],
diff --git a/scripts/ci/tests/test-gcp-gke-init-workflow.py b/scripts/ci/tests/test-gcp-gke-init-workflow.py
new file mode 100644
index 000000000..377ddf759
--- /dev/null
+++ b/scripts/ci/tests/test-gcp-gke-init-workflow.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# requires: python3
+"""gcp/gke/init's deploy jobs (GCP parity G-2/G-3). Each check is a bug a live
+GCP deploy hit: 09-11 bug 3 (no CA, no jwt adopt in stage 2), the missing
+deploy_identity_provider on the inline configure apply, the custom roles that a
+teardown deleted (GP-15), and a hosting stage 3 that must configure the IdP
+before its clients and mirror them into OpenBao (GP-5)."""
+import pathlib
+import re
+import sys
+
+ROOT = pathlib.Path(__file__).resolve().parents[3]
+TEXT = (ROOT / "opentofu/gcp/gke/init/workflows.tm.hcl").read_text()
+
+
+def job_body(text, name):
+    m = re.search(r'name\s*=\s*"%s"(.*?)(?=\n  job \{|\nscript "|\Z)' % re.escape(name), text, re.S)
+    return m.group(1) if m else ""
+
+
+fails = []
+
+
+def before(body, first, second, what):
+    i, j = body.find(first), body.find(second)
+    if i < 0 or j < 0 or i > j:
+        fails.append(what)
+
+
+s2 = job_body(TEXT, "stage2-cilium-and-flux")
+if not s2:
+    fails.append("no stage2-cilium-and-flux job")
+before(s2, 'openbao-config.sh" ca', "init -lock-timeout", "stage 2 writes OpenBao's CA before init")
+before(s2, "init -lock-timeout", "openbao-adopt-jwt-mount.sh", "stage 2 adopts jwt/gcp-0 after init")
+before(s2, "openbao-adopt-jwt-mount.sh", "apply -auto-approve", "stage 2 adopts jwt/gcp-0 before its apply")
+apply_line = next((l for l in s2.splitlines() if "apply -auto-approve" in l), "")
+if "deploy_identity_provider=${global.deploy_identity_provider_gcp}" not in apply_line:
+    fails.append("stage 2's apply passes deploy_identity_provider")
+
+for f in fails:
+    print("FAIL", f)
+if fails:
+    sys.exit(1)
+print("PASS")

From 78ed76331634596a4deb6365aaf91ca875a15fe1 Mon Sep 17 00:00:00 2001
From: Smana <smainklh@gmail.com>
Date: Tue, 29 Sep 2026 10:56:19 +0200
Subject: [PATCH 02/10] fix(gcp): gcp-0's private external-dns no longer waits
 on an AWS-only controller

---
 .../gcp-0/external-dns/kustomization.yaml     | 10 +++++++
 .../ci/tests/test-gcp-external-dns-deps.py    | 28 +++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 scripts/ci/tests/test-gcp-external-dns-deps.py

diff --git a/infrastructure/gcp-0/external-dns/kustomization.yaml b/infrastructure/gcp-0/external-dns/kustomization.yaml
index f9d203d77..a93255163 100644
--- a/infrastructure/gcp-0/external-dns/kustomization.yaml
+++ b/infrastructure/gcp-0/external-dns/kustomization.yaml
@@ -16,3 +16,13 @@ patches:
       group: helm.toolkit.fluxcd.io
       kind: HelmRelease
       name: external-dns
+  # base waits on aws-load-balancer-controller (#1975's webhook race), a release
+  # gcp-0 never runs: GKE's own controller creates its LoadBalancers. Kept, the
+  # edge never resolves and no private record is ever written (09-11 bug 6).
+  - target:
+      group: helm.toolkit.fluxcd.io
+      kind: HelmRelease
+      name: external-dns
+    patch: |-
+      - op: remove
+        path: /spec/dependsOn
diff --git a/scripts/ci/tests/test-gcp-external-dns-deps.py b/scripts/ci/tests/test-gcp-external-dns-deps.py
new file mode 100644
index 000000000..c25ad52ea
--- /dev/null
+++ b/scripts/ci/tests/test-gcp-external-dns-deps.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# requires: python3 kustomize
+"""GCP parity GP-27 (09-11 bug 6): gcp-0's private external-dns must not wait on
+aws-load-balancer-controller, which gcp-0 never runs. With the edge, the release
+never installs and no *.priv.gcp record is written, while the parent
+Kustomization (no `wait`) still reports Ready."""
+import pathlib
+import subprocess
+import sys
+
+try:
+    import yaml
+except ImportError:
+    print("PyYAML is not installed")
+    sys.exit(77)
+
+ROOT = pathlib.Path(__file__).resolve().parents[3]
+out = subprocess.run(["kustomize", "build", str(ROOT / "infrastructure/gcp-0/external-dns"),
+                      "--load-restrictor=LoadRestrictionsNone"], capture_output=True, text=True, check=True).stdout
+hrs = [d for d in yaml.safe_load_all(out) if d and d.get("kind") == "HelmRelease" and d["metadata"]["name"] == "external-dns"]
+if len(hrs) != 1:
+    print(f"FAIL expected one external-dns HelmRelease, found {len(hrs)}")
+    sys.exit(1)
+deps = hrs[0]["spec"].get("dependsOn") or []
+if any(d.get("name") == "aws-load-balancer-controller" for d in deps):
+    print("FAIL gcp-0's external-dns depends on aws-load-balancer-controller, which gcp-0 never runs")
+    sys.exit(1)
+print("PASS")

From f90c49695a2f79f9a4bd1387240f8fd9c58f0d33 Mon Sep 17 00:00:00 2001
From: Smana <smainklh@gmail.com>
Date: Tue, 29 Sep 2026 10:57:10 +0200
Subject: [PATCH 03/10] fix(gcp): Crossplane may grant objectCreator, so gcp-0
 takes its OpenBao snapshots

---
 opentofu/gcp/gke/init/iam.tf                  |  5 +++
 .../tests/test-gcp-bucket-grant-allowlist.sh  | 35 +++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100755 scripts/ci/tests/test-gcp-bucket-grant-allowlist.sh

diff --git a/opentofu/gcp/gke/init/iam.tf b/opentofu/gcp/gke/init/iam.tf
index 8caef5d89..505400955 100644
--- a/opentofu/gcp/gke/init/iam.tf
+++ b/opentofu/gcp/gke/init/iam.tf
@@ -234,6 +234,11 @@ locals {
   crossplane_bucket_grantable_roles = [
     "roles/storage.objectAdmin",
     "roles/storage.objectViewer",
+    # openbao-snapshot's bucket grant (security/gcp-0/openbao-snapshot): create
+    # only, strictly weaker than objectAdmin above. Missing, the condition denied
+    # the grant with a bare 403 and gcp-0 never took a scheduled snapshot
+    # (GCP parity GP-23, 09-11 bug 7).
+    "roles/storage.objectCreator",
     "roles/storage.legacyBucketReader",
   ]
 
diff --git a/scripts/ci/tests/test-gcp-bucket-grant-allowlist.sh b/scripts/ci/tests/test-gcp-bucket-grant-allowlist.sh
new file mode 100755
index 000000000..b4c95a91c
--- /dev/null
+++ b/scripts/ci/tests/test-gcp-bucket-grant-allowlist.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# requires: python3
+#
+# GCP parity GP-23 (09-11 bug 7; memory gcp_crossplane_grant_allowlist_contract):
+# every role a GCPWorkloadIdentity grants under bucketRoles must be in gke/init's
+# crossplane_bucket_grantable_roles, or the IAM condition denies the grant with a
+# bare 403. openbao-snapshot's objectCreator was missing, so gcp-0 never took a
+# scheduled snapshot.
+set -uo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+python3 - "$ROOT" <<'PY'
+import pathlib, re, sys
+root = pathlib.Path(sys.argv[1])
+iam = (root / "opentofu/gcp/gke/init/iam.tf").read_text()
+m = re.search(r"crossplane_bucket_grantable_roles\s*=\s*\[(.*?)\]", iam, re.S)
+if not m:
+    print("FAIL no crossplane_bucket_grantable_roles in iam.tf"); sys.exit(1)
+allowed = set(re.findall(r'"(roles/[^"]+)"', m.group(1)))
+wanted = {}
+for area in ("infrastructure", "security", "observability", "tooling", "apps"):
+    for f in (root / area).glob("gcp-0/**/*.yaml"):
+        text = f.read_text()
+        if "kind: GCPWorkloadIdentity" not in text or "bucketRoles:" not in text:
+            continue
+        for role in re.findall(r"^\s+role:\s*(roles/\S+)", text.split("bucketRoles:", 1)[1], re.M):
+            wanted.setdefault(role, []).append(str(f.relative_to(root)))
+if not wanted:
+    print("FAIL found no bucketRoles grant at all: the scan is broken"); sys.exit(1)
+missing = {r: fs for r, fs in wanted.items() if r not in allowed}
+for r, fs in sorted(missing.items()):
+    print(f"FAIL {r} (asked by {', '.join(fs)}) is not in crossplane_bucket_grantable_roles")
+if missing:
+    sys.exit(1)
+print("PASS")
+PY

From f4719ad5f00f4aedc4985ad2a758182828587d50 Mon Sep 17 00:00:00 2001
From: Smana <smainklh@gmail.com>
Date: Tue, 29 Sep 2026 11:05:20 +0200
Subject: [PATCH 04/10] fix(gcp): custom roles take a generation suffix and
 survive a teardown

---
 .../gcp-0/external-dns/workloadidentity.yaml  |  2 +-
 opentofu/gcp/gke/configure/kubernetes.tf      |  4 ++
 opentofu/gcp/gke/init/iam.tf                  |  6 +-
 opentofu/gcp/gke/init/outputs.tf              |  5 ++
 opentofu/gcp/gke/init/variables.tf            | 11 ++++
 opentofu/gcp/gke/init/variables.tfvars        |  2 +
 opentofu/gcp/gke/init/workflows.tm.hcl        | 13 +++++
 scripts/ci/flux-schema/render-bundle.py       |  2 +
 scripts/ci/tests/test-adopt-custom-roles.sh   | 41 +++++++++++++
 scripts/ci/tests/test-gcp-custom-role-refs.sh | 22 +++++++
 .../ci/tests/test-gcp-gke-init-workflow.py    |  7 +++
 scripts/ops/gcp/adopt-custom-roles.sh         | 58 +++++++++++++++++++
 12 files changed, 169 insertions(+), 4 deletions(-)
 create mode 100755 scripts/ci/tests/test-adopt-custom-roles.sh
 create mode 100755 scripts/ci/tests/test-gcp-custom-role-refs.sh
 create mode 100755 scripts/ops/gcp/adopt-custom-roles.sh

diff --git a/infrastructure/gcp-0/external-dns/workloadidentity.yaml b/infrastructure/gcp-0/external-dns/workloadidentity.yaml
index ec8bb7269..810ea2dc4 100644
--- a/infrastructure/gcp-0/external-dns/workloadidentity.yaml
+++ b/infrastructure/gcp-0/external-dns/workloadidentity.yaml
@@ -29,4 +29,4 @@ spec:
   serviceAccount:
     name: external-dns
   roles:
-    - "projects/${project_id}/roles/xplane_dns_editor"
+    - "${gcp_dns_editor_role}"
diff --git a/opentofu/gcp/gke/configure/kubernetes.tf b/opentofu/gcp/gke/configure/kubernetes.tf
index ed100d0d6..edb717d90 100644
--- a/opentofu/gcp/gke/configure/kubernetes.tf
+++ b/opentofu/gcp/gke/configure/kubernetes.tf
@@ -62,6 +62,10 @@ resource "kubectl_manifest" "flux_cluster_vars" {
       storage_class = local.storage_class
 
       # GCP-specific.
+      # Custom role names carry a generation suffix (gke/init var.custom_role_suffix),
+      # so claims read the name rather than spelling it.
+      gcp_dns_editor_role = local.init.dns_editor_role
+
       project_id     = var.project_id
       project_number = local.init.project_number
       zone           = local.init.cluster_location
diff --git a/opentofu/gcp/gke/init/iam.tf b/opentofu/gcp/gke/init/iam.tf
index 505400955..a3b5cd36c 100644
--- a/opentofu/gcp/gke/init/iam.tf
+++ b/opentofu/gcp/gke/init/iam.tf
@@ -75,7 +75,7 @@
 # spelled `xplane_` here.
 resource "google_project_iam_custom_role" "crossplane_dns" {
   project     = var.project_id
-  role_id     = "xplane_dns_editor"
+  role_id     = "xplane_dns_editor${var.custom_role_suffix}"
   title       = "Crossplane DNS editor"
   description = "Record-set management for external-dns and cert-manager DNS-01. Deliberately excludes zone deletion and response policies; see opentofu/gcp/gke/init/iam.tf."
 
@@ -131,7 +131,7 @@ resource "google_project_iam_custom_role" "crossplane_dns" {
 # pattern as crossplane_dns.
 resource "google_project_iam_custom_role" "crossplane_storage" {
   project     = var.project_id
-  role_id     = "xplane_storage_admin"
+  role_id     = "xplane_storage_admin${var.custom_role_suffix}"
   title       = "Crossplane storage bucket admin"
   description = "Bucket lifecycle + IAM for the App Composition's per-app buckets. See opentofu/gcp/gke/init/iam.tf."
 
@@ -370,7 +370,7 @@ resource "google_project_iam_member" "crossplane_storage" {
 # read requests, so a condition here would be vacuous anyway (see gap 1 below).
 resource "google_project_iam_custom_role" "crossplane_role_reader" {
   project     = var.project_id
-  role_id     = "xplane_role_reader"
+  role_id     = "xplane_role_reader${var.custom_role_suffix}"
   title       = "Crossplane role reader"
   description = "Read-only on IAM role definitions, so ProjectIAMMember can reference the custom DNS role. Confers no grant capability."
 
diff --git a/opentofu/gcp/gke/init/outputs.tf b/opentofu/gcp/gke/init/outputs.tf
index 4ac08f2b4..19bd400c1 100644
--- a/opentofu/gcp/gke/init/outputs.tf
+++ b/opentofu/gcp/gke/init/outputs.tf
@@ -94,3 +94,8 @@ output "private_domain_name" {
   description = "Cloud DNS private zone domain, substituted into manifests as $${private_domain_name}"
   value       = local.net.private_domain_name
 }
+
+output "dns_editor_role" {
+  description = "Full name of the custom DNS role, suffix included, for GCPWorkloadIdentity claims"
+  value       = google_project_iam_custom_role.crossplane_dns.name
+}
diff --git a/opentofu/gcp/gke/init/variables.tf b/opentofu/gcp/gke/init/variables.tf
index cec5e3d74..e48931eb7 100644
--- a/opentofu/gcp/gke/init/variables.tf
+++ b/opentofu/gcp/gke/init/variables.tf
@@ -152,3 +152,14 @@ variable "tailscale_oauth_secret_name" {
   type        = string
   default     = "tailscale-k8s-operator-oauth"
 }
+
+# GCP reserves a deleted custom role's ID for 37 days and allows undelete for 7
+# only, so a rebuild inside that window cannot recreate it (09-11 bug 2). The
+# destroy now keeps the roles (they grant nothing without the bindings it does
+# destroy) and the deploy adopts them; this suffix moved past the IDs already
+# burned (unsuffixed before 2026-09-11, `_v2` on 2026-09-14).
+variable "custom_role_suffix" {
+  description = "Generation suffix on the three custom role IDs; bump only if a generation is lost"
+  type        = string
+  default     = ""
+}
diff --git a/opentofu/gcp/gke/init/variables.tfvars b/opentofu/gcp/gke/init/variables.tfvars
index 77d7ef0f0..be35e8e18 100644
--- a/opentofu/gcp/gke/init/variables.tfvars
+++ b/opentofu/gcp/gke/init/variables.tfvars
@@ -5,6 +5,8 @@ region     = "europe-west4"
 cluster_name    = "gcp-0"
 release_channel = "REGULAR"
 
+custom_role_suffix = "_v3"
+
 # Slice 4 must pin the SAME image type on every ComputeClass.
 node_image_type   = "COS_CONTAINERD"
 node_machine_type = "e2-standard-4"
diff --git a/opentofu/gcp/gke/init/workflows.tm.hcl b/opentofu/gcp/gke/init/workflows.tm.hcl
index f70af6849..c72448f80 100644
--- a/opentofu/gcp/gke/init/workflows.tm.hcl
+++ b/opentofu/gcp/gke/init/workflows.tm.hcl
@@ -90,6 +90,9 @@ script "deploy" {
         set -euo pipefail
         ${global.provisioner} init
         ${global.provisioner} validate
+        bash "${terramate.root.path.fs.absolute}/scripts/ops/gcp/adopt-custom-roles.sh" \
+          --project ogenki-435905 \
+          --suffix "$(awk -F'"' '/^custom_role_suffix/{print $2}' variables.tfvars)" --apply
         trivy config --exit-code=1 --ignorefile=./.trivyignore.yaml .
         ${global.provisioner} apply -auto-approve -var-file=variables.tfvars
       BASH
@@ -326,6 +329,9 @@ script "deploy-stage1" {
         set -euo pipefail
         ${global.provisioner} init
         ${global.provisioner} validate
+        bash "${terramate.root.path.fs.absolute}/scripts/ops/gcp/adopt-custom-roles.sh" \
+          --project ogenki-435905 \
+          --suffix "$(awk -F'"' '/^custom_role_suffix/{print $2}' variables.tfvars)" --apply
         trivy config --exit-code=1 --ignorefile=./.trivyignore.yaml .
         ${global.provisioner} apply -auto-approve -var-file=variables.tfvars
       BASH
@@ -385,6 +391,13 @@ script "destroy" {
         # must fail here, not after resources have started disappearing. Same stack
         # dir as stage1-destroy-cluster, so that job inherits this init.
         ${global.provisioner} init -lock-timeout=5m
+        # Keep the custom roles (GCP parity GP-15): a deleted role ID stays
+        # reserved for 37 days, so the next rebuild could not recreate it. They
+        # grant nothing on their own; the bindings using them are destroyed as
+        # usual, and the next deploy adopts them (adopt-custom-roles.sh).
+        for addr in google_project_iam_custom_role.crossplane_dns google_project_iam_custom_role.crossplane_storage google_project_iam_custom_role.crossplane_role_reader; do
+          ${global.provisioner} state rm "$addr" 2>/dev/null || true
+        done
       BASH
       ],
     ]
diff --git a/scripts/ci/flux-schema/render-bundle.py b/scripts/ci/flux-schema/render-bundle.py
index 6ba6b4756..e9e79ffda 100755
--- a/scripts/ci/flux-schema/render-bundle.py
+++ b/scripts/ci/flux-schema/render-bundle.py
@@ -140,6 +140,8 @@ def _repo_lock(url):
     # is what lets CI catch a consumer that assumes a string.
     "project_number": "323586397743",
     "workload_pool": "ogenki-435905.svc.id.goog",
+    # gcp-0 only: the custom DNS role's full name, generation suffix included.
+    "gcp_dns_editor_role": "projects/ogenki-435905/roles/xplane_dns_editor_v3",
     "zone": "europe-west4-a",
     "network_name": "vpc-foobar",
     # Federated Route53 path (workstream 12). GCP-only: aws-0 authenticates to
diff --git a/scripts/ci/tests/test-adopt-custom-roles.sh b/scripts/ci/tests/test-adopt-custom-roles.sh
new file mode 100755
index 000000000..d67e88151
--- /dev/null
+++ b/scripts/ci/tests/test-adopt-custom-roles.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+#
+# adopt-custom-roles.sh (GCP parity GP-15) with gcloud and tofu stubbed on PATH:
+# a live role missing from state is imported, one in state is left alone, an
+# absent one is left to the apply, and a soft-deleted one is reported, never imported.
+set -uo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+T="$(mktemp -d)"; trap 'rm -rf "$T"' EXIT
+mkdir -p "$T/bin"
+cat >"$T/bin/gcloud" <<'EOF'
+#!/usr/bin/env bash
+case "$*" in
+  *"auth application-default print-access-token"*) echo fake-token ;;
+  *"roles describe xplane_dns_editor_v3 "*) echo "" ;;
+  *"roles describe xplane_storage_admin_v3 "*) echo "True" ;;
+  *"roles describe xplane_role_reader_v3 "*) exit 1 ;;
+  *) exit 1 ;;
+esac
+EOF
+cat >"$T/bin/tofu" <<EOF
+#!/usr/bin/env bash
+case "\$1" in
+  state) printf '%s\n' module.gke.google_container_cluster.primary ;;
+  import) echo "\$*" >>"$T/imports" ;;
+esac
+EOF
+chmod +x "$T/bin/gcloud" "$T/bin/tofu"
+# out and rc are read inside check's eval strings, which shellcheck cannot see.
+# shellcheck disable=SC2034
+out="$(PATH="$T/bin:$PATH" bash "$ROOT/scripts/ops/gcp/adopt-custom-roles.sh" --project ogenki-435905 --suffix _v3 --apply 2>&1)"
+# shellcheck disable=SC2034
+rc=$?
+fails=0
+check() { if eval "$2"; then echo "  ok   $1"; else echo "  FAIL $1"; fails=1; fi; }
+check "exit 0" '[ "$rc" -eq 0 ]'
+check "the live dns role is imported" 'grep -q "import -var-file=variables.tfvars google_project_iam_custom_role.crossplane_dns projects/ogenki-435905/roles/xplane_dns_editor_v3" "$T/imports"'
+check "exactly one import" '[ "$(wc -l <"$T/imports")" -eq 1 ]'
+check "the soft-deleted storage role is reported" 'grep -q "\[deleted\].*xplane_storage_admin_v3" <<<"$out"'
+check "the absent reader role is left to the apply" 'grep -q "\[absent \].*xplane_role_reader_v3" <<<"$out"'
+[ "$fails" -eq 0 ] && echo "all checks passed"
+exit "$fails"
diff --git a/scripts/ci/tests/test-gcp-custom-role-refs.sh b/scripts/ci/tests/test-gcp-custom-role-refs.sh
new file mode 100755
index 000000000..83bfdd5ad
--- /dev/null
+++ b/scripts/ci/tests/test-gcp-custom-role-refs.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# GCP parity GP-15: every custom role carries the generation suffix, and no
+# manifest spells a custom role name -- they read ${gcp_dns_editor_role}, so a
+# suffix bump never leaves a claim pointing at a deleted role.
+set -uo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+fails=0
+while IFS= read -r line; do
+  case "$line" in
+    *'${var.custom_role_suffix}"'*) ;;
+    *) echo "FAIL iam.tf: $line"; fails=$((fails + 1)) ;;
+  esac
+done < <(grep -E '^[[:space:]]*role_id[[:space:]]*=' "$ROOT/opentofu/gcp/gke/init/iam.tf")
+[ "$(grep -cE '^[[:space:]]*role_id[[:space:]]*=' "$ROOT/opentofu/gcp/gke/init/iam.tf")" -eq 3 ] \
+  || { echo "FAIL expected 3 custom roles in iam.tf"; fails=$((fails + 1)); }
+hits="$(grep -rnE 'roles/xplane_' --include=*.yaml "$ROOT"/{clusters,infrastructure,security,observability,tooling,apps} 2>/dev/null)"
+[ -z "$hits" ] || { echo "FAIL a manifest spells a custom role:"; echo "$hits"; fails=$((fails + 1)); }
+grep -q 'gcp_dns_editor_role' "$ROOT/opentofu/gcp/gke/configure/kubernetes.tf" \
+  || { echo "FAIL gke-gcp-0-vars has no gcp_dns_editor_role"; fails=$((fails + 1)); }
+[ "$fails" -eq 0 ] || exit 1
+echo PASS
diff --git a/scripts/ci/tests/test-gcp-gke-init-workflow.py b/scripts/ci/tests/test-gcp-gke-init-workflow.py
index 377ddf759..35aab7908 100644
--- a/scripts/ci/tests/test-gcp-gke-init-workflow.py
+++ b/scripts/ci/tests/test-gcp-gke-init-workflow.py
@@ -37,6 +37,13 @@ def before(body, first, second, what):
 if "deploy_identity_provider=${global.deploy_identity_provider_gcp}" not in apply_line:
     fails.append("stage 2's apply passes deploy_identity_provider")
 
+s1 = job_body(TEXT, "stage1-cluster")
+before(s1, "adopt-custom-roles.sh", "apply -auto-approve", "stage 1 adopts the kept custom roles before its apply")
+confirm = job_body(TEXT, "confirm")
+for addr in ("crossplane_dns", "crossplane_storage", "crossplane_role_reader"):
+    if f"google_project_iam_custom_role.{addr}" not in confirm or "state rm" not in confirm:
+        fails.append(f"the destroy keeps google_project_iam_custom_role.{addr} out of the teardown")
+
 for f in fails:
     print("FAIL", f)
 if fails:
diff --git a/scripts/ops/gcp/adopt-custom-roles.sh b/scripts/ops/gcp/adopt-custom-roles.sh
new file mode 100755
index 000000000..1c34a37db
--- /dev/null
+++ b/scripts/ops/gcp/adopt-custom-roles.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+#
+# Adopt gke/init's three custom IAM roles when they already exist.
+#
+# WHY: GCP reserves a deleted custom role's ID for 37 days and can undelete it
+# for 7 only, so a rebuild within 37 days of a teardown that deleted the roles
+# fails on the first create. gke/init's destroy therefore drops them from state
+# instead of deleting them, and every deploy adopts them here (GCP parity GP-15).
+#
+# Usage (from opentofu/gcp/gke/init, after `tofu init`):
+#   adopt-custom-roles.sh --project ID --suffix S [--apply]
+# Dry-run unless --apply.
+set -euo pipefail
+
+# shellcheck source=scripts/lib/gcloud-adc.sh
+. "$(dirname "$0")/../../lib/gcloud-adc.sh"
+
+PROJECT="" SUFFIX="" APPLY=false
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --project) PROJECT="$2"; shift 2 ;;
+    --suffix)  SUFFIX="$2"; shift 2 ;;
+    --apply)   APPLY=true; shift ;;
+    *) echo "unknown argument: $1" >&2; exit 2 ;;
+  esac
+done
+[ -n "$PROJECT" ] || { echo "--project is required" >&2; exit 2; }
+
+ROLES=(
+  "google_project_iam_custom_role.crossplane_dns=xplane_dns_editor"
+  "google_project_iam_custom_role.crossplane_storage=xplane_storage_admin"
+  "google_project_iam_custom_role.crossplane_role_reader=xplane_role_reader"
+)
+
+in_state="$(tofu state list 2>/dev/null || true)"
+for entry in "${ROLES[@]}"; do
+  addr="${entry%%=*}"
+  id="${entry#*=}${SUFFIX}"
+  name="projects/${PROJECT}/roles/${id}"
+  if grep -qxF "$addr" <<<"$in_state"; then
+    echo "[ok     ] ${addr} is in state"
+    continue
+  fi
+  if ! deleted="$(gcp_gcloud iam roles describe "$id" --project "$PROJECT" --format='value(deleted)' 2>/dev/null)"; then
+    echo "[absent ] ${name}: the apply creates it"
+    continue
+  fi
+  if [ "$deleted" = "True" ]; then
+    echo "[deleted] ${name} is soft-deleted: the provider undeletes it within 7 days; after that, bump custom_role_suffix" >&2
+    continue
+  fi
+  if [ "$APPLY" = true ]; then
+    tofu import -var-file=variables.tfvars "$addr" "$name"
+    echo "[adopted] ${addr} <- ${name}"
+  else
+    echo "[dry-run] would import ${addr} <- ${name}"
+  fi
+done

From b73487d0f2adb72c4e6a1a9a6ab13735f80d5977 Mon Sep 17 00:00:00 2001
From: Smana <smainklh@gmail.com>
Date: Tue, 29 Sep 2026 11:07:12 +0200
Subject: [PATCH 05/10] docs(gcp): restore the dated incident behind stage 2's
 flux-operator state rm

---
 opentofu/gcp/gke/init/workflows.tm.hcl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/opentofu/gcp/gke/init/workflows.tm.hcl b/opentofu/gcp/gke/init/workflows.tm.hcl
index c72448f80..a14d7a043 100644
--- a/opentofu/gcp/gke/init/workflows.tm.hcl
+++ b/opentofu/gcp/gke/init/workflows.tm.hcl
@@ -128,7 +128,8 @@ script "deploy" {
         # Forget flux-operator here, in the job that just created it -- NOT only
         # in gke/configure's own `deploy`, which this job bypasses. Left in state,
         # the standalone gke/configure stack plans count=0 against a resource that
-        # IS in state, and that is a destroy: a real `helm uninstall`.
+        # IS in state, and that is a destroy: a real `helm uninstall`. The AWS
+        # lane had the identical gap and hit it on 2026-09-16.
         ${global.provisioner} state rm helm_release.flux_operator 2>/dev/null || true
       BASH
       ],

From df9125d9797c497154c49881d4bb18bd52d21f53 Mon Sep 17 00:00:00 2001
From: Smana <smainklh@gmail.com>
Date: Tue, 29 Sep 2026 11:12:45 +0200
Subject: [PATCH 06/10] fix(teardown): sweep the forwarding rules, target pools
 and k8s-* firewall rules a GKE cluster leaves

---
 scripts/ci/tests/test-teardown-gcp-sweep.sh | 50 +++++++++++++++
 scripts/ops/gcp/sweep-lb-orphans.sh         | 68 +++++++++++++++++++++
 scripts/ops/teardown/teardown.sh            | 22 +++++++
 3 files changed, 140 insertions(+)
 create mode 100755 scripts/ci/tests/test-teardown-gcp-sweep.sh
 create mode 100755 scripts/ops/gcp/sweep-lb-orphans.sh

diff --git a/scripts/ci/tests/test-teardown-gcp-sweep.sh b/scripts/ci/tests/test-teardown-gcp-sweep.sh
new file mode 100755
index 000000000..e5e211eac
--- /dev/null
+++ b/scripts/ci/tests/test-teardown-gcp-sweep.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#
+# memory gke_lb_orphans_block_vpc_delete: a deleted GKE cluster leaves a target
+# pool and k8s-* firewall rules behind, and the firewall rules block the VPC
+# delete. teardown.sh --verify-only must report both, and fail while they exist.
+set -uo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+T="$(mktemp -d)"; trap 'rm -rf "$T"' EXIT
+mkdir -p "$T/bin"
+cat >"$T/bin/gcloud" <<EOF
+#!/usr/bin/env bash
+case "\$*" in
+  "projects describe"*) exit 0 ;;
+  *"print-access-token"*) echo fake-token ;;
+  *"container clusters list"*) printf '%s' "\${CLUSTERS:-}" ;;
+  *" delete "*) echo "\$*" >>"$T/deletes" ;;
+  *"forwarding-rules list"*) echo "a2c53478 europe-west4" ;;
+  *"target-pools list"*) echo "a2c53478 europe-west4" ;;
+  *"firewall-rules list"*) printf '%s\n' k8s-fw-a2c53478 k8s-a6e2efd9258d71a1-node-http-hc ;;
+  *) exit 0 ;;
+esac
+EOF
+chmod +x "$T/bin/gcloud"
+fails=0
+f() { echo "FAIL $*"; fails=1; }
+
+# The report (--verify-only).
+out="$(TM_CLOUD=gcp PATH="$T/bin:$PATH" bash "$ROOT/scripts/ops/teardown/teardown.sh" --verify-only 2>&1)"; rc=$?
+grep -q 'Target pools' <<<"$out" || f "no Target pools row"
+grep -q 'k8s-\* firewall rules' <<<"$out" || f "no k8s-* firewall rules row"
+[ "$rc" -ne 0 ] || f "leftovers must fail the verify"
+
+# The sweep (GP-22): dry-run deletes nothing, --apply deletes the four, a live cluster blocks it.
+S="$ROOT/scripts/ops/gcp/sweep-lb-orphans.sh"
+args=(--project ogenki-435905 --network vpc-europe-west4-dev --cluster gcp-0)
+[ -f "$S" ] || f "no $S"
+: >"$T/deletes"
+PATH="$T/bin:$PATH" bash "$S" "${args[@]}" >/dev/null 2>&1 || f "the dry run failed"
+[ ! -s "$T/deletes" ] || f "the dry run deleted something"
+: >"$T/deletes"
+PATH="$T/bin:$PATH" bash "$S" "${args[@]}" --apply >/dev/null 2>&1 || f "the sweep failed"
+[ "$(wc -l <"$T/deletes")" -eq 4 ] || f "expected 4 deletes (rule, pool, 2 firewalls), got: $(cat "$T/deletes")"
+grep -q 'forwarding-rules delete a2c53478 --region europe-west4' "$T/deletes" || f "the forwarding rule was not deleted in its region"
+: >"$T/deletes"
+CLUSTERS=gcp-0 PATH="$T/bin:$PATH" bash "$S" "${args[@]}" --apply >/dev/null 2>&1; rc=$?
+[ "$rc" -eq 3 ] || f "a live cluster must refuse with exit 3, got $rc"
+[ ! -s "$T/deletes" ] || f "a live cluster's LBs were deleted"
+
+[ "$fails" -eq 0 ] && echo PASS
+exit "$fails"
diff --git a/scripts/ops/gcp/sweep-lb-orphans.sh b/scripts/ops/gcp/sweep-lb-orphans.sh
new file mode 100755
index 000000000..dee927c3e
--- /dev/null
+++ b/scripts/ops/gcp/sweep-lb-orphans.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+#
+# Delete what a deleted GKE cluster leaves in the platform VPC: its
+# LoadBalancers' forwarding rules and target pools, and the k8s-* firewall
+# rules. No tofu state holds them, forwarding rules bill hourly, and the
+# firewall rules block the VPC delete (memory gke_lb_orphans_block_vpc_delete;
+# GCP parity GP-22). Dry-run unless --apply.
+#
+# Usage: sweep-lb-orphans.sh --project P --network N --cluster C [--apply]
+set -euo pipefail
+
+# shellcheck source=scripts/lib/gcloud-adc.sh
+. "$(dirname "$0")/../../lib/gcloud-adc.sh"
+
+PROJECT="" NETWORK="" CLUSTER="" APPLY=false
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --project) PROJECT="$2"; shift 2 ;;
+    --network) NETWORK="$2"; shift 2 ;;
+    --cluster) CLUSTER="$2"; shift 2 ;;
+    --apply)   APPLY=true; shift ;;
+    *) echo "unknown argument: $1" >&2; exit 2 ;;
+  esac
+done
+{ [ -n "$PROJECT" ] && [ -n "$NETWORK" ] && [ -n "$CLUSTER" ]; } \
+  || { echo "--project, --network and --cluster are required" >&2; exit 2; }
+
+# A live cluster still owns these: its service controller would recreate them,
+# and deleting a live LoadBalancer's rule is an outage, not a sweep.
+if ! clusters="$(gcp_gcloud container clusters list --project "$PROJECT" \
+                   --filter="name=${CLUSTER}" --format='value(name)')"; then
+  echo "cannot list clusters in ${PROJECT}: refusing to sweep blind" >&2; exit 3
+fi
+if [ -n "$clusters" ]; then
+  echo "cluster ${CLUSTER} still exists: nothing is swept while it does" >&2; exit 3
+fi
+
+# GKE's service controller writes {"kubernetes.io/service-name": ...} into the
+# description of every forwarding rule and target pool it creates; that, not a
+# name pattern, is what makes an object GKE's.
+rules="$(gcp_gcloud compute forwarding-rules list --project "$PROJECT" \
+  --filter='description~kubernetes.io/service-name' --format='value(name,region.basename())')"
+pools="$(gcp_gcloud compute target-pools list --project "$PROJECT" \
+  --filter='description~kubernetes.io/service-name' --format='value(name,region.basename())')"
+firewalls="$(gcp_gcloud compute firewall-rules list --project "$PROJECT" \
+  --filter="name~^k8s- AND network~/${NETWORK}\$" --format='value(name)')"
+
+failed=0
+act() { # $1 = what, then the delete command
+  local what="$1"; shift
+  if [ "$APPLY" != true ]; then echo "[dry-run] would delete ${what}"; return 0; fi
+  if "$@" --quiet >/dev/null; then echo "[deleted] ${what}"; else echo "[FAILED ] ${what}" >&2; failed=1; fi
+}
+# Rules before pools (a rule targets a pool), firewalls last.
+while read -r name region; do
+  [ -n "$name" ] || continue
+  scope=(--global); [ -n "$region" ] && scope=(--region "$region")
+  act "forwarding rule ${name}" gcp_gcloud compute forwarding-rules delete "$name" "${scope[@]}" --project "$PROJECT"
+done <<<"$rules"
+while read -r name region; do
+  [ -n "$name" ] || continue
+  act "target pool ${name}" gcp_gcloud compute target-pools delete "$name" --region "$region" --project "$PROJECT"
+done <<<"$pools"
+while read -r name; do
+  [ -n "$name" ] || continue
+  act "firewall rule ${name}" gcp_gcloud compute firewall-rules delete "$name" --project "$PROJECT"
+done <<<"$firewalls"
+exit "$failed"
diff --git a/scripts/ops/teardown/teardown.sh b/scripts/ops/teardown/teardown.sh
index a7ca04b68..5c7ec47f7 100755
--- a/scripts/ops/teardown/teardown.sh
+++ b/scripts/ops/teardown/teardown.sh
@@ -102,6 +102,20 @@ if [ "$VERIFY_ONLY" -eq 0 ]; then
       echo
     fi
   fi
+
+  # GKE's LB leftovers (GCP parity GP-22, owner 2026-09-29): swept only on a
+  # confirmed teardown, and only once gcp-0 is gone (the script refuses otherwise).
+  if wants gcp && [ "${TM_DESTROY_CONFIRMED:-false}" = "true" ]; then
+    echo "=== sweeping the LoadBalancer leftovers GKE left in the platform VPC ==="
+    if bash "${ROOT}/scripts/ops/gcp/sweep-lb-orphans.sh" --project "${GCP_PROJECT:-ogenki-435905}" \
+         --network "${GCP_NETWORK:-vpc-europe-west4-dev}" --cluster "${GKE_CLUSTER_NAME:-gcp-0}" --apply; then
+      echo
+      echo "=== retrying the destroy after the GCP sweep ==="
+      ( cd "${ROOT}/opentofu" && terramate script run --reverse --continue-on-error destroy )
+      destroy_rc=$?
+      echo "=== terramate exit after retry: ${destroy_rc} ==="
+    fi
+  fi
 fi
 
 # ---------------------------------------------------------------------------
@@ -159,6 +173,14 @@ if wants gcp; then
       --format='value(name)' 2>/dev/null)"
     report "Forwarding rules" "$(gcloud compute forwarding-rules list --project "$project" \
       --format='value(name)' 2>/dev/null)"
+    # A deleted cluster leaves its LoadBalancers' target pools and k8s-* firewall
+    # rules behind, in no tofu state, and the firewall rules block the VPC delete
+    # (memory gke_lb_orphans_block_vpc_delete). The health-check rule is named
+    # after the node pool, not the LB, hence the prefix filter.
+    report "Target pools" "$(gcloud compute target-pools list --project "$project" \
+      --format='value(name)' 2>/dev/null)"
+    report "k8s-* firewall rules" "$(gcloud compute firewall-rules list --project "$project" \
+      --filter='name~^k8s-' --format='value(name)' 2>/dev/null)"
     report "Disks" "$(gcloud compute disks list --project "$project" \
       --format='value(name)' 2>/dev/null)"
   fi

From 781da0ca568dc00f76371bfce78c2681399cc109 Mon Sep 17 00:00:00 2001
From: Smana <smainklh@gmail.com>
Date: Tue, 29 Sep 2026 11:20:19 +0200
Subject: [PATCH 07/10] fix(gcp): a custom-role teardown fails loudly, and its
 adopt script names the burned-ID case

---
 opentofu/gcp/gke/init/variables.tf            |  4 +--
 opentofu/gcp/gke/init/workflows.tm.hcl        | 11 ++++++-
 scripts/ci/tests/test-adopt-custom-roles.sh   | 30 ++++++++++++++++---
 .../ci/tests/test-gcp-gke-init-workflow.py    | 24 ++++++++++++---
 scripts/ops/gcp/adopt-custom-roles.sh         | 18 +++++++++--
 5 files changed, 73 insertions(+), 14 deletions(-)

diff --git a/opentofu/gcp/gke/init/variables.tf b/opentofu/gcp/gke/init/variables.tf
index e48931eb7..0b437c2a0 100644
--- a/opentofu/gcp/gke/init/variables.tf
+++ b/opentofu/gcp/gke/init/variables.tf
@@ -157,9 +157,9 @@ variable "tailscale_oauth_secret_name" {
 # only, so a rebuild inside that window cannot recreate it (09-11 bug 2). The
 # destroy now keeps the roles (they grant nothing without the bindings it does
 # destroy) and the deploy adopts them; this suffix moved past the IDs already
-# burned (unsuffixed before 2026-09-11, `_v2` on 2026-09-14).
+# burned (unsuffixed before 2026-09-11, `_v2` on 2026-09-14). No default: a run
+# without variables.tfvars must fail, not plan the live roles' replacement.
 variable "custom_role_suffix" {
   description = "Generation suffix on the three custom role IDs; bump only if a generation is lost"
   type        = string
-  default     = ""
 }
diff --git a/opentofu/gcp/gke/init/workflows.tm.hcl b/opentofu/gcp/gke/init/workflows.tm.hcl
index a14d7a043..b9d7765f7 100644
--- a/opentofu/gcp/gke/init/workflows.tm.hcl
+++ b/opentofu/gcp/gke/init/workflows.tm.hcl
@@ -353,6 +353,7 @@ script "preview" {
         ${global.provisioner} init
         ${global.provisioner} validate
         trivy config --exit-code=1 --ignorefile=./.trivyignore.yaml .
+        # After a teardown the three custom roles plan as creates; deploy imports them (adopt-custom-roles.sh).
         ${global.provisioner} plan -out=out.tfplan -var-file=variables.tfvars
       BASH
       ],
@@ -396,9 +397,16 @@ script "destroy" {
         # reserved for 37 days, so the next rebuild could not recreate it. They
         # grant nothing on their own; the bindings using them are destroyed as
         # usual, and the next deploy adopts them (adopt-custom-roles.sh).
+        # Never `|| true` here: a state rm lost to a lock, backend or auth error
+        # would let the cluster destroy delete the roles and burn their IDs.
+        in_state="$(${global.provisioner} state list)"
+        keep=()
         for addr in google_project_iam_custom_role.crossplane_dns google_project_iam_custom_role.crossplane_storage google_project_iam_custom_role.crossplane_role_reader; do
-          ${global.provisioner} state rm "$addr" 2>/dev/null || true
+          if grep -qxF "$addr" <<<"$in_state"; then keep+=("$addr"); fi
         done
+        if [ "$${#keep[@]}" -gt 0 ]; then
+          ${global.provisioner} state rm -lock-timeout=5m "$${keep[@]}"
+        fi
       BASH
       ],
     ]
@@ -598,6 +606,7 @@ script "drift" "reconcile" {
       ["bash", "-c", <<-BASH
         ${global.cloud_gate}
         set -euo pipefail
+        # After a teardown the three custom roles plan as creates that fail here; run deploy, which imports them (adopt-custom-roles.sh).
         ${global.provisioner} apply -input=false -auto-approve -lock-timeout=5m -var-file=variables.tfvars drift.tfplan
       BASH
       ],
diff --git a/scripts/ci/tests/test-adopt-custom-roles.sh b/scripts/ci/tests/test-adopt-custom-roles.sh
index d67e88151..a49984f8b 100755
--- a/scripts/ci/tests/test-adopt-custom-roles.sh
+++ b/scripts/ci/tests/test-adopt-custom-roles.sh
@@ -3,6 +3,8 @@
 # adopt-custom-roles.sh (GCP parity GP-15) with gcloud and tofu stubbed on PATH:
 # a live role missing from state is imported, one in state is left alone, an
 # absent one is left to the apply, and a soft-deleted one is reported, never imported.
+# A describe that fails for any reason other than NOT_FOUND (expired credentials)
+# must stop the deploy, not be read as "absent".
 set -uo pipefail
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
 T="$(mktemp -d)"; trap 'rm -rf "$T"' EXIT
@@ -11,9 +13,16 @@ cat >"$T/bin/gcloud" <<'EOF'
 #!/usr/bin/env bash
 case "$*" in
   *"auth application-default print-access-token"*) echo fake-token ;;
-  *"roles describe xplane_dns_editor_v3 "*) echo "" ;;
+  *"roles describe xplane_dns_editor_v3 "*)
+    if [ -n "${STUB_AUTH_FAIL:-}" ]; then
+      echo "ERROR: (gcloud.iam.roles.describe) There was a problem refreshing your current auth tokens: invalid_rapt" >&2
+      exit 1
+    fi
+    echo "" ;;
   *"roles describe xplane_storage_admin_v3 "*) echo "True" ;;
-  *"roles describe xplane_role_reader_v3 "*) exit 1 ;;
+  *"roles describe xplane_role_reader_v3 "*)
+    echo "ERROR: (gcloud.iam.roles.describe) NOT_FOUND: The role named projects/ogenki-435905/roles/xplane_role_reader_v3 was not found." >&2
+    exit 1 ;;
   *) exit 1 ;;
 esac
 EOF
@@ -25,11 +34,18 @@ case "\$1" in
 esac
 EOF
 chmod +x "$T/bin/gcloud" "$T/bin/tofu"
-# out and rc are read inside check's eval strings, which shellcheck cannot see.
+adopt() { PATH="$T/bin:$PATH" bash "$ROOT/scripts/ops/gcp/adopt-custom-roles.sh" --project ogenki-435905 --suffix _v3 --apply; }
+
+# out, rc and the auth_* values are read inside check's eval strings, where the
+# linter cannot see them.
 # shellcheck disable=SC2034
-out="$(PATH="$T/bin:$PATH" bash "$ROOT/scripts/ops/gcp/adopt-custom-roles.sh" --project ogenki-435905 --suffix _v3 --apply 2>&1)"
+out="$(adopt 2>&1)"
 # shellcheck disable=SC2034
 rc=$?
+# shellcheck disable=SC2034
+auth_out="$(STUB_AUTH_FAIL=1 adopt 2>"$T/auth_err")"
+# shellcheck disable=SC2034
+auth_rc=$?
 fails=0
 check() { if eval "$2"; then echo "  ok   $1"; else echo "  FAIL $1"; fails=1; fi; }
 check "exit 0" '[ "$rc" -eq 0 ]'
@@ -37,5 +53,11 @@ check "the live dns role is imported" 'grep -q "import -var-file=variables.tfvar
 check "exactly one import" '[ "$(wc -l <"$T/imports")" -eq 1 ]'
 check "the soft-deleted storage role is reported" 'grep -q "\[deleted\].*xplane_storage_admin_v3" <<<"$out"'
 check "the absent reader role is left to the apply" 'grep -q "\[absent \].*xplane_role_reader_v3" <<<"$out"'
+check "the absent line names the burned-ID fix" 'grep "\[absent \]" <<<"$out" | grep -q "bump custom_role_suffix"'
+check "a non-NOT_FOUND describe failure exits non-zero" '[ "$auth_rc" -ne 0 ]'
+check "  ...naming the role and the credentials" 'grep -q "xplane_dns_editor_v3" "$T/auth_err" && grep -qi "credentials" "$T/auth_err"'
+check "  ...not reported as absent" '! grep -q "\[absent \].*xplane_dns_editor_v3" <<<"$auth_out$(cat "$T/auth_err")"'
+check "  ...importing nothing more" '[ "$(wc -l <"$T/imports")" -eq 1 ]'
+check "no token is ever printed" '! grep -q fake-token <<<"$out$auth_out$(cat "$T/auth_err")"'
 [ "$fails" -eq 0 ] && echo "all checks passed"
 exit "$fails"
diff --git a/scripts/ci/tests/test-gcp-gke-init-workflow.py b/scripts/ci/tests/test-gcp-gke-init-workflow.py
index 35aab7908..37a712b14 100644
--- a/scripts/ci/tests/test-gcp-gke-init-workflow.py
+++ b/scripts/ci/tests/test-gcp-gke-init-workflow.py
@@ -13,9 +13,12 @@
 TEXT = (ROOT / "opentofu/gcp/gke/init/workflows.tm.hcl").read_text()
 
 
+def job_bodies(text, name):
+    return re.findall(r'name\s*=\s*"%s"(.*?)(?=\n  job \{|\nscript "|\Z)' % re.escape(name), text, re.S)
+
+
 def job_body(text, name):
-    m = re.search(r'name\s*=\s*"%s"(.*?)(?=\n  job \{|\nscript "|\Z)' % re.escape(name), text, re.S)
-    return m.group(1) if m else ""
+    return next(iter(job_bodies(text, name)), "")
 
 
 fails = []
@@ -37,12 +40,25 @@ def before(body, first, second, what):
 if "deploy_identity_provider=${global.deploy_identity_provider_gcp}" not in apply_line:
     fails.append("stage 2's apply passes deploy_identity_provider")
 
-s1 = job_body(TEXT, "stage1-cluster")
-before(s1, "adopt-custom-roles.sh", "apply -auto-approve", "stage 1 adopts the kept custom roles before its apply")
+# Two stage1-cluster jobs: `deploy` and `deploy-stage1`.
+s1_jobs = job_bodies(TEXT, "stage1-cluster")
+if len(s1_jobs) != 2:
+    fails.append(f"expected 2 stage1-cluster jobs, found {len(s1_jobs)}")
+for n, s1 in enumerate(s1_jobs, 1):
+    before(s1, "adopt-custom-roles.sh", "apply -auto-approve",
+           f"stage1-cluster job {n} adopts the kept custom roles before its apply")
 confirm = job_body(TEXT, "confirm")
 for addr in ("crossplane_dns", "crossplane_storage", "crossplane_role_reader"):
     if f"google_project_iam_custom_role.{addr}" not in confirm or "state rm" not in confirm:
         fails.append(f"the destroy keeps google_project_iam_custom_role.{addr} out of the teardown")
+# A swallowed state rm (lock, backend, auth) lets the cluster destroy delete the
+# roles and burn their IDs for 37 days.
+state_lines = [l for l in confirm.splitlines()
+               if ("state rm" in l or "state list" in l) and not l.lstrip().startswith("#")]
+if any("|| true" in l or "2>/dev/null" in l for l in state_lines):
+    fails.append("the destroy's custom-role state rm swallows its failures")
+if not any("state rm -lock-timeout=" in l for l in state_lines):
+    fails.append("the destroy's custom-role state rm waits for the state lock")
 
 for f in fails:
     print("FAIL", f)
diff --git a/scripts/ops/gcp/adopt-custom-roles.sh b/scripts/ops/gcp/adopt-custom-roles.sh
index 1c34a37db..95d7a5dd6 100755
--- a/scripts/ops/gcp/adopt-custom-roles.sh
+++ b/scripts/ops/gcp/adopt-custom-roles.sh
@@ -32,6 +32,9 @@ ROLES=(
   "google_project_iam_custom_role.crossplane_role_reader=xplane_role_reader"
 )
 
+err="$(mktemp)"
+trap 'rm -f "$err"' EXIT
+
 in_state="$(tofu state list 2>/dev/null || true)"
 for entry in "${ROLES[@]}"; do
   addr="${entry%%=*}"
@@ -41,9 +44,18 @@ for entry in "${ROLES[@]}"; do
     echo "[ok     ] ${addr} is in state"
     continue
   fi
-  if ! deleted="$(gcp_gcloud iam roles describe "$id" --project "$PROJECT" --format='value(deleted)' 2>/dev/null)"; then
-    echo "[absent ] ${name}: the apply creates it"
-    continue
+  if ! deleted="$(gcp_gcloud iam roles describe "$id" --project "$PROJECT" --format='value(deleted)' 2>"$err")"; then
+    # A role deleted more than 7 days ago is NOT_FOUND too, yet its ID stays
+    # reserved for 37 days, so the create can still fail.
+    if grep -q 'NOT_FOUND' "$err"; then
+      echo "[absent ] ${name}: the apply creates it; if that create fails because the ID was recently deleted, bump custom_role_suffix in variables.tfvars"
+      continue
+    fi
+    # Anything else (auth, invalid_rapt) must not pass for "absent": the apply
+    # would then try to create a role that exists.
+    echo "[error  ] cannot describe ${name}; gcloud credentials may have expired (gcloud auth application-default login):" >&2
+    grep '^ERROR:' "$err" >&2 || true
+    exit 1
   fi
   if [ "$deleted" = "True" ]; then
     echo "[deleted] ${name} is soft-deleted: the provider undeletes it within 7 days; after that, bump custom_role_suffix" >&2

From 5af8f4a92ffadfa0bcaae531e9622a7a92bcf8a0 Mon Sep 17 00:00:00 2001
From: Smana <smainklh@gmail.com>
Date: Tue, 29 Sep 2026 11:26:36 +0200
Subject: [PATCH 08/10] fix(teardown): the GKE LB sweep refuses while any
 cluster lives, and retries only while the network stands

---
 scripts/ci/tests/test-teardown-gcp-sweep.sh | 63 +++++++++++++++++----
 scripts/ops/gcp/sweep-lb-orphans.sh         | 20 ++++---
 scripts/ops/tasks.yaml                      |  7 +++
 scripts/ops/teardown/teardown.sh            | 38 +++++++------
 4 files changed, 94 insertions(+), 34 deletions(-)

diff --git a/scripts/ci/tests/test-teardown-gcp-sweep.sh b/scripts/ci/tests/test-teardown-gcp-sweep.sh
index e5e211eac..b66df01d4 100755
--- a/scripts/ci/tests/test-teardown-gcp-sweep.sh
+++ b/scripts/ci/tests/test-teardown-gcp-sweep.sh
@@ -3,24 +3,40 @@
 # memory gke_lb_orphans_block_vpc_delete: a deleted GKE cluster leaves a target
 # pool and k8s-* firewall rules behind, and the firewall rules block the VPC
 # delete. teardown.sh --verify-only must report both, and fail while they exist.
+# The sweep lists LBs project-wide, so ANY live cluster must block it, and the
+# destroy is retried only while the network still stands (GP-22).
 set -uo pipefail
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
 T="$(mktemp -d)"; trap 'rm -rf "$T"' EXIT
 mkdir -p "$T/bin"
 cat >"$T/bin/gcloud" <<EOF
 #!/usr/bin/env bash
+echo "\$*" >>"$T/calls"
 case "\$*" in
   "projects describe"*) exit 0 ;;
   *"print-access-token"*) echo fake-token ;;
-  *"container clusters list"*) printf '%s' "\${CLUSTERS:-}" ;;
+  *"container clusters list"*)
+    [ -z "\${CLUSTERS_FAIL:-}" ] || exit 1
+    # Honour a name filter, as gcloud does: a guard that filters on one name
+    # must not see the other clusters.
+    want="\$(printf '%s\n' "\$@" | sed -n 's/^--filter=name=//p')"
+    for c in \${CLUSTERS:-}; do
+      [ -z "\$want" ] || [ "\$c" = "\$want" ] || continue
+      printf '%s\teurope-west4-a\tRUNNING\n' "\$c"
+    done ;;
+  *"networks describe"*) [ -n "\${NETWORK_UP:-}" ] || exit 1 ;;
   *" delete "*) echo "\$*" >>"$T/deletes" ;;
-  *"forwarding-rules list"*) echo "a2c53478 europe-west4" ;;
-  *"target-pools list"*) echo "a2c53478 europe-west4" ;;
-  *"firewall-rules list"*) printf '%s\n' k8s-fw-a2c53478 k8s-a6e2efd9258d71a1-node-http-hc ;;
+  *"forwarding-rules list"*) [ -n "\${EMPTY:-}" ] || echo "a2c53478 europe-west4" ;;
+  *"target-pools list"*) [ -n "\${EMPTY:-}" ] || echo "a2c53478 europe-west4" ;;
+  *"firewall-rules list"*) [ -n "\${EMPTY:-}" ] || printf '%s\n' k8s-fw-a2c53478 k8s-a6e2efd9258d71a1-node-http-hc ;;
   *) exit 0 ;;
 esac
 EOF
-chmod +x "$T/bin/gcloud"
+cat >"$T/bin/terramate" <<EOF
+#!/usr/bin/env bash
+echo "\$*" >>"$T/destroys"
+EOF
+chmod +x "$T/bin/gcloud" "$T/bin/terramate"
 fails=0
 f() { echo "FAIL $*"; fails=1; }
 
@@ -37,14 +53,41 @@ args=(--project ogenki-435905 --network vpc-europe-west4-dev --cluster gcp-0)
 : >"$T/deletes"
 PATH="$T/bin:$PATH" bash "$S" "${args[@]}" >/dev/null 2>&1 || f "the dry run failed"
 [ ! -s "$T/deletes" ] || f "the dry run deleted something"
-: >"$T/deletes"
+: >"$T/deletes"; : >"$T/calls"
 PATH="$T/bin:$PATH" bash "$S" "${args[@]}" --apply >/dev/null 2>&1 || f "the sweep failed"
 [ "$(wc -l <"$T/deletes")" -eq 4 ] || f "expected 4 deletes (rule, pool, 2 firewalls), got: $(cat "$T/deletes")"
 grep -q 'forwarding-rules delete a2c53478 --region europe-west4' "$T/deletes" || f "the forwarding rule was not deleted in its region"
-: >"$T/deletes"
-CLUSTERS=gcp-0 PATH="$T/bin:$PATH" bash "$S" "${args[@]}" --apply >/dev/null 2>&1; rc=$?
-[ "$rc" -eq 3 ] || f "a live cluster must refuse with exit 3, got $rc"
-[ ! -s "$T/deletes" ] || f "a live cluster's LBs were deleted"
+# The filters are the whole scoping: without them the sweep deletes by project.
+grep 'firewall-rules list' "$T/calls" | grep -q 'network~/vpc-europe-west4-dev\$' \
+  || f "the firewall list is not scoped to the network"
+for kind in forwarding-rules target-pools; do
+  grep "$kind list" "$T/calls" | grep -q 'description~kubernetes.io/service-name' \
+    || f "the $kind list is not scoped to GKE's description marker"
+done
+
+refuses() { # $1 = case, then env assignments
+  local what="$1" rc; shift
+  : >"$T/deletes"
+  env "$@" PATH="$T/bin:$PATH" bash "$S" "${args[@]}" --apply >/dev/null 2>&1; rc=$?
+  [ "$rc" -eq 3 ] || f "$what must refuse with exit 3, got $rc"
+  [ ! -s "$T/deletes" ] || f "$what, yet LBs were deleted"
+}
+refuses "the named cluster alive" CLUSTERS=gcp-0
+# The rule and pool lists are project-wide, so another cluster's LBs are in them.
+refuses "another cluster alive" CLUSTERS=gcp-1
+refuses "a failed cluster list" CLUSTERS_FAIL=1
+
+# teardown (GP-22): the destroy is retried only while the network stands.
+destroys() { # $1 = expected terramate runs, $2 = case, then env assignments
+  local want="$1" what="$2"; shift 2
+  : >"$T/destroys"
+  env "$@" TM_CLOUD=gcp TM_DESTROY_CONFIRMED=true PATH="$T/bin:$PATH" \
+    bash "$ROOT/scripts/ops/teardown/teardown.sh" >/dev/null 2>&1
+  [ "$(wc -l <"$T/destroys")" -eq "$want" ] \
+    || f "$what: expected $want destroy run(s), got $(wc -l <"$T/destroys")"
+}
+destroys 1 "a clean teardown with the network gone" EMPTY=1
+destroys 2 "a teardown with the network still standing" NETWORK_UP=1
 
 [ "$fails" -eq 0 ] && echo PASS
 exit "$fails"
diff --git a/scripts/ops/gcp/sweep-lb-orphans.sh b/scripts/ops/gcp/sweep-lb-orphans.sh
index dee927c3e..4d4ecadbb 100755
--- a/scripts/ops/gcp/sweep-lb-orphans.sh
+++ b/scripts/ops/gcp/sweep-lb-orphans.sh
@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 #
-# Delete what a deleted GKE cluster leaves in the platform VPC: its
-# LoadBalancers' forwarding rules and target pools, and the k8s-* firewall
-# rules. No tofu state holds them, forwarding rules bill hourly, and the
+# Delete what a deleted GKE cluster leaves behind: every GKE LoadBalancer's
+# forwarding rules and target pools IN THE PROJECT, and the k8s-* firewall rules
+# on the network. No tofu state holds them, forwarding rules bill hourly, and the
 # firewall rules block the VPC delete (memory gke_lb_orphans_block_vpc_delete;
-# GCP parity GP-22). Dry-run unless --apply.
+# GCP parity GP-22). Dry-run unless --apply. --cluster only names the teardown
+# in the output: the guard below is project-wide.
 #
 # Usage: sweep-lb-orphans.sh --project P --network N --cluster C [--apply]
 set -euo pipefail
@@ -25,14 +26,17 @@ done
 { [ -n "$PROJECT" ] && [ -n "$NETWORK" ] && [ -n "$CLUSTER" ]; } \
   || { echo "--project, --network and --cluster are required" >&2; exit 2; }
 
-# A live cluster still owns these: its service controller would recreate them,
-# and deleting a live LoadBalancer's rule is an outage, not a sweep.
+# The rule and pool lists below are project-wide, so ANY cluster, in any state,
+# may own what they return: deleting a live LoadBalancer's rule is an outage,
+# not a sweep, and its service controller would recreate it anyway.
 if ! clusters="$(gcp_gcloud container clusters list --project "$PROJECT" \
-                   --filter="name=${CLUSTER}" --format='value(name)')"; then
+                   --format='value(name,location,status)')"; then
   echo "cannot list clusters in ${PROJECT}: refusing to sweep blind" >&2; exit 3
 fi
 if [ -n "$clusters" ]; then
-  echo "cluster ${CLUSTER} still exists: nothing is swept while it does" >&2; exit 3
+  echo "GKE clusters still exist in ${PROJECT}; nothing is swept for ${CLUSTER} while any does:" >&2
+  sed 's/^/  /' <<<"$clusters" >&2
+  exit 3
 fi
 
 # GKE's service controller writes {"kubernetes.io/service-name": ...} into the
diff --git a/scripts/ops/tasks.yaml b/scripts/ops/tasks.yaml
index 52161fe10..5cccd508c 100644
--- a/scripts/ops/tasks.yaml
+++ b/scripts/ops/tasks.yaml
@@ -28,12 +28,19 @@ tasks:
     desc: Make the workforce pool survive a teardown and rebuild
     dir: "{{.ROOT_DIR}}/opentofu/gcp/workforce-identity"
     cmds: ["{{.TASKFILE_DIR}}/gcp/adopt-workforce-pool.sh {{.CLI_ARGS}}"]
+  gcp:adopt-custom-roles:
+    desc: Import gke/init's kept custom IAM roles so a rebuild adopts them instead of recreating them
+    dir: "{{.ROOT_DIR}}/opentofu/gcp/gke/init"
+    cmds: ["{{.TASKFILE_DIR}}/gcp/adopt-custom-roles.sh {{.CLI_ARGS}}"]
   gcp:purge-dns-records:
     desc: Empty a Cloud DNS managed zone so the zone can be destroyed
     cmds: ["{{.TASKFILE_DIR}}/gcp/purge-dns-records.sh {{.CLI_ARGS}}"]
   gcp:sweep-orphaned-disks:
     desc: Delete Persistent Disks GKE's CSI driver created that nothing uses
     cmds: ["{{.TASKFILE_DIR}}/gcp/sweep-orphaned-disks.sh {{.CLI_ARGS}}"]
+  gcp:sweep-lb-orphans:
+    desc: Delete the forwarding rules, target pools and k8s-* firewall rules GKE left (refuses while any cluster exists)
+    cmds: ["{{.TASKFILE_DIR}}/gcp/sweep-lb-orphans.sh {{.CLI_ARGS}}"]
   k8s:cnpg-prepare-restore:
     desc: Clear a CNPG cluster's live WAL archive so a new cluster can start (off the normal path)
     cmds: ["{{.TASKFILE_DIR}}/k8s/cnpg-prepare-restore.sh {{.CLI_ARGS}}"]
diff --git a/scripts/ops/teardown/teardown.sh b/scripts/ops/teardown/teardown.sh
index 5c7ec47f7..1ce266252 100755
--- a/scripts/ops/teardown/teardown.sh
+++ b/scripts/ops/teardown/teardown.sh
@@ -49,6 +49,7 @@ wants() { # $1 = lane
 }
 
 destroy_rc=0
+retry=0
 if [ "$VERIFY_ONLY" -eq 0 ]; then
   echo "=== destroying (TM_CLOUD=${CLOUDS}) ==="
   # --continue-on-error is the point: one stack whose target is already gone must
@@ -93,29 +94,34 @@ if [ "$VERIFY_ONLY" -eq 0 ]; then
         --cluster-name "${EKS_CLUSTER_NAME:-aws-0}" --region "$_region" --apply || true
       bash "${ROOT}/scripts/ops/aws/sweep-orphaned-volumes.sh" \
         --cluster-name "${EKS_CLUSTER_NAME:-aws-0}" --region "$_region" --apply || true
-
-      echo
-      echo "=== retrying the destroy after the sweep ==="
-      ( cd "${ROOT}/opentofu" && terramate script run --reverse --continue-on-error destroy )
-      destroy_rc=$?
-      echo "=== terramate exit after retry: ${destroy_rc} ==="
-      echo
+      retry=1
     fi
   fi
 
   # GKE's LB leftovers (GCP parity GP-22, owner 2026-09-29): swept only on a
-  # confirmed teardown, and only once gcp-0 is gone (the script refuses otherwise).
+  # confirmed teardown, and only once no GKE cluster is left in the project (the
+  # script refuses otherwise). Retried only while the network stands, like the
+  # AWS VPC check above: a clean teardown must not re-run the whole destroy.
   if wants gcp && [ "${TM_DESTROY_CONFIRMED:-false}" = "true" ]; then
-    echo "=== sweeping the LoadBalancer leftovers GKE left in the platform VPC ==="
-    if bash "${ROOT}/scripts/ops/gcp/sweep-lb-orphans.sh" --project "${GCP_PROJECT:-ogenki-435905}" \
-         --network "${GCP_NETWORK:-vpc-europe-west4-dev}" --cluster "${GKE_CLUSTER_NAME:-gcp-0}" --apply; then
-      echo
-      echo "=== retrying the destroy after the GCP sweep ==="
-      ( cd "${ROOT}/opentofu" && terramate script run --reverse --continue-on-error destroy )
-      destroy_rc=$?
-      echo "=== terramate exit after retry: ${destroy_rc} ==="
+    _project="${GCP_PROJECT:-ogenki-435905}"
+    _network="${GCP_NETWORK:-vpc-europe-west4-dev}"
+    echo "=== sweeping the LoadBalancer leftovers GKE left in ${_project} ==="
+    bash "${ROOT}/scripts/ops/gcp/sweep-lb-orphans.sh" --project "$_project" \
+      --network "$_network" --cluster "${GKE_CLUSTER_NAME:-gcp-0}" --apply || true
+    if gcloud compute networks describe "$_network" --project "$_project" >/dev/null 2>&1; then
+      retry=1
     fi
   fi
+
+  # One retry for every lane, so TM_CLOUD=all never destroys three times.
+  if [ "$retry" -eq 1 ]; then
+    echo
+    echo "=== retrying the destroy after the sweep ==="
+    ( cd "${ROOT}/opentofu" && terramate script run --reverse --continue-on-error destroy )
+    destroy_rc=$?
+    echo "=== terramate exit after retry: ${destroy_rc} ==="
+    echo
+  fi
 fi
 
 # ---------------------------------------------------------------------------

From 5b1246c9b0663a686ecbaf03a4ed7dd96e51bef9 Mon Sep 17 00:00:00 2001
From: Smana <smainklh@gmail.com>
Date: Tue, 29 Sep 2026 11:44:16 +0200
Subject: [PATCH 09/10] fix(teardown): the sweep test refuses to run unless the
 cloud CLIs are its stubs, and a retry's failure is explained

---
 opentofu/gcp/gke/init/variables.tf          |  5 ++-
 opentofu/gcp/gke/init/workflows.tm.hcl      |  7 +--
 scripts/ci/tests/test-teardown-gcp-sweep.sh | 47 ++++++++++++++++++---
 scripts/ops/gcp/adopt-custom-roles.sh       |  4 +-
 scripts/ops/teardown/teardown.sh            |  5 +++
 website/content/docs/reference/commands.md  |  6 +++
 6 files changed, 61 insertions(+), 13 deletions(-)

diff --git a/opentofu/gcp/gke/init/variables.tf b/opentofu/gcp/gke/init/variables.tf
index 0b437c2a0..455a1fc81 100644
--- a/opentofu/gcp/gke/init/variables.tf
+++ b/opentofu/gcp/gke/init/variables.tf
@@ -155,8 +155,9 @@ variable "tailscale_oauth_secret_name" {
 
 # GCP reserves a deleted custom role's ID for 37 days and allows undelete for 7
 # only, so a rebuild inside that window cannot recreate it (09-11 bug 2). The
-# destroy now keeps the roles (they grant nothing without the bindings it does
-# destroy) and the deploy adopts them; this suffix moved past the IDs already
+# destroy now keeps the roles (not inert: Crossplane's external-dns
+# ProjectIAMMember outlives the cluster and keeps its grant live until gcp-0
+# returns) and the deploy adopts them; this suffix moved past the IDs already
 # burned (unsuffixed before 2026-09-11, `_v2` on 2026-09-14). No default: a run
 # without variables.tfvars must fail, not plan the live roles' replacement.
 variable "custom_role_suffix" {
diff --git a/opentofu/gcp/gke/init/workflows.tm.hcl b/opentofu/gcp/gke/init/workflows.tm.hcl
index b9d7765f7..11b352b2d 100644
--- a/opentofu/gcp/gke/init/workflows.tm.hcl
+++ b/opentofu/gcp/gke/init/workflows.tm.hcl
@@ -394,9 +394,10 @@ script "destroy" {
         # dir as stage1-destroy-cluster, so that job inherits this init.
         ${global.provisioner} init -lock-timeout=5m
         # Keep the custom roles (GCP parity GP-15): a deleted role ID stays
-        # reserved for 37 days, so the next rebuild could not recreate it. They
-        # grant nothing on their own; the bindings using them are destroyed as
-        # usual, and the next deploy adopts them (adopt-custom-roles.sh).
+        # reserved for 37 days, so the next rebuild could not recreate it. The
+        # next deploy adopts them (adopt-custom-roles.sh). Kept roles are not
+        # inert: Crossplane's ProjectIAMMember for ns/kube-system/sa/external-dns
+        # outlives the cluster, so that grant stays live until gcp-0 returns.
         # Never `|| true` here: a state rm lost to a lock, backend or auth error
         # would let the cluster destroy delete the roles and burn their IDs.
         in_state="$(${global.provisioner} state list)"
diff --git a/scripts/ci/tests/test-teardown-gcp-sweep.sh b/scripts/ci/tests/test-teardown-gcp-sweep.sh
index b66df01d4..31b17bca0 100755
--- a/scripts/ci/tests/test-teardown-gcp-sweep.sh
+++ b/scripts/ci/tests/test-teardown-gcp-sweep.sh
@@ -5,14 +5,22 @@
 # delete. teardown.sh --verify-only must report both, and fail while they exist.
 # The sweep lists LBs project-wide, so ANY live cluster must block it, and the
 # destroy is retried only while the network still stands (GP-22).
+#
+# This runs the REAL teardown.sh with TM_DESTROY_CONFIRMED=true, kept offline
+# only by PATH shadowing. If a stub failed to land or cannot execute, bash would
+# resolve the real terramate and gcloud and, with live ADC, destroy gcp-0. So
+# the setup fails closed and every invocation first proves the stubs are what
+# runs (require_stubs).
 set -uo pipefail
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+set -e
 T="$(mktemp -d)"; trap 'rm -rf "$T"' EXIT
 mkdir -p "$T/bin"
 cat >"$T/bin/gcloud" <<EOF
 #!/usr/bin/env bash
 echo "\$*" >>"$T/calls"
 case "\$*" in
+  --stub-probe) echo stub ;;
   "projects describe"*) exit 0 ;;
   *"print-access-token"*) echo fake-token ;;
   *"container clusters list"*)
@@ -34,14 +42,36 @@ esac
 EOF
 cat >"$T/bin/terramate" <<EOF
 #!/usr/bin/env bash
+[ "\$1" != --stub-probe ] || { echo stub; exit 0; }
 echo "\$*" >>"$T/destroys"
 EOF
-chmod +x "$T/bin/gcloud" "$T/bin/terramate"
+# Never called with TM_CLOUD=gcp; stubbed so a regression cannot reach AWS either.
+cat >"$T/bin/aws" <<EOF
+#!/usr/bin/env bash
+[ "\$1" != --stub-probe ] || { echo stub; exit 0; }
+echo "\$*" >>"$T/aws-calls"; exit 1
+EOF
+chmod +x "$T/bin/gcloud" "$T/bin/terramate" "$T/bin/aws"
+set +e
+export PATH="$T/bin:$PATH"
+
+require_stubs() { # abort the whole test unless every cloud CLI is the stub
+  local tool
+  for tool in gcloud terramate aws; do
+    if [ "$(command -v "$tool")" != "$T/bin/$tool" ] \
+       || [ "$("$T/bin/$tool" --stub-probe 2>/dev/null)" != stub ]; then
+      echo "ABORT: $tool resolves to '$(command -v "$tool")', not the stub in $T/bin;" \
+           "refusing to run teardown.sh or the sweep against a real cloud" >&2
+      exit 1
+    fi
+  done
+}
 fails=0
 f() { echo "FAIL $*"; fails=1; }
 
 # The report (--verify-only).
-out="$(TM_CLOUD=gcp PATH="$T/bin:$PATH" bash "$ROOT/scripts/ops/teardown/teardown.sh" --verify-only 2>&1)"; rc=$?
+require_stubs
+out="$(TM_CLOUD=gcp bash "$ROOT/scripts/ops/teardown/teardown.sh" --verify-only 2>&1)"; rc=$?
 grep -q 'Target pools' <<<"$out" || f "no Target pools row"
 grep -q 'k8s-\* firewall rules' <<<"$out" || f "no k8s-* firewall rules row"
 [ "$rc" -ne 0 ] || f "leftovers must fail the verify"
@@ -51,10 +81,12 @@ S="$ROOT/scripts/ops/gcp/sweep-lb-orphans.sh"
 args=(--project ogenki-435905 --network vpc-europe-west4-dev --cluster gcp-0)
 [ -f "$S" ] || f "no $S"
 : >"$T/deletes"
-PATH="$T/bin:$PATH" bash "$S" "${args[@]}" >/dev/null 2>&1 || f "the dry run failed"
+require_stubs
+bash "$S" "${args[@]}" >/dev/null 2>&1 || f "the dry run failed"
 [ ! -s "$T/deletes" ] || f "the dry run deleted something"
 : >"$T/deletes"; : >"$T/calls"
-PATH="$T/bin:$PATH" bash "$S" "${args[@]}" --apply >/dev/null 2>&1 || f "the sweep failed"
+require_stubs
+bash "$S" "${args[@]}" --apply >/dev/null 2>&1 || f "the sweep failed"
 [ "$(wc -l <"$T/deletes")" -eq 4 ] || f "expected 4 deletes (rule, pool, 2 firewalls), got: $(cat "$T/deletes")"
 grep -q 'forwarding-rules delete a2c53478 --region europe-west4' "$T/deletes" || f "the forwarding rule was not deleted in its region"
 # The filters are the whole scoping: without them the sweep deletes by project.
@@ -68,7 +100,8 @@ done
 refuses() { # $1 = case, then env assignments
   local what="$1" rc; shift
   : >"$T/deletes"
-  env "$@" PATH="$T/bin:$PATH" bash "$S" "${args[@]}" --apply >/dev/null 2>&1; rc=$?
+  require_stubs
+  env "$@" bash "$S" "${args[@]}" --apply >/dev/null 2>&1; rc=$?
   [ "$rc" -eq 3 ] || f "$what must refuse with exit 3, got $rc"
   [ ! -s "$T/deletes" ] || f "$what, yet LBs were deleted"
 }
@@ -81,8 +114,8 @@ refuses "a failed cluster list" CLUSTERS_FAIL=1
 destroys() { # $1 = expected terramate runs, $2 = case, then env assignments
   local want="$1" what="$2"; shift 2
   : >"$T/destroys"
-  env "$@" TM_CLOUD=gcp TM_DESTROY_CONFIRMED=true PATH="$T/bin:$PATH" \
-    bash "$ROOT/scripts/ops/teardown/teardown.sh" >/dev/null 2>&1
+  require_stubs
+  env "$@" TM_CLOUD=gcp TM_DESTROY_CONFIRMED=true bash "$ROOT/scripts/ops/teardown/teardown.sh" >/dev/null 2>&1
   [ "$(wc -l <"$T/destroys")" -eq "$want" ] \
     || f "$what: expected $want destroy run(s), got $(wc -l <"$T/destroys")"
 }
diff --git a/scripts/ops/gcp/adopt-custom-roles.sh b/scripts/ops/gcp/adopt-custom-roles.sh
index 95d7a5dd6..5da8ad27e 100755
--- a/scripts/ops/gcp/adopt-custom-roles.sh
+++ b/scripts/ops/gcp/adopt-custom-roles.sh
@@ -58,7 +58,9 @@ for entry in "${ROLES[@]}"; do
     exit 1
   fi
   if [ "$deleted" = "True" ]; then
-    echo "[deleted] ${name} is soft-deleted: the provider undeletes it within 7 days; after that, bump custom_role_suffix" >&2
+    # describe still answers only inside the 7-day undelete window; past it the
+    # role reads NOT_FOUND and takes the [absent ] branch above.
+    echo "[deleted] ${name} is soft-deleted: the apply's create undeletes it (GCP allows that for 7 days)" >&2
     continue
   fi
   if [ "$APPLY" = true ]; then
diff --git a/scripts/ops/teardown/teardown.sh b/scripts/ops/teardown/teardown.sh
index 1ce266252..ddf71b21f 100755
--- a/scripts/ops/teardown/teardown.sh
+++ b/scripts/ops/teardown/teardown.sh
@@ -203,6 +203,11 @@ else
   [ "$unverified" -gt 0 ] && \
     echo "  cloud:      ${unverified} cloud(s) COULD NOT BE CHECKED — status unknown, assume not torn down"
 fi
+# Not auto-skipped: if the first pass never reached openbao/cluster, skipping
+# its pre-destroy snapshot would destroy a node without one.
+if [ "$retry" -eq 1 ] && [ "$destroy_rc" -ne 0 ] && [ "$leftovers" -eq 0 ] && [ "$unverified" -eq 0 ]; then
+  echo "  note:       the non-zero exit came from the retry against already-destroyed stacks (e.g. openbao/cluster's snapshot of a gone node); re-run with TM_OPENBAO_SKIP_SNAPSHOT=true ONLY if the first pass's snapshot succeeded"
+fi
 
 # Exit non-zero if the destroy failed, OR anything is still standing, OR a cloud
 # could not be checked. A green exit here means the provider itself reports
diff --git a/website/content/docs/reference/commands.md b/website/content/docs/reference/commands.md
index 12fedc9a0..efac71f6b 100644
--- a/website/content/docs/reference/commands.md
+++ b/website/content/docs/reference/commands.md
@@ -144,6 +144,12 @@ gcloud compute disks list --project <project>
 gcloud compute addresses list --project <project>
 ```
 
+`scripts/ops/teardown/teardown.sh` does both, sweeps GKE's LoadBalancer
+leftovers, and retries once while the VPC stands. A non-zero terramate exit
+beside `cloud: clean` comes from that retry failing on already-destroyed stacks
+(typically `opentofu/gcp/openbao/cluster`'s snapshot of a gone node): re-run
+with `TM_OPENBAO_SKIP_SNAPSHOT=true` only if the first pass's snapshot succeeded.
+
 ## Opt-in stacks
 
 `opentofu/aws/llm-platform/` is tagged `opt-in` (see `opentofu/aws/llm-platform/workflows.tm.hcl`):

From 2b0ce63c5eecb2b0be53154b979b657c6420ab3e Mon Sep 17 00:00:00 2001
From: Smana <smainklh@gmail.com>
Date: Tue, 29 Sep 2026 11:48:20 +0200
Subject: [PATCH 10/10] fix(ci): the teardown test's stub directory is absolute

A relative TMPDIR made the stub PATH entry relative. The guard passed from
the test's cwd, then teardown.sh cd'd into opentofu/ where the entry no longer
resolved, and the real terramate ran with the destroy confirmed. The stub
directory is now resolved to an absolute path, and the guard refuses a
relative one.
---
 scripts/ci/tests/test-teardown-gcp-sweep.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/ci/tests/test-teardown-gcp-sweep.sh b/scripts/ci/tests/test-teardown-gcp-sweep.sh
index 31b17bca0..678b290d2 100755
--- a/scripts/ci/tests/test-teardown-gcp-sweep.sh
+++ b/scripts/ci/tests/test-teardown-gcp-sweep.sh
@@ -14,7 +14,9 @@
 set -uo pipefail
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
 set -e
-T="$(mktemp -d)"; trap 'rm -rf "$T"' EXIT
+# Absolute: teardown.sh cd's before calling terramate, and a relative PATH entry
+# stops resolving there, letting the real CLI run with the destroy confirmed.
+T="$(cd "$(mktemp -d)" && pwd -P)"; trap 'rm -rf "$T"' EXIT
 mkdir -p "$T/bin"
 cat >"$T/bin/gcloud" <<EOF
 #!/usr/bin/env bash
@@ -57,6 +59,7 @@ export PATH="$T/bin:$PATH"
 
 require_stubs() { # abort the whole test unless every cloud CLI is the stub
   local tool
+  case "$T" in /*) ;; *) echo "ABORT: stub dir '$T' is not absolute" >&2; exit 1 ;; esac
   for tool in gcloud terramate aws; do
     if [ "$(command -v "$tool")" != "$T/bin/$tool" ] \
        || [ "$("$T/bin/$tool" --stub-probe 2>/dev/null)" != stub ]; then