fix(gcp): GKE rebuilds that can repeat

infrastructure/gcp-0/external-dns/kustomization.yaml

@@ -16,3 +16,13 @@ patches:
       group: helm.toolkit.fluxcd.io
       kind: HelmRelease
       name: external-dns
+  # base waits on aws-load-balancer-controller (#1975's webhook race), a release
+  # gcp-0 never runs: GKE's own controller creates its LoadBalancers. Kept, the
+  # edge never resolves and no private record is ever written (09-11 bug 6).
+  - target:
+      group: helm.toolkit.fluxcd.io
+      kind: HelmRelease
+      name: external-dns
+    patch: |-
+      - op: remove
+        path: /spec/dependsOn

infrastructure/gcp-0/external-dns/workloadidentity.yaml

@@ -29,4 +29,4 @@ spec:
   serviceAccount:
     name: external-dns
   roles:
-    - "projects/${project_id}/roles/xplane_dns_editor"
+    - "${gcp_dns_editor_role}"

opentofu/gcp/gke/configure/kubernetes.tf

@@ -62,6 +62,10 @@ resource "kubectl_manifest" "flux_cluster_vars" {
       storage_class = local.storage_class
 
       # GCP-specific.
+      # Custom role names carry a generation suffix (gke/init var.custom_role_suffix),
+      # so claims read the name rather than spelling it.
+      gcp_dns_editor_role = local.init.dns_editor_role
+
       project_id     = var.project_id
       project_number = local.init.project_number
       zone           = local.init.cluster_location

opentofu/gcp/gke/init/iam.tf

@@ -75,7 +75,7 @@
 # spelled `xplane_` here.
 resource "google_project_iam_custom_role" "crossplane_dns" {
   project     = var.project_id
-  role_id     = "xplane_dns_editor"
+  role_id     = "xplane_dns_editor${var.custom_role_suffix}"
   title       = "Crossplane DNS editor"
   description = "Record-set management for external-dns and cert-manager DNS-01. Deliberately excludes zone deletion and response policies; see opentofu/gcp/gke/init/iam.tf."
 
@@ -131,7 +131,7 @@ resource "google_project_iam_custom_role" "crossplane_dns" {
 # pattern as crossplane_dns.
 resource "google_project_iam_custom_role" "crossplane_storage" {
   project     = var.project_id
-  role_id     = "xplane_storage_admin"
+  role_id     = "xplane_storage_admin${var.custom_role_suffix}"
   title       = "Crossplane storage bucket admin"
   description = "Bucket lifecycle + IAM for the App Composition's per-app buckets. See opentofu/gcp/gke/init/iam.tf."
 
@@ -234,6 +234,11 @@ locals {
   crossplane_bucket_grantable_roles = [
     "roles/storage.objectAdmin",
     "roles/storage.objectViewer",
+    # openbao-snapshot's bucket grant (security/gcp-0/openbao-snapshot): create
+    # only, strictly weaker than objectAdmin above. Missing, the condition denied
+    # the grant with a bare 403 and gcp-0 never took a scheduled snapshot
+    # (GCP parity GP-23, 09-11 bug 7).
+    "roles/storage.objectCreator",
     "roles/storage.legacyBucketReader",
   ]
 
@@ -365,7 +370,7 @@ resource "google_project_iam_member" "crossplane_storage" {
 # read requests, so a condition here would be vacuous anyway (see gap 1 below).
 resource "google_project_iam_custom_role" "crossplane_role_reader" {
   project     = var.project_id
-  role_id     = "xplane_role_reader"
+  role_id     = "xplane_role_reader${var.custom_role_suffix}"
   title       = "Crossplane role reader"
   description = "Read-only on IAM role definitions, so ProjectIAMMember can reference the custom DNS role. Confers no grant capability."
 

opentofu/gcp/gke/init/outputs.tf

@@ -94,3 +94,8 @@ output "private_domain_name" {
   description = "Cloud DNS private zone domain, substituted into manifests as $${private_domain_name}"
   value       = local.net.private_domain_name
 }
+
+output "dns_editor_role" {
+  description = "Full name of the custom DNS role, suffix included, for GCPWorkloadIdentity claims"
+  value       = google_project_iam_custom_role.crossplane_dns.name
+}

opentofu/gcp/gke/init/variables.tf

@@ -152,3 +152,15 @@ variable "tailscale_oauth_secret_name" {
   type        = string
   default     = "tailscale-k8s-operator-oauth"
 }
+
+# GCP reserves a deleted custom role's ID for 37 days and allows undelete for 7
+# only, so a rebuild inside that window cannot recreate it (09-11 bug 2). The
+# destroy now keeps the roles (not inert: Crossplane's external-dns
+# ProjectIAMMember outlives the cluster and keeps its grant live until gcp-0
+# returns) and the deploy adopts them; this suffix moved past the IDs already
+# burned (unsuffixed before 2026-09-11, `_v2` on 2026-09-14). No default: a run
+# without variables.tfvars must fail, not plan the live roles' replacement.
+variable "custom_role_suffix" {
+  description = "Generation suffix on the three custom role IDs; bump only if a generation is lost"
+  type        = string
+}

opentofu/gcp/gke/init/variables.tfvars

@@ -5,6 +5,8 @@ region     = "europe-west4"
 cluster_name    = "gcp-0"
 release_channel = "REGULAR"
 
+custom_role_suffix = "_v3"
+
 # Slice 4 must pin the SAME image type on every ComputeClass.
 node_image_type   = "COS_CONTAINERD"
 node_machine_type = "e2-standard-4"

opentofu/gcp/gke/init/workflows.tm.hcl

@@ -90,6 +90,9 @@ script "deploy" {
         set -euo pipefail
         ${global.provisioner} init
         ${global.provisioner} validate
+        bash "${terramate.root.path.fs.absolute}/scripts/ops/gcp/adopt-custom-roles.sh" \
+          --project ogenki-435905 \
+          --suffix "$(awk -F'"' '/^custom_role_suffix/{print $2}' variables.tfvars)" --apply
         trivy config --exit-code=1 --ignorefile=./.trivyignore.yaml .
         ${global.provisioner} apply -auto-approve -var-file=variables.tfvars
       BASH
@@ -105,13 +108,28 @@ script "deploy" {
         ${global.cloud_gate}
         set -euo pipefail
         cd ../configure
+        # ../configure's vault provider needs the CA chain before init, and a
+        # fresh checkout has no .tls/ (gitignored). gke/configure's own deploy
+        # writes it; this inline apply bypasses that script (09-11 bug 3).
+        bash "${terramate.root.path.fs.absolute}/scripts/provision/openbao-config.sh" ca \
+          --cloud gcp --project ogenki-435905 \
+          --root-ca-secret-name openbao-priv-gcp-ca-chain --ca-output-file .tls/ca.pem
         ${global.provisioner} init -lock-timeout=5m
-        ${global.provisioner} apply -auto-approve -var-file=variables.tfvars -var='cilium_version=${global.cilium_version}' -var='gateway_api_version=${global.gateway_api_version}' -var='flux_operator_version=${global.flux_operator_version}' -var='flux_instance_version=${global.flux_instance_version}' $${TF_VAR_flux_git_ref:+-var="flux_git_ref=$${TF_VAR_flux_git_ref}"}
+        # A restored lineage already holds jwt/gcp-0, and creating it again 400s
+        # with "path is already in use". Same call as gke/configure's deploy.
+        bash "${terramate.root.path.fs.absolute}/scripts/provision/openbao-adopt-jwt-mount.sh" \
+          --cluster-name gcp-0 --url https://bao.priv.gcp.ogenki.io:8200 \
+          --root-token-secret-name openbao-priv-gcp-root-token \
+          --ca-file .tls/ca.pem --cloud gcp --project ogenki-435905 \
+          -- -var='cilium_version=${global.cilium_version}' -var='gateway_api_version=${global.gateway_api_version}' -var='flux_operator_version=${global.flux_operator_version}' -var='flux_instance_version=${global.flux_instance_version}' -var='deploy_identity_provider=${global.deploy_identity_provider_gcp}' $${TF_VAR_flux_git_ref:+-var="flux_git_ref=$${TF_VAR_flux_git_ref}"}
+        # deploy_identity_provider here too: without it this apply publishes the
+        # consumed (AWS) identity_provider_url until the standalone configure run.
+        ${global.provisioner} apply -auto-approve -var-file=variables.tfvars -var='cilium_version=${global.cilium_version}' -var='gateway_api_version=${global.gateway_api_version}' -var='flux_operator_version=${global.flux_operator_version}' -var='flux_instance_version=${global.flux_instance_version}' -var='deploy_identity_provider=${global.deploy_identity_provider_gcp}' $${TF_VAR_flux_git_ref:+-var="flux_git_ref=$${TF_VAR_flux_git_ref}"}
         # Forget flux-operator here, in the job that just created it -- NOT only
         # in gke/configure's own `deploy`, which this job bypasses. Left in state,
         # the standalone gke/configure stack plans count=0 against a resource that
-        # IS in state, and that is a destroy: a real `helm uninstall`. The AWS lane
-        # had the identical gap and hit it on 2026-09-16.
+        # IS in state, and that is a destroy: a real `helm uninstall`. The AWS
+        # lane had the identical gap and hit it on 2026-09-16.
         ${global.provisioner} state rm helm_release.flux_operator 2>/dev/null || true
       BASH
       ],
@@ -312,6 +330,9 @@ script "deploy-stage1" {
         set -euo pipefail
         ${global.provisioner} init
         ${global.provisioner} validate
+        bash "${terramate.root.path.fs.absolute}/scripts/ops/gcp/adopt-custom-roles.sh" \
+          --project ogenki-435905 \
+          --suffix "$(awk -F'"' '/^custom_role_suffix/{print $2}' variables.tfvars)" --apply
         trivy config --exit-code=1 --ignorefile=./.trivyignore.yaml .
         ${global.provisioner} apply -auto-approve -var-file=variables.tfvars
       BASH
@@ -332,6 +353,7 @@ script "preview" {
         ${global.provisioner} init
         ${global.provisioner} validate
         trivy config --exit-code=1 --ignorefile=./.trivyignore.yaml .
+        # After a teardown the three custom roles plan as creates; deploy imports them (adopt-custom-roles.sh).
         ${global.provisioner} plan -out=out.tfplan -var-file=variables.tfvars
       BASH
       ],
@@ -371,6 +393,21 @@ script "destroy" {
         # must fail here, not after resources have started disappearing. Same stack
         # dir as stage1-destroy-cluster, so that job inherits this init.
         ${global.provisioner} init -lock-timeout=5m
+        # Keep the custom roles (GCP parity GP-15): a deleted role ID stays
+        # reserved for 37 days, so the next rebuild could not recreate it. The
+        # next deploy adopts them (adopt-custom-roles.sh). Kept roles are not
+        # inert: Crossplane's ProjectIAMMember for ns/kube-system/sa/external-dns
+        # outlives the cluster, so that grant stays live until gcp-0 returns.
+        # Never `|| true` here: a state rm lost to a lock, backend or auth error
+        # would let the cluster destroy delete the roles and burn their IDs.
+        in_state="$(${global.provisioner} state list)"
+        keep=()
+        for addr in google_project_iam_custom_role.crossplane_dns google_project_iam_custom_role.crossplane_storage google_project_iam_custom_role.crossplane_role_reader; do
+          if grep -qxF "$addr" <<<"$in_state"; then keep+=("$addr"); fi
+        done
+        if [ "$${#keep[@]}" -gt 0 ]; then
+          ${global.provisioner} state rm -lock-timeout=5m "$${keep[@]}"
+        fi
       BASH
       ],
     ]
@@ -570,6 +607,7 @@ script "drift" "reconcile" {
       ["bash", "-c", <<-BASH
         ${global.cloud_gate}
         set -euo pipefail
+        # After a teardown the three custom roles plan as creates that fail here; run deploy, which imports them (adopt-custom-roles.sh).
         ${global.provisioner} apply -input=false -auto-approve -lock-timeout=5m -var-file=variables.tfvars drift.tfplan
       BASH
       ],

scripts/ci/flux-schema/render-bundle.py

@@ -140,6 +140,8 @@ def _repo_lock(url):
     # is what lets CI catch a consumer that assumes a string.
     "project_number": "323586397743",
     "workload_pool": "ogenki-435905.svc.id.goog",
+    # gcp-0 only: the custom DNS role's full name, generation suffix included.
+    "gcp_dns_editor_role": "projects/ogenki-435905/roles/xplane_dns_editor_v3",
     "zone": "europe-west4-a",
     "network_name": "vpc-foobar",
     # Federated Route53 path (workstream 12). GCP-only: aws-0 authenticates to

scripts/ci/tests/test-adopt-custom-roles.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+#
+# adopt-custom-roles.sh (GCP parity GP-15) with gcloud and tofu stubbed on PATH:
+# a live role missing from state is imported, one in state is left alone, an
+# absent one is left to the apply, and a soft-deleted one is reported, never imported.
+# A describe that fails for any reason other than NOT_FOUND (expired credentials)
+# must stop the deploy, not be read as "absent".
+set -uo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+T="$(mktemp -d)"; trap 'rm -rf "$T"' EXIT
+mkdir -p "$T/bin"
+cat >"$T/bin/gcloud" <<'EOF'
+#!/usr/bin/env bash
+case "$*" in
+  *"auth application-default print-access-token"*) echo fake-token ;;
+  *"roles describe xplane_dns_editor_v3 "*)
+    if [ -n "${STUB_AUTH_FAIL:-}" ]; then
+      echo "ERROR: (gcloud.iam.roles.describe) There was a problem refreshing your current auth tokens: invalid_rapt" >&2
+      exit 1
+    fi
+    echo "" ;;
+  *"roles describe xplane_storage_admin_v3 "*) echo "True" ;;
+  *"roles describe xplane_role_reader_v3 "*)
+    echo "ERROR: (gcloud.iam.roles.describe) NOT_FOUND: The role named projects/ogenki-435905/roles/xplane_role_reader_v3 was not found." >&2
+    exit 1 ;;
+  *) exit 1 ;;
+esac
+EOF
+cat >"$T/bin/tofu" <<EOF
+#!/usr/bin/env bash
+case "\$1" in
+  state) printf '%s\n' module.gke.google_container_cluster.primary ;;
+  import) echo "\$*" >>"$T/imports" ;;
+esac
+EOF
+chmod +x "$T/bin/gcloud" "$T/bin/tofu"
+adopt() { PATH="$T/bin:$PATH" bash "$ROOT/scripts/ops/gcp/adopt-custom-roles.sh" --project ogenki-435905 --suffix _v3 --apply; }
+
+# out, rc and the auth_* values are read inside check's eval strings, where the
+# linter cannot see them.
+# shellcheck disable=SC2034
+out="$(adopt 2>&1)"
+# shellcheck disable=SC2034
+rc=$?
+# shellcheck disable=SC2034
+auth_out="$(STUB_AUTH_FAIL=1 adopt 2>"$T/auth_err")"
+# shellcheck disable=SC2034
+auth_rc=$?
+fails=0
+check() { if eval "$2"; then echo "  ok   $1"; else echo "  FAIL $1"; fails=1; fi; }
+check "exit 0" '[ "$rc" -eq 0 ]'
+check "the live dns role is imported" 'grep -q "import -var-file=variables.tfvars google_project_iam_custom_role.crossplane_dns projects/ogenki-435905/roles/xplane_dns_editor_v3" "$T/imports"'
+check "exactly one import" '[ "$(wc -l <"$T/imports")" -eq 1 ]'
+check "the soft-deleted storage role is reported" 'grep -q "\[deleted\].*xplane_storage_admin_v3" <<<"$out"'
+check "the absent reader role is left to the apply" 'grep -q "\[absent \].*xplane_role_reader_v3" <<<"$out"'
+check "the absent line names the burned-ID fix" 'grep "\[absent \]" <<<"$out" | grep -q "bump custom_role_suffix"'
+check "a non-NOT_FOUND describe failure exits non-zero" '[ "$auth_rc" -ne 0 ]'
+check "  ...naming the role and the credentials" 'grep -q "xplane_dns_editor_v3" "$T/auth_err" && grep -qi "credentials" "$T/auth_err"'
+check "  ...not reported as absent" '! grep -q "\[absent \].*xplane_dns_editor_v3" <<<"$auth_out$(cat "$T/auth_err")"'
+check "  ...importing nothing more" '[ "$(wc -l <"$T/imports")" -eq 1 ]'
+check "no token is ever printed" '! grep -q fake-token <<<"$out$auth_out$(cat "$T/auth_err")"'
+[ "$fails" -eq 0 ] && echo "all checks passed"
+exit "$fails"

scripts/ci/tests/test-gcp-bucket-grant-allowlist.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# requires: python3
+#
+# GCP parity GP-23 (09-11 bug 7; memory gcp_crossplane_grant_allowlist_contract):
+# every role a GCPWorkloadIdentity grants under bucketRoles must be in gke/init's
+# crossplane_bucket_grantable_roles, or the IAM condition denies the grant with a
+# bare 403. openbao-snapshot's objectCreator was missing, so gcp-0 never took a
+# scheduled snapshot.
+set -uo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+python3 - "$ROOT" <<'PY'
+import pathlib, re, sys
+root = pathlib.Path(sys.argv[1])
+iam = (root / "opentofu/gcp/gke/init/iam.tf").read_text()
+m = re.search(r"crossplane_bucket_grantable_roles\s*=\s*\[(.*?)\]", iam, re.S)
+if not m:
+    print("FAIL no crossplane_bucket_grantable_roles in iam.tf"); sys.exit(1)
+allowed = set(re.findall(r'"(roles/[^"]+)"', m.group(1)))
+wanted = {}
+for area in ("infrastructure", "security", "observability", "tooling", "apps"):
+    for f in (root / area).glob("gcp-0/**/*.yaml"):
+        text = f.read_text()
+        if "kind: GCPWorkloadIdentity" not in text or "bucketRoles:" not in text:
+            continue
+        for role in re.findall(r"^\s+role:\s*(roles/\S+)", text.split("bucketRoles:", 1)[1], re.M):
+            wanted.setdefault(role, []).append(str(f.relative_to(root)))
+if not wanted:
+    print("FAIL found no bucketRoles grant at all: the scan is broken"); sys.exit(1)
+missing = {r: fs for r, fs in wanted.items() if r not in allowed}
+for r, fs in sorted(missing.items()):
+    print(f"FAIL {r} (asked by {', '.join(fs)}) is not in crossplane_bucket_grantable_roles")
+if missing:
+    sys.exit(1)
+print("PASS")
+PY

scripts/ci/tests/test-gcp-custom-role-refs.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# GCP parity GP-15: every custom role carries the generation suffix, and no
+# manifest spells a custom role name -- they read ${gcp_dns_editor_role}, so a
+# suffix bump never leaves a claim pointing at a deleted role.
+set -uo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+fails=0
+while IFS= read -r line; do
+  case "$line" in
+    *'${var.custom_role_suffix}"'*) ;;
+    *) echo "FAIL iam.tf: $line"; fails=$((fails + 1)) ;;
+  esac
+done < <(grep -E '^[[:space:]]*role_id[[:space:]]*=' "$ROOT/opentofu/gcp/gke/init/iam.tf")
+[ "$(grep -cE '^[[:space:]]*role_id[[:space:]]*=' "$ROOT/opentofu/gcp/gke/init/iam.tf")" -eq 3 ] \
+  || { echo "FAIL expected 3 custom roles in iam.tf"; fails=$((fails + 1)); }
+hits="$(grep -rnE 'roles/xplane_' --include=*.yaml "$ROOT"/{clusters,infrastructure,security,observability,tooling,apps} 2>/dev/null)"
+[ -z "$hits" ] || { echo "FAIL a manifest spells a custom role:"; echo "$hits"; fails=$((fails + 1)); }
+grep -q 'gcp_dns_editor_role' "$ROOT/opentofu/gcp/gke/configure/kubernetes.tf" \
+  || { echo "FAIL gke-gcp-0-vars has no gcp_dns_editor_role"; fails=$((fails + 1)); }
+[ "$fails" -eq 0 ] || exit 1
+echo PASS

scripts/ci/tests/test-gcp-external-dns-deps.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# requires: python3 kustomize
+"""GCP parity GP-27 (09-11 bug 6): gcp-0's private external-dns must not wait on
+aws-load-balancer-controller, which gcp-0 never runs. With the edge, the release
+never installs and no *.priv.gcp record is written, while the parent
+Kustomization (no `wait`) still reports Ready."""
+import pathlib
+import subprocess
+import sys
+
+try:
+    import yaml
+except ImportError:
+    print("PyYAML is not installed")
+    sys.exit(77)
+
+ROOT = pathlib.Path(__file__).resolve().parents[3]
+out = subprocess.run(["kustomize", "build", str(ROOT / "infrastructure/gcp-0/external-dns"),
+                      "--load-restrictor=LoadRestrictionsNone"], capture_output=True, text=True, check=True).stdout
+hrs = [d for d in yaml.safe_load_all(out) if d and d.get("kind") == "HelmRelease" and d["metadata"]["name"] == "external-dns"]
+if len(hrs) != 1:
+    print(f"FAIL expected one external-dns HelmRelease, found {len(hrs)}")
+    sys.exit(1)
+deps = hrs[0]["spec"].get("dependsOn") or []
+if any(d.get("name") == "aws-load-balancer-controller" for d in deps):
+    print("FAIL gcp-0's external-dns depends on aws-load-balancer-controller, which gcp-0 never runs")
+    sys.exit(1)
+print("PASS")