diff --git a/infrastructure/gcp-0/external-dns/kustomization.yaml b/infrastructure/gcp-0/external-dns/kustomization.yaml index f9d203d77..a93255163 100644 --- a/infrastructure/gcp-0/external-dns/kustomization.yaml +++ b/infrastructure/gcp-0/external-dns/kustomization.yaml @@ -16,3 +16,13 @@ patches: group: helm.toolkit.fluxcd.io kind: HelmRelease name: external-dns + # base waits on aws-load-balancer-controller (#1975's webhook race), a release + # gcp-0 never runs: GKE's own controller creates its LoadBalancers. Kept, the + # edge never resolves and no private record is ever written (09-11 bug 6). + - target: + group: helm.toolkit.fluxcd.io + kind: HelmRelease + name: external-dns + patch: |- + - op: remove + path: /spec/dependsOn diff --git a/infrastructure/gcp-0/external-dns/workloadidentity.yaml b/infrastructure/gcp-0/external-dns/workloadidentity.yaml index ec8bb7269..810ea2dc4 100644 --- a/infrastructure/gcp-0/external-dns/workloadidentity.yaml +++ b/infrastructure/gcp-0/external-dns/workloadidentity.yaml @@ -29,4 +29,4 @@ spec: serviceAccount: name: external-dns roles: - - "projects/${project_id}/roles/xplane_dns_editor" + - "${gcp_dns_editor_role}" diff --git a/opentofu/gcp/gke/configure/kubernetes.tf b/opentofu/gcp/gke/configure/kubernetes.tf index ed100d0d6..edb717d90 100644 --- a/opentofu/gcp/gke/configure/kubernetes.tf +++ b/opentofu/gcp/gke/configure/kubernetes.tf @@ -62,6 +62,10 @@ resource "kubectl_manifest" "flux_cluster_vars" { storage_class = local.storage_class # GCP-specific. + # Custom role names carry a generation suffix (gke/init var.custom_role_suffix), + # so claims read the name rather than spelling it. + gcp_dns_editor_role = local.init.dns_editor_role + project_id = var.project_id project_number = local.init.project_number zone = local.init.cluster_location diff --git a/opentofu/gcp/gke/init/iam.tf b/opentofu/gcp/gke/init/iam.tf index 8caef5d89..a3b5cd36c 100644 --- a/opentofu/gcp/gke/init/iam.tf +++ b/opentofu/gcp/gke/init/iam.tf @@ -75,7 +75,7 @@ # spelled `xplane_` here. resource "google_project_iam_custom_role" "crossplane_dns" { project = var.project_id - role_id = "xplane_dns_editor" + role_id = "xplane_dns_editor${var.custom_role_suffix}" title = "Crossplane DNS editor" description = "Record-set management for external-dns and cert-manager DNS-01. Deliberately excludes zone deletion and response policies; see opentofu/gcp/gke/init/iam.tf." @@ -131,7 +131,7 @@ resource "google_project_iam_custom_role" "crossplane_dns" { # pattern as crossplane_dns. resource "google_project_iam_custom_role" "crossplane_storage" { project = var.project_id - role_id = "xplane_storage_admin" + role_id = "xplane_storage_admin${var.custom_role_suffix}" title = "Crossplane storage bucket admin" description = "Bucket lifecycle + IAM for the App Composition's per-app buckets. See opentofu/gcp/gke/init/iam.tf." @@ -234,6 +234,11 @@ locals { crossplane_bucket_grantable_roles = [ "roles/storage.objectAdmin", "roles/storage.objectViewer", + # openbao-snapshot's bucket grant (security/gcp-0/openbao-snapshot): create + # only, strictly weaker than objectAdmin above. Missing, the condition denied + # the grant with a bare 403 and gcp-0 never took a scheduled snapshot + # (GCP parity GP-23, 09-11 bug 7). + "roles/storage.objectCreator", "roles/storage.legacyBucketReader", ] @@ -365,7 +370,7 @@ resource "google_project_iam_member" "crossplane_storage" { # read requests, so a condition here would be vacuous anyway (see gap 1 below). resource "google_project_iam_custom_role" "crossplane_role_reader" { project = var.project_id - role_id = "xplane_role_reader" + role_id = "xplane_role_reader${var.custom_role_suffix}" title = "Crossplane role reader" description = "Read-only on IAM role definitions, so ProjectIAMMember can reference the custom DNS role. Confers no grant capability." diff --git a/opentofu/gcp/gke/init/outputs.tf b/opentofu/gcp/gke/init/outputs.tf index 4ac08f2b4..19bd400c1 100644 --- a/opentofu/gcp/gke/init/outputs.tf +++ b/opentofu/gcp/gke/init/outputs.tf @@ -94,3 +94,8 @@ output "private_domain_name" { description = "Cloud DNS private zone domain, substituted into manifests as $${private_domain_name}" value = local.net.private_domain_name } + +output "dns_editor_role" { + description = "Full name of the custom DNS role, suffix included, for GCPWorkloadIdentity claims" + value = google_project_iam_custom_role.crossplane_dns.name +} diff --git a/opentofu/gcp/gke/init/variables.tf b/opentofu/gcp/gke/init/variables.tf index cec5e3d74..455a1fc81 100644 --- a/opentofu/gcp/gke/init/variables.tf +++ b/opentofu/gcp/gke/init/variables.tf @@ -152,3 +152,15 @@ variable "tailscale_oauth_secret_name" { type = string default = "tailscale-k8s-operator-oauth" } + +# GCP reserves a deleted custom role's ID for 37 days and allows undelete for 7 +# only, so a rebuild inside that window cannot recreate it (09-11 bug 2). The +# destroy now keeps the roles (not inert: Crossplane's external-dns +# ProjectIAMMember outlives the cluster and keeps its grant live until gcp-0 +# returns) and the deploy adopts them; this suffix moved past the IDs already +# burned (unsuffixed before 2026-09-11, `_v2` on 2026-09-14). No default: a run +# without variables.tfvars must fail, not plan the live roles' replacement. +variable "custom_role_suffix" { + description = "Generation suffix on the three custom role IDs; bump only if a generation is lost" + type = string +} diff --git a/opentofu/gcp/gke/init/variables.tfvars b/opentofu/gcp/gke/init/variables.tfvars index 77d7ef0f0..be35e8e18 100644 --- a/opentofu/gcp/gke/init/variables.tfvars +++ b/opentofu/gcp/gke/init/variables.tfvars @@ -5,6 +5,8 @@ region = "europe-west4" cluster_name = "gcp-0" release_channel = "REGULAR" +custom_role_suffix = "_v3" + # Slice 4 must pin the SAME image type on every ComputeClass. node_image_type = "COS_CONTAINERD" node_machine_type = "e2-standard-4" diff --git a/opentofu/gcp/gke/init/workflows.tm.hcl b/opentofu/gcp/gke/init/workflows.tm.hcl index e59767fb8..11b352b2d 100644 --- a/opentofu/gcp/gke/init/workflows.tm.hcl +++ b/opentofu/gcp/gke/init/workflows.tm.hcl @@ -90,6 +90,9 @@ script "deploy" { set -euo pipefail ${global.provisioner} init ${global.provisioner} validate + bash "${terramate.root.path.fs.absolute}/scripts/ops/gcp/adopt-custom-roles.sh" \ + --project ogenki-435905 \ + --suffix "$(awk -F'"' '/^custom_role_suffix/{print $2}' variables.tfvars)" --apply trivy config --exit-code=1 --ignorefile=./.trivyignore.yaml . ${global.provisioner} apply -auto-approve -var-file=variables.tfvars BASH @@ -105,13 +108,28 @@ script "deploy" { ${global.cloud_gate} set -euo pipefail cd ../configure + # ../configure's vault provider needs the CA chain before init, and a + # fresh checkout has no .tls/ (gitignored). gke/configure's own deploy + # writes it; this inline apply bypasses that script (09-11 bug 3). + bash "${terramate.root.path.fs.absolute}/scripts/provision/openbao-config.sh" ca \ + --cloud gcp --project ogenki-435905 \ + --root-ca-secret-name openbao-priv-gcp-ca-chain --ca-output-file .tls/ca.pem ${global.provisioner} init -lock-timeout=5m - ${global.provisioner} apply -auto-approve -var-file=variables.tfvars -var='cilium_version=${global.cilium_version}' -var='gateway_api_version=${global.gateway_api_version}' -var='flux_operator_version=${global.flux_operator_version}' -var='flux_instance_version=${global.flux_instance_version}' $${TF_VAR_flux_git_ref:+-var="flux_git_ref=$${TF_VAR_flux_git_ref}"} + # A restored lineage already holds jwt/gcp-0, and creating it again 400s + # with "path is already in use". Same call as gke/configure's deploy. + bash "${terramate.root.path.fs.absolute}/scripts/provision/openbao-adopt-jwt-mount.sh" \ + --cluster-name gcp-0 --url https://bao.priv.gcp.ogenki.io:8200 \ + --root-token-secret-name openbao-priv-gcp-root-token \ + --ca-file .tls/ca.pem --cloud gcp --project ogenki-435905 \ + -- -var='cilium_version=${global.cilium_version}' -var='gateway_api_version=${global.gateway_api_version}' -var='flux_operator_version=${global.flux_operator_version}' -var='flux_instance_version=${global.flux_instance_version}' -var='deploy_identity_provider=${global.deploy_identity_provider_gcp}' $${TF_VAR_flux_git_ref:+-var="flux_git_ref=$${TF_VAR_flux_git_ref}"} + # deploy_identity_provider here too: without it this apply publishes the + # consumed (AWS) identity_provider_url until the standalone configure run. + ${global.provisioner} apply -auto-approve -var-file=variables.tfvars -var='cilium_version=${global.cilium_version}' -var='gateway_api_version=${global.gateway_api_version}' -var='flux_operator_version=${global.flux_operator_version}' -var='flux_instance_version=${global.flux_instance_version}' -var='deploy_identity_provider=${global.deploy_identity_provider_gcp}' $${TF_VAR_flux_git_ref:+-var="flux_git_ref=$${TF_VAR_flux_git_ref}"} # Forget flux-operator here, in the job that just created it -- NOT only # in gke/configure's own `deploy`, which this job bypasses. Left in state, # the standalone gke/configure stack plans count=0 against a resource that - # IS in state, and that is a destroy: a real `helm uninstall`. The AWS lane - # had the identical gap and hit it on 2026-09-16. + # IS in state, and that is a destroy: a real `helm uninstall`. The AWS + # lane had the identical gap and hit it on 2026-09-16. ${global.provisioner} state rm helm_release.flux_operator 2>/dev/null || true BASH ], @@ -312,6 +330,9 @@ script "deploy-stage1" { set -euo pipefail ${global.provisioner} init ${global.provisioner} validate + bash "${terramate.root.path.fs.absolute}/scripts/ops/gcp/adopt-custom-roles.sh" \ + --project ogenki-435905 \ + --suffix "$(awk -F'"' '/^custom_role_suffix/{print $2}' variables.tfvars)" --apply trivy config --exit-code=1 --ignorefile=./.trivyignore.yaml . ${global.provisioner} apply -auto-approve -var-file=variables.tfvars BASH @@ -332,6 +353,7 @@ script "preview" { ${global.provisioner} init ${global.provisioner} validate trivy config --exit-code=1 --ignorefile=./.trivyignore.yaml . + # After a teardown the three custom roles plan as creates; deploy imports them (adopt-custom-roles.sh). ${global.provisioner} plan -out=out.tfplan -var-file=variables.tfvars BASH ], @@ -371,6 +393,21 @@ script "destroy" { # must fail here, not after resources have started disappearing. Same stack # dir as stage1-destroy-cluster, so that job inherits this init. ${global.provisioner} init -lock-timeout=5m + # Keep the custom roles (GCP parity GP-15): a deleted role ID stays + # reserved for 37 days, so the next rebuild could not recreate it. The + # next deploy adopts them (adopt-custom-roles.sh). Kept roles are not + # inert: Crossplane's ProjectIAMMember for ns/kube-system/sa/external-dns + # outlives the cluster, so that grant stays live until gcp-0 returns. + # Never `|| true` here: a state rm lost to a lock, backend or auth error + # would let the cluster destroy delete the roles and burn their IDs. + in_state="$(${global.provisioner} state list)" + keep=() + for addr in google_project_iam_custom_role.crossplane_dns google_project_iam_custom_role.crossplane_storage google_project_iam_custom_role.crossplane_role_reader; do + if grep -qxF "$addr" <<<"$in_state"; then keep+=("$addr"); fi + done + if [ "$${#keep[@]}" -gt 0 ]; then + ${global.provisioner} state rm -lock-timeout=5m "$${keep[@]}" + fi BASH ], ] @@ -570,6 +607,7 @@ script "drift" "reconcile" { ["bash", "-c", <<-BASH ${global.cloud_gate} set -euo pipefail + # After a teardown the three custom roles plan as creates that fail here; run deploy, which imports them (adopt-custom-roles.sh). ${global.provisioner} apply -input=false -auto-approve -lock-timeout=5m -var-file=variables.tfvars drift.tfplan BASH ], diff --git a/scripts/ci/flux-schema/render-bundle.py b/scripts/ci/flux-schema/render-bundle.py index 6ba6b4756..e9e79ffda 100755 --- a/scripts/ci/flux-schema/render-bundle.py +++ b/scripts/ci/flux-schema/render-bundle.py @@ -140,6 +140,8 @@ def _repo_lock(url): # is what lets CI catch a consumer that assumes a string. "project_number": "323586397743", "workload_pool": "ogenki-435905.svc.id.goog", + # gcp-0 only: the custom DNS role's full name, generation suffix included. + "gcp_dns_editor_role": "projects/ogenki-435905/roles/xplane_dns_editor_v3", "zone": "europe-west4-a", "network_name": "vpc-foobar", # Federated Route53 path (workstream 12). GCP-only: aws-0 authenticates to diff --git a/scripts/ci/tests/test-adopt-custom-roles.sh b/scripts/ci/tests/test-adopt-custom-roles.sh new file mode 100755 index 000000000..a49984f8b --- /dev/null +++ b/scripts/ci/tests/test-adopt-custom-roles.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# +# adopt-custom-roles.sh (GCP parity GP-15) with gcloud and tofu stubbed on PATH: +# a live role missing from state is imported, one in state is left alone, an +# absent one is left to the apply, and a soft-deleted one is reported, never imported. +# A describe that fails for any reason other than NOT_FOUND (expired credentials) +# must stop the deploy, not be read as "absent". +set -uo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +T="$(mktemp -d)"; trap 'rm -rf "$T"' EXIT +mkdir -p "$T/bin" +cat >"$T/bin/gcloud" <<'EOF' +#!/usr/bin/env bash +case "$*" in + *"auth application-default print-access-token"*) echo fake-token ;; + *"roles describe xplane_dns_editor_v3 "*) + if [ -n "${STUB_AUTH_FAIL:-}" ]; then + echo "ERROR: (gcloud.iam.roles.describe) There was a problem refreshing your current auth tokens: invalid_rapt" >&2 + exit 1 + fi + echo "" ;; + *"roles describe xplane_storage_admin_v3 "*) echo "True" ;; + *"roles describe xplane_role_reader_v3 "*) + echo "ERROR: (gcloud.iam.roles.describe) NOT_FOUND: The role named projects/ogenki-435905/roles/xplane_role_reader_v3 was not found." >&2 + exit 1 ;; + *) exit 1 ;; +esac +EOF +cat >"$T/bin/tofu" <>"$T/imports" ;; +esac +EOF +chmod +x "$T/bin/gcloud" "$T/bin/tofu" +adopt() { PATH="$T/bin:$PATH" bash "$ROOT/scripts/ops/gcp/adopt-custom-roles.sh" --project ogenki-435905 --suffix _v3 --apply; } + +# out, rc and the auth_* values are read inside check's eval strings, where the +# linter cannot see them. +# shellcheck disable=SC2034 +out="$(adopt 2>&1)" +# shellcheck disable=SC2034 +rc=$? +# shellcheck disable=SC2034 +auth_out="$(STUB_AUTH_FAIL=1 adopt 2>"$T/auth_err")" +# shellcheck disable=SC2034 +auth_rc=$? +fails=0 +check() { if eval "$2"; then echo " ok $1"; else echo " FAIL $1"; fails=1; fi; } +check "exit 0" '[ "$rc" -eq 0 ]' +check "the live dns role is imported" 'grep -q "import -var-file=variables.tfvars google_project_iam_custom_role.crossplane_dns projects/ogenki-435905/roles/xplane_dns_editor_v3" "$T/imports"' +check "exactly one import" '[ "$(wc -l <"$T/imports")" -eq 1 ]' +check "the soft-deleted storage role is reported" 'grep -q "\[deleted\].*xplane_storage_admin_v3" <<<"$out"' +check "the absent reader role is left to the apply" 'grep -q "\[absent \].*xplane_role_reader_v3" <<<"$out"' +check "the absent line names the burned-ID fix" 'grep "\[absent \]" <<<"$out" | grep -q "bump custom_role_suffix"' +check "a non-NOT_FOUND describe failure exits non-zero" '[ "$auth_rc" -ne 0 ]' +check " ...naming the role and the credentials" 'grep -q "xplane_dns_editor_v3" "$T/auth_err" && grep -qi "credentials" "$T/auth_err"' +check " ...not reported as absent" '! grep -q "\[absent \].*xplane_dns_editor_v3" <<<"$auth_out$(cat "$T/auth_err")"' +check " ...importing nothing more" '[ "$(wc -l <"$T/imports")" -eq 1 ]' +check "no token is ever printed" '! grep -q fake-token <<<"$out$auth_out$(cat "$T/auth_err")"' +[ "$fails" -eq 0 ] && echo "all checks passed" +exit "$fails" diff --git a/scripts/ci/tests/test-gcp-bucket-grant-allowlist.sh b/scripts/ci/tests/test-gcp-bucket-grant-allowlist.sh new file mode 100755 index 000000000..b4c95a91c --- /dev/null +++ b/scripts/ci/tests/test-gcp-bucket-grant-allowlist.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# requires: python3 +# +# GCP parity GP-23 (09-11 bug 7; memory gcp_crossplane_grant_allowlist_contract): +# every role a GCPWorkloadIdentity grants under bucketRoles must be in gke/init's +# crossplane_bucket_grantable_roles, or the IAM condition denies the grant with a +# bare 403. openbao-snapshot's objectCreator was missing, so gcp-0 never took a +# scheduled snapshot. +set -uo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +python3 - "$ROOT" <<'PY' +import pathlib, re, sys +root = pathlib.Path(sys.argv[1]) +iam = (root / "opentofu/gcp/gke/init/iam.tf").read_text() +m = re.search(r"crossplane_bucket_grantable_roles\s*=\s*\[(.*?)\]", iam, re.S) +if not m: + print("FAIL no crossplane_bucket_grantable_roles in iam.tf"); sys.exit(1) +allowed = set(re.findall(r'"(roles/[^"]+)"', m.group(1))) +wanted = {} +for area in ("infrastructure", "security", "observability", "tooling", "apps"): + for f in (root / area).glob("gcp-0/**/*.yaml"): + text = f.read_text() + if "kind: GCPWorkloadIdentity" not in text or "bucketRoles:" not in text: + continue + for role in re.findall(r"^\s+role:\s*(roles/\S+)", text.split("bucketRoles:", 1)[1], re.M): + wanted.setdefault(role, []).append(str(f.relative_to(root))) +if not wanted: + print("FAIL found no bucketRoles grant at all: the scan is broken"); sys.exit(1) +missing = {r: fs for r, fs in wanted.items() if r not in allowed} +for r, fs in sorted(missing.items()): + print(f"FAIL {r} (asked by {', '.join(fs)}) is not in crossplane_bucket_grantable_roles") +if missing: + sys.exit(1) +print("PASS") +PY diff --git a/scripts/ci/tests/test-gcp-custom-role-refs.sh b/scripts/ci/tests/test-gcp-custom-role-refs.sh new file mode 100755 index 000000000..83bfdd5ad --- /dev/null +++ b/scripts/ci/tests/test-gcp-custom-role-refs.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# GCP parity GP-15: every custom role carries the generation suffix, and no +# manifest spells a custom role name -- they read ${gcp_dns_editor_role}, so a +# suffix bump never leaves a claim pointing at a deleted role. +set -uo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +fails=0 +while IFS= read -r line; do + case "$line" in + *'${var.custom_role_suffix}"'*) ;; + *) echo "FAIL iam.tf: $line"; fails=$((fails + 1)) ;; + esac +done < <(grep -E '^[[:space:]]*role_id[[:space:]]*=' "$ROOT/opentofu/gcp/gke/init/iam.tf") +[ "$(grep -cE '^[[:space:]]*role_id[[:space:]]*=' "$ROOT/opentofu/gcp/gke/init/iam.tf")" -eq 3 ] \ + || { echo "FAIL expected 3 custom roles in iam.tf"; fails=$((fails + 1)); } +hits="$(grep -rnE 'roles/xplane_' --include=*.yaml "$ROOT"/{clusters,infrastructure,security,observability,tooling,apps} 2>/dev/null)" +[ -z "$hits" ] || { echo "FAIL a manifest spells a custom role:"; echo "$hits"; fails=$((fails + 1)); } +grep -q 'gcp_dns_editor_role' "$ROOT/opentofu/gcp/gke/configure/kubernetes.tf" \ + || { echo "FAIL gke-gcp-0-vars has no gcp_dns_editor_role"; fails=$((fails + 1)); } +[ "$fails" -eq 0 ] || exit 1 +echo PASS diff --git a/scripts/ci/tests/test-gcp-external-dns-deps.py b/scripts/ci/tests/test-gcp-external-dns-deps.py new file mode 100644 index 000000000..c25ad52ea --- /dev/null +++ b/scripts/ci/tests/test-gcp-external-dns-deps.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# requires: python3 kustomize +"""GCP parity GP-27 (09-11 bug 6): gcp-0's private external-dns must not wait on +aws-load-balancer-controller, which gcp-0 never runs. With the edge, the release +never installs and no *.priv.gcp record is written, while the parent +Kustomization (no `wait`) still reports Ready.""" +import pathlib +import subprocess +import sys + +try: + import yaml +except ImportError: + print("PyYAML is not installed") + sys.exit(77) + +ROOT = pathlib.Path(__file__).resolve().parents[3] +out = subprocess.run(["kustomize", "build", str(ROOT / "infrastructure/gcp-0/external-dns"), + "--load-restrictor=LoadRestrictionsNone"], capture_output=True, text=True, check=True).stdout +hrs = [d for d in yaml.safe_load_all(out) if d and d.get("kind") == "HelmRelease" and d["metadata"]["name"] == "external-dns"] +if len(hrs) != 1: + print(f"FAIL expected one external-dns HelmRelease, found {len(hrs)}") + sys.exit(1) +deps = hrs[0]["spec"].get("dependsOn") or [] +if any(d.get("name") == "aws-load-balancer-controller" for d in deps): + print("FAIL gcp-0's external-dns depends on aws-load-balancer-controller, which gcp-0 never runs") + sys.exit(1) +print("PASS") diff --git a/scripts/ci/tests/test-gcp-gke-init-workflow.py b/scripts/ci/tests/test-gcp-gke-init-workflow.py new file mode 100644 index 000000000..37a712b14 --- /dev/null +++ b/scripts/ci/tests/test-gcp-gke-init-workflow.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# requires: python3 +"""gcp/gke/init's deploy jobs (GCP parity G-2/G-3). Each check is a bug a live +GCP deploy hit: 09-11 bug 3 (no CA, no jwt adopt in stage 2), the missing +deploy_identity_provider on the inline configure apply, the custom roles that a +teardown deleted (GP-15), and a hosting stage 3 that must configure the IdP +before its clients and mirror them into OpenBao (GP-5).""" +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[3] +TEXT = (ROOT / "opentofu/gcp/gke/init/workflows.tm.hcl").read_text() + + +def job_bodies(text, name): + return re.findall(r'name\s*=\s*"%s"(.*?)(?=\n job \{|\nscript "|\Z)' % re.escape(name), text, re.S) + + +def job_body(text, name): + return next(iter(job_bodies(text, name)), "") + + +fails = [] + + +def before(body, first, second, what): + i, j = body.find(first), body.find(second) + if i < 0 or j < 0 or i > j: + fails.append(what) + + +s2 = job_body(TEXT, "stage2-cilium-and-flux") +if not s2: + fails.append("no stage2-cilium-and-flux job") +before(s2, 'openbao-config.sh" ca', "init -lock-timeout", "stage 2 writes OpenBao's CA before init") +before(s2, "init -lock-timeout", "openbao-adopt-jwt-mount.sh", "stage 2 adopts jwt/gcp-0 after init") +before(s2, "openbao-adopt-jwt-mount.sh", "apply -auto-approve", "stage 2 adopts jwt/gcp-0 before its apply") +apply_line = next((l for l in s2.splitlines() if "apply -auto-approve" in l), "") +if "deploy_identity_provider=${global.deploy_identity_provider_gcp}" not in apply_line: + fails.append("stage 2's apply passes deploy_identity_provider") + +# Two stage1-cluster jobs: `deploy` and `deploy-stage1`. +s1_jobs = job_bodies(TEXT, "stage1-cluster") +if len(s1_jobs) != 2: + fails.append(f"expected 2 stage1-cluster jobs, found {len(s1_jobs)}") +for n, s1 in enumerate(s1_jobs, 1): + before(s1, "adopt-custom-roles.sh", "apply -auto-approve", + f"stage1-cluster job {n} adopts the kept custom roles before its apply") +confirm = job_body(TEXT, "confirm") +for addr in ("crossplane_dns", "crossplane_storage", "crossplane_role_reader"): + if f"google_project_iam_custom_role.{addr}" not in confirm or "state rm" not in confirm: + fails.append(f"the destroy keeps google_project_iam_custom_role.{addr} out of the teardown") +# A swallowed state rm (lock, backend, auth) lets the cluster destroy delete the +# roles and burn their IDs for 37 days. +state_lines = [l for l in confirm.splitlines() + if ("state rm" in l or "state list" in l) and not l.lstrip().startswith("#")] +if any("|| true" in l or "2>/dev/null" in l for l in state_lines): + fails.append("the destroy's custom-role state rm swallows its failures") +if not any("state rm -lock-timeout=" in l for l in state_lines): + fails.append("the destroy's custom-role state rm waits for the state lock") + +for f in fails: + print("FAIL", f) +if fails: + sys.exit(1) +print("PASS") diff --git a/scripts/ci/tests/test-teardown-gcp-sweep.sh b/scripts/ci/tests/test-teardown-gcp-sweep.sh new file mode 100755 index 000000000..678b290d2 --- /dev/null +++ b/scripts/ci/tests/test-teardown-gcp-sweep.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# memory gke_lb_orphans_block_vpc_delete: a deleted GKE cluster leaves a target +# pool and k8s-* firewall rules behind, and the firewall rules block the VPC +# delete. teardown.sh --verify-only must report both, and fail while they exist. +# The sweep lists LBs project-wide, so ANY live cluster must block it, and the +# destroy is retried only while the network still stands (GP-22). +# +# This runs the REAL teardown.sh with TM_DESTROY_CONFIRMED=true, kept offline +# only by PATH shadowing. If a stub failed to land or cannot execute, bash would +# resolve the real terramate and gcloud and, with live ADC, destroy gcp-0. So +# the setup fails closed and every invocation first proves the stubs are what +# runs (require_stubs). +set -uo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +set -e +# Absolute: teardown.sh cd's before calling terramate, and a relative PATH entry +# stops resolving there, letting the real CLI run with the destroy confirmed. +T="$(cd "$(mktemp -d)" && pwd -P)"; trap 'rm -rf "$T"' EXIT +mkdir -p "$T/bin" +cat >"$T/bin/gcloud" <>"$T/calls" +case "\$*" in + --stub-probe) echo stub ;; + "projects describe"*) exit 0 ;; + *"print-access-token"*) echo fake-token ;; + *"container clusters list"*) + [ -z "\${CLUSTERS_FAIL:-}" ] || exit 1 + # Honour a name filter, as gcloud does: a guard that filters on one name + # must not see the other clusters. + want="\$(printf '%s\n' "\$@" | sed -n 's/^--filter=name=//p')" + for c in \${CLUSTERS:-}; do + [ -z "\$want" ] || [ "\$c" = "\$want" ] || continue + printf '%s\teurope-west4-a\tRUNNING\n' "\$c" + done ;; + *"networks describe"*) [ -n "\${NETWORK_UP:-}" ] || exit 1 ;; + *" delete "*) echo "\$*" >>"$T/deletes" ;; + *"forwarding-rules list"*) [ -n "\${EMPTY:-}" ] || echo "a2c53478 europe-west4" ;; + *"target-pools list"*) [ -n "\${EMPTY:-}" ] || echo "a2c53478 europe-west4" ;; + *"firewall-rules list"*) [ -n "\${EMPTY:-}" ] || printf '%s\n' k8s-fw-a2c53478 k8s-a6e2efd9258d71a1-node-http-hc ;; + *) exit 0 ;; +esac +EOF +cat >"$T/bin/terramate" <>"$T/destroys" +EOF +# Never called with TM_CLOUD=gcp; stubbed so a regression cannot reach AWS either. +cat >"$T/bin/aws" <>"$T/aws-calls"; exit 1 +EOF +chmod +x "$T/bin/gcloud" "$T/bin/terramate" "$T/bin/aws" +set +e +export PATH="$T/bin:$PATH" + +require_stubs() { # abort the whole test unless every cloud CLI is the stub + local tool + case "$T" in /*) ;; *) echo "ABORT: stub dir '$T' is not absolute" >&2; exit 1 ;; esac + for tool in gcloud terramate aws; do + if [ "$(command -v "$tool")" != "$T/bin/$tool" ] \ + || [ "$("$T/bin/$tool" --stub-probe 2>/dev/null)" != stub ]; then + echo "ABORT: $tool resolves to '$(command -v "$tool")', not the stub in $T/bin;" \ + "refusing to run teardown.sh or the sweep against a real cloud" >&2 + exit 1 + fi + done +} +fails=0 +f() { echo "FAIL $*"; fails=1; } + +# The report (--verify-only). +require_stubs +out="$(TM_CLOUD=gcp bash "$ROOT/scripts/ops/teardown/teardown.sh" --verify-only 2>&1)"; rc=$? +grep -q 'Target pools' <<<"$out" || f "no Target pools row" +grep -q 'k8s-\* firewall rules' <<<"$out" || f "no k8s-* firewall rules row" +[ "$rc" -ne 0 ] || f "leftovers must fail the verify" + +# The sweep (GP-22): dry-run deletes nothing, --apply deletes the four, a live cluster blocks it. +S="$ROOT/scripts/ops/gcp/sweep-lb-orphans.sh" +args=(--project ogenki-435905 --network vpc-europe-west4-dev --cluster gcp-0) +[ -f "$S" ] || f "no $S" +: >"$T/deletes" +require_stubs +bash "$S" "${args[@]}" >/dev/null 2>&1 || f "the dry run failed" +[ ! -s "$T/deletes" ] || f "the dry run deleted something" +: >"$T/deletes"; : >"$T/calls" +require_stubs +bash "$S" "${args[@]}" --apply >/dev/null 2>&1 || f "the sweep failed" +[ "$(wc -l <"$T/deletes")" -eq 4 ] || f "expected 4 deletes (rule, pool, 2 firewalls), got: $(cat "$T/deletes")" +grep -q 'forwarding-rules delete a2c53478 --region europe-west4' "$T/deletes" || f "the forwarding rule was not deleted in its region" +# The filters are the whole scoping: without them the sweep deletes by project. +grep 'firewall-rules list' "$T/calls" | grep -q 'network~/vpc-europe-west4-dev\$' \ + || f "the firewall list is not scoped to the network" +for kind in forwarding-rules target-pools; do + grep "$kind list" "$T/calls" | grep -q 'description~kubernetes.io/service-name' \ + || f "the $kind list is not scoped to GKE's description marker" +done + +refuses() { # $1 = case, then env assignments + local what="$1" rc; shift + : >"$T/deletes" + require_stubs + env "$@" bash "$S" "${args[@]}" --apply >/dev/null 2>&1; rc=$? + [ "$rc" -eq 3 ] || f "$what must refuse with exit 3, got $rc" + [ ! -s "$T/deletes" ] || f "$what, yet LBs were deleted" +} +refuses "the named cluster alive" CLUSTERS=gcp-0 +# The rule and pool lists are project-wide, so another cluster's LBs are in them. +refuses "another cluster alive" CLUSTERS=gcp-1 +refuses "a failed cluster list" CLUSTERS_FAIL=1 + +# teardown (GP-22): the destroy is retried only while the network stands. +destroys() { # $1 = expected terramate runs, $2 = case, then env assignments + local want="$1" what="$2"; shift 2 + : >"$T/destroys" + require_stubs + env "$@" TM_CLOUD=gcp TM_DESTROY_CONFIRMED=true bash "$ROOT/scripts/ops/teardown/teardown.sh" >/dev/null 2>&1 + [ "$(wc -l <"$T/destroys")" -eq "$want" ] \ + || f "$what: expected $want destroy run(s), got $(wc -l <"$T/destroys")" +} +destroys 1 "a clean teardown with the network gone" EMPTY=1 +destroys 2 "a teardown with the network still standing" NETWORK_UP=1 + +[ "$fails" -eq 0 ] && echo PASS +exit "$fails" diff --git a/scripts/ops/gcp/adopt-custom-roles.sh b/scripts/ops/gcp/adopt-custom-roles.sh new file mode 100755 index 000000000..5da8ad27e --- /dev/null +++ b/scripts/ops/gcp/adopt-custom-roles.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# +# Adopt gke/init's three custom IAM roles when they already exist. +# +# WHY: GCP reserves a deleted custom role's ID for 37 days and can undelete it +# for 7 only, so a rebuild within 37 days of a teardown that deleted the roles +# fails on the first create. gke/init's destroy therefore drops them from state +# instead of deleting them, and every deploy adopts them here (GCP parity GP-15). +# +# Usage (from opentofu/gcp/gke/init, after `tofu init`): +# adopt-custom-roles.sh --project ID --suffix S [--apply] +# Dry-run unless --apply. +set -euo pipefail + +# shellcheck source=scripts/lib/gcloud-adc.sh +. "$(dirname "$0")/../../lib/gcloud-adc.sh" + +PROJECT="" SUFFIX="" APPLY=false +while [ $# -gt 0 ]; do + case "$1" in + --project) PROJECT="$2"; shift 2 ;; + --suffix) SUFFIX="$2"; shift 2 ;; + --apply) APPLY=true; shift ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done +[ -n "$PROJECT" ] || { echo "--project is required" >&2; exit 2; } + +ROLES=( + "google_project_iam_custom_role.crossplane_dns=xplane_dns_editor" + "google_project_iam_custom_role.crossplane_storage=xplane_storage_admin" + "google_project_iam_custom_role.crossplane_role_reader=xplane_role_reader" +) + +err="$(mktemp)" +trap 'rm -f "$err"' EXIT + +in_state="$(tofu state list 2>/dev/null || true)" +for entry in "${ROLES[@]}"; do + addr="${entry%%=*}" + id="${entry#*=}${SUFFIX}" + name="projects/${PROJECT}/roles/${id}" + if grep -qxF "$addr" <<<"$in_state"; then + echo "[ok ] ${addr} is in state" + continue + fi + if ! deleted="$(gcp_gcloud iam roles describe "$id" --project "$PROJECT" --format='value(deleted)' 2>"$err")"; then + # A role deleted more than 7 days ago is NOT_FOUND too, yet its ID stays + # reserved for 37 days, so the create can still fail. + if grep -q 'NOT_FOUND' "$err"; then + echo "[absent ] ${name}: the apply creates it; if that create fails because the ID was recently deleted, bump custom_role_suffix in variables.tfvars" + continue + fi + # Anything else (auth, invalid_rapt) must not pass for "absent": the apply + # would then try to create a role that exists. + echo "[error ] cannot describe ${name}; gcloud credentials may have expired (gcloud auth application-default login):" >&2 + grep '^ERROR:' "$err" >&2 || true + exit 1 + fi + if [ "$deleted" = "True" ]; then + # describe still answers only inside the 7-day undelete window; past it the + # role reads NOT_FOUND and takes the [absent ] branch above. + echo "[deleted] ${name} is soft-deleted: the apply's create undeletes it (GCP allows that for 7 days)" >&2 + continue + fi + if [ "$APPLY" = true ]; then + tofu import -var-file=variables.tfvars "$addr" "$name" + echo "[adopted] ${addr} <- ${name}" + else + echo "[dry-run] would import ${addr} <- ${name}" + fi +done diff --git a/scripts/ops/gcp/sweep-lb-orphans.sh b/scripts/ops/gcp/sweep-lb-orphans.sh new file mode 100755 index 000000000..4d4ecadbb --- /dev/null +++ b/scripts/ops/gcp/sweep-lb-orphans.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# +# Delete what a deleted GKE cluster leaves behind: every GKE LoadBalancer's +# forwarding rules and target pools IN THE PROJECT, and the k8s-* firewall rules +# on the network. No tofu state holds them, forwarding rules bill hourly, and the +# firewall rules block the VPC delete (memory gke_lb_orphans_block_vpc_delete; +# GCP parity GP-22). Dry-run unless --apply. --cluster only names the teardown +# in the output: the guard below is project-wide. +# +# Usage: sweep-lb-orphans.sh --project P --network N --cluster C [--apply] +set -euo pipefail + +# shellcheck source=scripts/lib/gcloud-adc.sh +. "$(dirname "$0")/../../lib/gcloud-adc.sh" + +PROJECT="" NETWORK="" CLUSTER="" APPLY=false +while [ $# -gt 0 ]; do + case "$1" in + --project) PROJECT="$2"; shift 2 ;; + --network) NETWORK="$2"; shift 2 ;; + --cluster) CLUSTER="$2"; shift 2 ;; + --apply) APPLY=true; shift ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done +{ [ -n "$PROJECT" ] && [ -n "$NETWORK" ] && [ -n "$CLUSTER" ]; } \ + || { echo "--project, --network and --cluster are required" >&2; exit 2; } + +# The rule and pool lists below are project-wide, so ANY cluster, in any state, +# may own what they return: deleting a live LoadBalancer's rule is an outage, +# not a sweep, and its service controller would recreate it anyway. +if ! clusters="$(gcp_gcloud container clusters list --project "$PROJECT" \ + --format='value(name,location,status)')"; then + echo "cannot list clusters in ${PROJECT}: refusing to sweep blind" >&2; exit 3 +fi +if [ -n "$clusters" ]; then + echo "GKE clusters still exist in ${PROJECT}; nothing is swept for ${CLUSTER} while any does:" >&2 + sed 's/^/ /' <<<"$clusters" >&2 + exit 3 +fi + +# GKE's service controller writes {"kubernetes.io/service-name": ...} into the +# description of every forwarding rule and target pool it creates; that, not a +# name pattern, is what makes an object GKE's. +rules="$(gcp_gcloud compute forwarding-rules list --project "$PROJECT" \ + --filter='description~kubernetes.io/service-name' --format='value(name,region.basename())')" +pools="$(gcp_gcloud compute target-pools list --project "$PROJECT" \ + --filter='description~kubernetes.io/service-name' --format='value(name,region.basename())')" +firewalls="$(gcp_gcloud compute firewall-rules list --project "$PROJECT" \ + --filter="name~^k8s- AND network~/${NETWORK}\$" --format='value(name)')" + +failed=0 +act() { # $1 = what, then the delete command + local what="$1"; shift + if [ "$APPLY" != true ]; then echo "[dry-run] would delete ${what}"; return 0; fi + if "$@" --quiet >/dev/null; then echo "[deleted] ${what}"; else echo "[FAILED ] ${what}" >&2; failed=1; fi +} +# Rules before pools (a rule targets a pool), firewalls last. +while read -r name region; do + [ -n "$name" ] || continue + scope=(--global); [ -n "$region" ] && scope=(--region "$region") + act "forwarding rule ${name}" gcp_gcloud compute forwarding-rules delete "$name" "${scope[@]}" --project "$PROJECT" +done <<<"$rules" +while read -r name region; do + [ -n "$name" ] || continue + act "target pool ${name}" gcp_gcloud compute target-pools delete "$name" --region "$region" --project "$PROJECT" +done <<<"$pools" +while read -r name; do + [ -n "$name" ] || continue + act "firewall rule ${name}" gcp_gcloud compute firewall-rules delete "$name" --project "$PROJECT" +done <<<"$firewalls" +exit "$failed" diff --git a/scripts/ops/tasks.yaml b/scripts/ops/tasks.yaml index 52161fe10..5cccd508c 100644 --- a/scripts/ops/tasks.yaml +++ b/scripts/ops/tasks.yaml @@ -28,12 +28,19 @@ tasks: desc: Make the workforce pool survive a teardown and rebuild dir: "{{.ROOT_DIR}}/opentofu/gcp/workforce-identity" cmds: ["{{.TASKFILE_DIR}}/gcp/adopt-workforce-pool.sh {{.CLI_ARGS}}"] + gcp:adopt-custom-roles: + desc: Import gke/init's kept custom IAM roles so a rebuild adopts them instead of recreating them + dir: "{{.ROOT_DIR}}/opentofu/gcp/gke/init" + cmds: ["{{.TASKFILE_DIR}}/gcp/adopt-custom-roles.sh {{.CLI_ARGS}}"] gcp:purge-dns-records: desc: Empty a Cloud DNS managed zone so the zone can be destroyed cmds: ["{{.TASKFILE_DIR}}/gcp/purge-dns-records.sh {{.CLI_ARGS}}"] gcp:sweep-orphaned-disks: desc: Delete Persistent Disks GKE's CSI driver created that nothing uses cmds: ["{{.TASKFILE_DIR}}/gcp/sweep-orphaned-disks.sh {{.CLI_ARGS}}"] + gcp:sweep-lb-orphans: + desc: Delete the forwarding rules, target pools and k8s-* firewall rules GKE left (refuses while any cluster exists) + cmds: ["{{.TASKFILE_DIR}}/gcp/sweep-lb-orphans.sh {{.CLI_ARGS}}"] k8s:cnpg-prepare-restore: desc: Clear a CNPG cluster's live WAL archive so a new cluster can start (off the normal path) cmds: ["{{.TASKFILE_DIR}}/k8s/cnpg-prepare-restore.sh {{.CLI_ARGS}}"] diff --git a/scripts/ops/teardown/teardown.sh b/scripts/ops/teardown/teardown.sh index a7ca04b68..ddf71b21f 100755 --- a/scripts/ops/teardown/teardown.sh +++ b/scripts/ops/teardown/teardown.sh @@ -49,6 +49,7 @@ wants() { # $1 = lane } destroy_rc=0 +retry=0 if [ "$VERIFY_ONLY" -eq 0 ]; then echo "=== destroying (TM_CLOUD=${CLOUDS}) ===" # --continue-on-error is the point: one stack whose target is already gone must @@ -93,15 +94,34 @@ if [ "$VERIFY_ONLY" -eq 0 ]; then --cluster-name "${EKS_CLUSTER_NAME:-aws-0}" --region "$_region" --apply || true bash "${ROOT}/scripts/ops/aws/sweep-orphaned-volumes.sh" \ --cluster-name "${EKS_CLUSTER_NAME:-aws-0}" --region "$_region" --apply || true + retry=1 + fi + fi - echo - echo "=== retrying the destroy after the sweep ===" - ( cd "${ROOT}/opentofu" && terramate script run --reverse --continue-on-error destroy ) - destroy_rc=$? - echo "=== terramate exit after retry: ${destroy_rc} ===" - echo + # GKE's LB leftovers (GCP parity GP-22, owner 2026-09-29): swept only on a + # confirmed teardown, and only once no GKE cluster is left in the project (the + # script refuses otherwise). Retried only while the network stands, like the + # AWS VPC check above: a clean teardown must not re-run the whole destroy. + if wants gcp && [ "${TM_DESTROY_CONFIRMED:-false}" = "true" ]; then + _project="${GCP_PROJECT:-ogenki-435905}" + _network="${GCP_NETWORK:-vpc-europe-west4-dev}" + echo "=== sweeping the LoadBalancer leftovers GKE left in ${_project} ===" + bash "${ROOT}/scripts/ops/gcp/sweep-lb-orphans.sh" --project "$_project" \ + --network "$_network" --cluster "${GKE_CLUSTER_NAME:-gcp-0}" --apply || true + if gcloud compute networks describe "$_network" --project "$_project" >/dev/null 2>&1; then + retry=1 fi fi + + # One retry for every lane, so TM_CLOUD=all never destroys three times. + if [ "$retry" -eq 1 ]; then + echo + echo "=== retrying the destroy after the sweep ===" + ( cd "${ROOT}/opentofu" && terramate script run --reverse --continue-on-error destroy ) + destroy_rc=$? + echo "=== terramate exit after retry: ${destroy_rc} ===" + echo + fi fi # --------------------------------------------------------------------------- @@ -159,6 +179,14 @@ if wants gcp; then --format='value(name)' 2>/dev/null)" report "Forwarding rules" "$(gcloud compute forwarding-rules list --project "$project" \ --format='value(name)' 2>/dev/null)" + # A deleted cluster leaves its LoadBalancers' target pools and k8s-* firewall + # rules behind, in no tofu state, and the firewall rules block the VPC delete + # (memory gke_lb_orphans_block_vpc_delete). The health-check rule is named + # after the node pool, not the LB, hence the prefix filter. + report "Target pools" "$(gcloud compute target-pools list --project "$project" \ + --format='value(name)' 2>/dev/null)" + report "k8s-* firewall rules" "$(gcloud compute firewall-rules list --project "$project" \ + --filter='name~^k8s-' --format='value(name)' 2>/dev/null)" report "Disks" "$(gcloud compute disks list --project "$project" \ --format='value(name)' 2>/dev/null)" fi @@ -175,6 +203,11 @@ else [ "$unverified" -gt 0 ] && \ echo " cloud: ${unverified} cloud(s) COULD NOT BE CHECKED — status unknown, assume not torn down" fi +# Not auto-skipped: if the first pass never reached openbao/cluster, skipping +# its pre-destroy snapshot would destroy a node without one. +if [ "$retry" -eq 1 ] && [ "$destroy_rc" -ne 0 ] && [ "$leftovers" -eq 0 ] && [ "$unverified" -eq 0 ]; then + echo " note: the non-zero exit came from the retry against already-destroyed stacks (e.g. openbao/cluster's snapshot of a gone node); re-run with TM_OPENBAO_SKIP_SNAPSHOT=true ONLY if the first pass's snapshot succeeded" +fi # Exit non-zero if the destroy failed, OR anything is still standing, OR a cloud # could not be checked. A green exit here means the provider itself reports diff --git a/website/content/docs/reference/commands.md b/website/content/docs/reference/commands.md index 12fedc9a0..efac71f6b 100644 --- a/website/content/docs/reference/commands.md +++ b/website/content/docs/reference/commands.md @@ -144,6 +144,12 @@ gcloud compute disks list --project gcloud compute addresses list --project ``` +`scripts/ops/teardown/teardown.sh` does both, sweeps GKE's LoadBalancer +leftovers, and retries once while the VPC stands. A non-zero terramate exit +beside `cloud: clean` comes from that retry failing on already-destroyed stacks +(typically `opentofu/gcp/openbao/cluster`'s snapshot of a gone node): re-run +with `TM_OPENBAO_SKIP_SNAPSHOT=true` only if the first pass's snapshot succeeded. + ## Opt-in stacks `opentofu/aws/llm-platform/` is tagged `opt-in` (see `opentofu/aws/llm-platform/workflows.tm.hcl`):