fix(openbao): Stage 2 on GCP's OpenBao

.github/workflows/ci.yaml

@@ -83,6 +83,11 @@ jobs:
           # What the pre-commit-terraform hooks call instead of `terraform`.
           PCT_TFPATH: tofu
 
+      - name: Test the OpenBao store-of-record module
+        run: |
+          tofu -chdir=opentofu/shared/modules/openbao-store-of-record init -backend=false -input=false
+          tofu -chdir=opentofu/shared/modules/openbao-store-of-record test
+
   security-scan:
     name: Security scanning 🔒
     runs-on: ubuntu-latest
@@ -322,6 +327,9 @@ jobs:
       - name: Check identity provider topology
         run: ./scripts/ci/validate-idp-topology.sh
 
+      - name: OpenBao policy parity
+        run: ./scripts/ci/validate-openbao-policies.sh
+
   # ---------------------------------------------------------------------------
   # Post-merge only. A red run on `main` is already visible next to the commit,
   # but only to someone looking at it -- and the whole point of the `push`

opentofu/gcp/openbao/management/auth.tf

@@ -5,5 +5,6 @@
 # pinned cert-manager role_id and the two Secret Manager entries, is gone: a
 # JWT login mints nothing long-lived.
 #
-# No human auth method on GCP yet; operators use the root token from
-# openbao-priv-gcp-root-token, as documented.
+# Human logins -- the userpass break-glass and the ZITADEL OIDC method -- come
+# from the shared module (store-of-record.tf). The break-glass password is
+# published to openbao-priv-gcp-admin-credentials.

opentofu/gcp/openbao/management/outputs.tf

@@ -7,3 +7,8 @@ output "cert_manager_role_name" {
   description = "PKI role name cert-manager's ClusterIssuer must reference"
   value       = vault_pki_secret_backend_role.cert_manager.name
 }
+
+output "admin_credentials_secret_name" {
+  description = "GCP Secret Manager entry holding the break-glass userpass credentials"
+  value       = google_secret_manager_secret.admin_credentials.secret_id
+}

opentofu/gcp/openbao/management/policies.tf

@@ -1,6 +1,7 @@
-# Two policies now. The AWS stack also carries admin, pki-admin and app
-# policies; none of those has a consumer on GCP, and the design scopes this
-# stack to PKI only. Add them when something needs them, not before.
+# The cert-manager and snapshot policies below are GCP's own. The Stage 2
+# policies -- admin, pki-admin, secrets-admin, external-secrets and the per-app
+# ones -- come from the shared module in store-of-record.tf, the same set AWS
+# defines inline.
 #
 # `templatefile` rather than `file` so the granted path is derived from the mount
 # and role that actually exist, instead of being restated as a literal that

opentofu/gcp/openbao/management/store-of-record.tf

@@ -0,0 +1,80 @@
+# Stage 2 on GCP (salvaged from ac62abf2): the same mounts, policies and logins
+# AWS defines, from the shared module. Until this existed, jwt/gcp-0's
+# external-secrets role named a policy
+# no GCP OpenBao ever had, and the 14 OpenBao-backed ExternalSecrets on gcp-0
+# authenticated and then read nothing.
+
+# OIDC is gated on the `openbao-oidc` entry EXISTING, not on the variable
+# being set. Reading a version of a missing secret is a hard error; listing
+# first turns "ZITADEL not bootstrapped yet" into "OIDC off", so a first deploy
+# converges and the second apply after zitadel-oidc-clients.sh picks it up --
+# the same two-pass shape as opentofu/aws/openbao/management/oidc.tf.
+#
+# The gate checks that the SECRET exists, not that it has a VERSION -- the
+# provider has no version-listing data source. zitadel-oidc-clients.sh writes on
+# GCP as create + versions add, so a failed add leaves a version-less secret and
+# this plan then fails on "latest" not found. Recovery: add the version (re-run
+# zitadel-oidc-clients.sh sync --apply) or delete the empty secret
+# (gcloud secrets delete openbao-oidc --project <id>), then re-apply.
+data "google_secret_manager_secrets" "openbao_oidc" {
+  count   = var.openbao_oidc_secret_id == "" ? 0 : 1
+  project = var.project_id
+  # Not an exact match in Secret Manager's list filter; contains() below is.
+  filter = "name:${var.openbao_oidc_secret_id}"
+}
+
+locals {
+  oidc_secret_present = var.openbao_oidc_secret_id != "" && contains(
+    [for s in try(data.google_secret_manager_secrets.openbao_oidc[0].secrets, []) : s.secret_id],
+    var.openbao_oidc_secret_id
+  )
+}
+
+data "google_secret_manager_secret_version" "openbao_oidc" {
+  count   = local.oidc_secret_present ? 1 : 0
+  secret  = var.openbao_oidc_secret_id
+  project = var.project_id
+}
+
+locals {
+  oidc_raw           = try(jsondecode(data.google_secret_manager_secret_version.openbao_oidc[0].secret_data), {})
+  oidc_client_id     = try(local.oidc_raw["client_id"], "")
+  oidc_client_secret = try(local.oidc_raw["client_secret"], "")
+  # Falls back to the `endpoint` the registration script stores alongside the
+  # credentials, so the IdP hostname is not configured twice.
+  oidc_issuer = var.openbao_oidc_issuer != "" ? var.openbao_oidc_issuer : try(local.oidc_raw["endpoint"], "")
+}
+
+module "store_of_record" {
+  source = "../../../shared/modules/openbao-store-of-record"
+
+  pki_mount_path     = vault_mount.pki.path
+  openbao_address    = local.openbao_address
+  admin_username     = var.admin_username
+  admin_group_alias  = "admin"
+  secret_owning_apps = var.secret_owning_apps
+  oidc_client_id     = local.oidc_client_id
+  oidc_client_secret = local.oidc_client_secret
+  oidc_issuer        = local.oidc_issuer
+}
+
+# Published so an operator can retrieve the break-glass password -- the GCP
+# twin of opentofu/aws/openbao/management/secrets.tf. Google-managed key,
+# matching the AVD-AWS-0098 decision recorded in .trivyignore.yaml.
+resource "google_secret_manager_secret" "admin_credentials" {
+  project   = var.project_id
+  secret_id = var.admin_credentials_secret_name
+
+  replication {
+    auto {}
+  }
+}
+
+resource "google_secret_manager_secret_version" "admin_credentials" {
+  secret = google_secret_manager_secret.admin_credentials.id
+  secret_data = jsonencode({
+    username = var.admin_username
+    password = module.store_of_record.admin_password
+    address  = local.openbao_address
+  })
+}

opentofu/gcp/openbao/management/variables.tf

@@ -122,3 +122,33 @@ variable "external_secrets_service_account" {
   type        = string
   default     = "external-secrets"
 }
+
+variable "admin_username" {
+  description = "Username of the userpass break-glass login, carrying admin, pki-admin and secrets-admin"
+  type        = string
+  default     = "admin"
+}
+
+variable "admin_credentials_secret_name" {
+  description = "GCP Secret Manager entry this stack publishes the break-glass credentials to, as {username, password, address}"
+  type        = string
+  default     = "openbao-priv-gcp-admin-credentials"
+}
+
+variable "openbao_oidc_secret_id" {
+  description = "GCP Secret Manager entry holding {client_id, client_secret, endpoint} for OpenBao's ZITADEL OIDC client, written by scripts/provision/zitadel-oidc-clients.sh. An entry that does not exist yet disables OIDC rather than failing."
+  type        = string
+  default     = "openbao-oidc"
+}
+
+variable "openbao_oidc_issuer" {
+  description = "ZITADEL issuer URL. Defaults to the `endpoint` field of openbao_oidc_secret_id."
+  type        = string
+  default     = ""
+}
+
+variable "secret_owning_apps" {
+  description = "Apps that own a prefix under apps/ (ADR-0036). Each gets a policy, an external identity group and an alias once OIDC is configured."
+  type        = set(string)
+  default     = []
+}

opentofu/gcp/openbao/management/variables.tfvars

@@ -2,3 +2,7 @@
 # project's real values (europe-west4, priv.gcp.ogenki.io, the Secret Manager
 # entry names the ceremony and init script create).
 project_id = "ogenki-435905"
+
+# The same apps as AWS (opentofu/aws/openbao/management/variables.tfvars), so a
+# persona behaves identically on either cloud's OpenBao.
+secret_owning_apps = ["app-wizard", "image-gallery"]

opentofu/gcp/openbao/management/versions.tf

@@ -11,5 +11,9 @@ terraform {
       source  = "hashicorp/vault"
       version = "~> 5.4"
     }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.6"
+    }
   }
 }

opentofu/gcp/openbao/management/workflows.tm.hcl

@@ -219,11 +219,16 @@ script "destroy" {
         # gated on TM_LINEAGE_DESTROY=true, so this path is only ever taken when
         # an operator has deliberately asked to tear the lineage down.
         #
-        # Only `vault_*` is dropped from state. The google_secret_manager_* and
-        # random_password resources here are real and do not match the prefix,
-        # so tofu still deletes them.
+        # Two prefixes are dropped from state: this stack's own `vault_*`
+        # resources, and the shared module's `module.store_of_record.vault_*`
+        # ones (matching is on the START of the state address, so `vault_`
+        # alone does not match a `module.` address -- see the script's header).
+        # The module's `random_password` and this stack's `google_secret_manager_*`
+        # resources are real and do not match either prefix, so tofu still
+        # deletes them.
         bash "${terramate.root.path.fs.absolute}/scripts/ops/teardown/tofu-destroy-contained.sh" \
-          --contained-prefix vault_ -- \
+          --contained-prefix vault_ \
+          --contained-prefix module.store_of_record.vault_ -- \
           -auto-approve -parallelism=1 -var-file=variables.tfvars
       BASH
       ],

opentofu/shared/modules/openbao-store-of-record/apps.tf

@@ -0,0 +1,32 @@
+# Per-app ownership of apps/ (ADR-0036). Gated on OIDC: without the OIDC mount
+# there is no accessor to alias against.
+locals {
+  secret_owning_apps = local.oidc_on == 1 ? var.secret_owning_apps : toset([])
+}
+
+resource "vault_policy" "app_prefix" {
+  for_each = local.secret_owning_apps
+
+  name = "app-${each.value}"
+  policy = templatefile("${path.module}/policies/app-prefix.hcl", {
+    mount = vault_mount.apps.path
+    app   = each.value
+  })
+}
+
+resource "vault_identity_group" "app" {
+  for_each = local.secret_owning_apps
+
+  name     = "openbao-app-${each.value}"
+  type     = "external"
+  policies = [vault_policy.app_prefix[each.value].name]
+}
+
+resource "vault_identity_group_alias" "app" {
+  for_each = local.secret_owning_apps
+
+  # Must equal the ZITADEL project role name as it lands in the groups claim.
+  name           = "app-${each.value}"
+  mount_accessor = vault_jwt_auth_backend.oidc[0].accessor
+  canonical_id   = vault_identity_group.app[each.value].id
+}

opentofu/shared/modules/openbao-store-of-record/auth.tf

@@ -0,0 +1,37 @@
+# The break-glass login. It stays alongside OIDC on purpose (ADR-0034): ZITADEL's
+# own credential lives in platform/zitadel/envvars, so an OIDC-only login would
+# have no way back in when ZITADEL is down. It MUST carry secrets-admin, or it
+# authenticates and then reads nothing it exists to recover.
+#
+# Deliberately no token_bound_cidrs: the only route to this API is the private
+# load balancer, so the network is already constrained, and a CIDR bind on the
+# break-glass credential buys nothing against the risk of locking yourself out.
+resource "vault_auth_backend" "userpass" {
+  type = "userpass"
+  path = "userpass"
+}
+
+# Lands in state, as the root token already does. The caller publishes it to
+# its own cloud's secret store from the `admin_password` output. The write-only
+# `password_wo` would keep it out of state, but it needs write-only attribute
+# support, so it is deliberately not used here.
+resource "random_password" "admin" {
+  length           = 32
+  special          = true
+  override_special = "!#%*-_=+"
+}
+
+resource "vault_generic_endpoint" "admin_user" {
+  path = "auth/${vault_auth_backend.userpass.path}/users/${var.admin_username}"
+  # The password is never readable back, so a read would always look like drift.
+  disable_read         = true
+  disable_delete       = false
+  ignore_absent_fields = true
+
+  data_json = jsonencode({
+    policies      = [vault_policy.admin.name, vault_policy.pki_admin.name, vault_policy.secrets_admin.name]
+    password      = random_password.admin.result
+    token_ttl     = 3600
+    token_max_ttl = 28800
+  })
+}

opentofu/shared/modules/openbao-store-of-record/mounts.tf

@@ -0,0 +1,19 @@
+# The Stage 2 store of record (ADR-0033). Root namespace, deliberately: a policy
+# binds only within the namespace it is created in, and the oidc/ mount, the
+# identity groups and every policy live in root.
+#
+# Grammar: platform/<component>/<name>, one-to-one onto ADR-0023's dash names.
+resource "vault_mount" "platform" {
+  path        = "platform"
+  type        = "kv-v2"
+  description = "Platform component secrets; store of record (ADR-0033 Stage 2)"
+}
+
+# Separate from platform/ because External Secrets' vault provider takes ONE
+# mount per store, so the split is what lets the two audiences carry different
+# policies. Grammar: apps/<app>/<key>.
+resource "vault_mount" "apps" {
+  path        = "apps"
+  type        = "kv-v2"
+  description = "Application secrets, owned per app (ADR-0036)"
+}

opentofu/shared/modules/openbao-store-of-record/oidc.tf

@@ -0,0 +1,100 @@
+# Human login through ZITADEL, authorised by project roles (ADR-0034). Everything
+# is count-gated on the caller supplying a client id and an issuer, so a cluster
+# whose ZITADEL is not bootstrapped yet applies cleanly with no OIDC method.
+locals {
+  oidc_enabled = var.oidc_client_id != "" && var.oidc_issuer != "" ? 1 : 0
+
+  # The gate is derived from the caller's OIDC secret payload, so it may arrive
+  # marked sensitive -- and a sensitive value cannot be a for_each argument.
+  # The keys come from a plain list; only the gate needs unwrapping. try()
+  # covers the non-sensitive case, where nonsensitive() would error.
+  oidc_on = try(nonsensitive(local.oidc_enabled), local.oidc_enabled)
+
+  # BOTH callbacks are required, and must match the `openbao` entry in
+  # scripts/provision/zitadel-oidc-clients.sh. The UI path embeds the mount path twice,
+  # which is why `path` below is pinned to "oidc".
+  oidc_redirect_uris = [
+    "${var.openbao_address}/ui/vault/auth/oidc/oidc/callback",
+    "http://localhost:8250/oidc/callback",
+  ]
+}
+
+resource "vault_jwt_auth_backend" "oidc" {
+  count = local.oidc_enabled
+
+  path               = "oidc"
+  type               = "oidc"
+  description        = "ZITADEL OIDC for human operators (ADR-0034)"
+  oidc_discovery_url = var.oidc_issuer
+  oidc_client_id     = var.oidc_client_id
+  oidc_client_secret = var.oidc_client_secret
+
+  # A LITERAL, not a reference to the role below: referencing it would order the
+  # role before the mount it lives in, and a fresh apply fails with
+  # `no handler for route "auth/oidc/role/default"`.
+  default_role = "default"
+
+  # checkov:skip=CKV_SECRET_6:False positive on `token_type = "default-service"` -- an OpenBao token-type constant, not a base64 secret.
+  tune {
+    listing_visibility = "unauth"
+    default_lease_ttl  = "1h"
+    max_lease_ttl      = "8h"
+    token_type         = "default-service"
+  }
+
+  lifecycle {
+    # Same as opentofu/aws/openbao/management/oidc.tf (#2078): OpenBao never
+    # returns the secret on read, and every fresh ZITADEL issues a new client id,
+    # so an apply here would replay discovery while the IdP is still down.
+    # zitadel-oidc-clients.sh's reconcile_openbao_oidc rotates both fields.
+    #
+    # Caveat: ANY other in-place update to this resource re-sends the whole
+    # config, oidc_client_secret taken from state (the creation-time value).
+    # After one, re-run the sync with --openbao-*. The real fix,
+    # oidc_client_secret_wo, is a follow-up.
+    ignore_changes = [oidc_client_id, oidc_client_secret]
+  }
+}
+
+resource "vault_jwt_auth_backend_role" "oidc_default" {
+  count = local.oidc_enabled
+
+  backend   = vault_jwt_auth_backend.oidc[0].path
+  role_name = "default"
+  role_type = "oidc"
+
+  allowed_redirect_uris = local.oidc_redirect_uris
+  user_claim            = "email"
+  # Without these ZITADEL issues a token with neither `email` nor `groups`, and
+  # the login dies on `claim "email" not found in token`.
+  oidc_scopes     = ["profile", "email", "groups"]
+  groups_claim    = "groups"
+  bound_audiences = [var.oidc_client_id]
+
+  # Authorisation comes from the external group below, never from the role.
+  token_policies = []
+  token_ttl      = 3600
+  token_max_ttl  = 28800
+
+  lifecycle {
+    # Tracks the client id, so it rotates with it (#2078).
+    ignore_changes = [bound_audiences]
+  }
+}
+
+resource "vault_identity_group" "oidc_admin" {
+  count = local.oidc_enabled
+
+  name     = "openbao-admin"
+  type     = "external"
+  policies = [vault_policy.admin.name, vault_policy.pki_admin.name, vault_policy.secrets_admin.name]
+}
+
+resource "vault_identity_group_alias" "oidc_admin" {
+  count = local.oidc_enabled
+
+  # Must equal the value in the token's `groups` array, exactly.
+  name           = var.admin_group_alias
+  mount_accessor = vault_jwt_auth_backend.oidc[0].accessor
+  canonical_id   = vault_identity_group.oidc_admin[0].id
+}