From d5b80c17cee74a09c31d9e34859f0dc22e0f4e5c Mon Sep 17 00:00:00 2001 From: Smana Date: Tue, 29 Sep 2026 09:24:05 +0200 Subject: [PATCH 1/8] feat(ci): a JWT role may only name a policy its OpenBao defines --- .../tests/test-validate-openbao-policies.sh | 121 ++++++++++++++++++ scripts/ci/validate-openbao-policies.sh | 96 ++++++++++++++ 2 files changed, 217 insertions(+) create mode 100755 scripts/ci/tests/test-validate-openbao-policies.sh create mode 100755 scripts/ci/validate-openbao-policies.sh diff --git a/scripts/ci/tests/test-validate-openbao-policies.sh b/scripts/ci/tests/test-validate-openbao-policies.sh new file mode 100755 index 000000000..f618ec262 --- /dev/null +++ b/scripts/ci/tests/test-validate-openbao-policies.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# Fixture-driven tests for validate-openbao-policies.sh. +# +# The state it exists to catch -- a JWT role naming a policy its OpenBao never +# defines -- shipped silently when Stage 2 landed (gcp-0's external-secrets +# role). Reproducing that live costs a GCP deploy, so it is tested against +# synthetic trees instead. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VALIDATOR="${SCRIPT_DIR}/../validate-openbao-policies.sh" +failures=0 + +# role_policies -- a configure stack naming policies +role_policies() { + mkdir -p "$1/opentofu/$2/k8s/configure" + cat >"$1/opentofu/$2/k8s/configure/openbao.tf" < -- a vault_policy with a literal name +define_policy() { + mkdir -p "$1" + cat >>"$1/policies.tf" < [substring] +expect() { + local label="$1" want="$2" root="$3" needle="${4:-}" out rc=0 + out="$("$VALIDATOR" "$root" 2>&1)" || rc=$? + if [ "$rc" != "$want" ]; then + echo "FAIL ${label}: exit ${rc}, expected ${want}"; echo "$out" | sed 's/^/ /' + failures=$((failures + 1)); return + fi + if [ -n "$needle" ] && ! grep -qF -- "$needle" <<<"$out"; then + echo "FAIL ${label}: output lacks: ${needle}"; echo "$out" | sed 's/^/ /' + failures=$((failures + 1)); return + fi + echo "ok ${label}" +} + +t=$(mktemp -d); trap 'rm -rf "$t"' EXIT + +r="$t/defined"; role_policies "$r" aws '"default", "external-secrets"' +define_policy "$r/opentofu/aws/openbao/management" external-secrets +expect "a defined policy passes" 0 "$r" + +r="$t/missing"; role_policies "$r" gcp '"default", "external-secrets"' +define_policy "$r/opentofu/gcp/openbao/management" cert-manager +expect "the Stage 2 gap fails, naming cloud and policy" 1 "$r" 'gcp: a JWT role names policy "external-secrets"' + +r="$t/module"; role_policies "$r" gcp '"external-secrets"' +mkdir -p "$r/opentofu/gcp/openbao/management" +cat >"$r/opentofu/gcp/openbao/management/store.tf" <<'EOF' +module "store" { + source = "../../../shared/modules/store" +} +EOF +define_policy "$r/opentofu/shared/modules/store" external-secrets +expect "a policy defined through a called module passes" 0 "$r" + +r="$t/default"; role_policies "$r" aws '"default"' +mkdir -p "$r/opentofu/aws/openbao/management" +expect "the built-in default policy is never required" 0 "$r" + +r="$t/templated"; role_policies "$r" aws '"app-x"' +mkdir -p "$r/opentofu/aws/openbao/management" +cat >"$r/opentofu/aws/openbao/management/apps.tf" <<'EOF' +resource "vault_policy" "app" { + name = "app-${each.value}" + policy = "{}" +} +EOF +expect "a templated name does not count as defining a literal one" 1 "$r" 'policy "app-x"' + +r="$t/nomgmt"; role_policies "$r" gcp '"external-secrets"' +expect "a cloud with roles but no management stack fails" 1 "$r" "no opentofu/gcp/openbao/management" + +r="$t/empty"; mkdir -p "$r/opentofu" +expect "a tree with no configure stack at all fails" 1 "$r" "no opentofu//*/configure/openbao.tf" + +r="$t/multiline"; mkdir -p "$r/opentofu/aws/k8s/configure" "$r/opentofu/aws/openbao/management" +cat >"$r/opentofu/aws/k8s/configure/openbao.tf" <<'EOF' +locals { + openbao_roles = { + external-secrets = { + policies = [ + "default", + "external-secrets", + ] + } + } +} +EOF +expect "a multi-line policies list is read" 1 "$r" 'policy "external-secrets"' + +r="$t/token"; mkdir -p "$r/opentofu/aws/k8s/configure" "$r/opentofu/aws/openbao/management" +cat >"$r/opentofu/aws/k8s/configure/openbao.tf" <<'EOF' +resource "vault_jwt_auth_backend_role" "r" { + token_policies = ["undefined-x"] +} +EOF +expect "a literal token_policies list on a role is a reference too" 1 "$r" 'policy "undefined-x"' + +expect "the repository itself passes" 0 "$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +if [ "$failures" -ne 0 ]; then echo "==> ${failures} failure(s)"; exit 1; fi +echo "==> all checks passed" diff --git a/scripts/ci/validate-openbao-policies.sh b/scripts/ci/validate-openbao-policies.sh new file mode 100755 index 000000000..fb9d973b1 --- /dev/null +++ b/scripts/ci/validate-openbao-policies.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# +# Every policy a cluster's OpenBao JWT roles name must be DEFINED by that cloud's +# OpenBao management stack -- directly, or through a module it calls. +# +# WHY THIS EXISTS +# +# A JWT role references its policies BY NAME, from a different state: the roles +# live in opentofu///configure/openbao.tf, the policies in +# opentofu//openbao/management. OpenBao accepts a role naming a policy +# that does not exist, and the login succeeds -- it just grants nothing. When +# Stage 2 shipped (2026-09-10), jwt/gcp-0's external-secrets role named +# `external-secrets`, which only the AWS management stack defined, so all 14 +# OpenBao-backed ExternalSecrets on gcp-0 authenticated and read nothing. CI +# rendered every manifest and stayed green; nothing tied the two states. +# +# This reads committed HCL, not a live OpenBao, so it answers "would this +# configuration leave a role pointing at nothing?", not "does it right now?". +# Templated policy names (app-${each.value}) are not literal names and never +# satisfy a role; the built-in `default` policy is never required. +# +# SCOPE EDGES (none reachable today, but honest): roles are read only from +# opentofu//*/configure/openbao.tf, so a role moved to a sibling file +# goes unchecked; a `vault_policy` gated to `count = 0` still counts as +# defined; and a policy name built from a variable, not a literal string, is +# not resolved either way. +# +# Usage: validate-openbao-policies.sh [ROOT_DIR] +set -euo pipefail + +ROOT="${1:-$(git rev-parse --show-toplevel)}" + +exec python3 - "$ROOT" <<'PY' +import pathlib +import re +import sys + +root = pathlib.Path(sys.argv[1]) +QUOTED = re.compile(r'"([^"$]+)"') +fails = [] + +def referenced(path): + names = set() + for m in re.finditer(r'policies\s*=\s*\[([^\]]*)\]', path.read_text()): + names.update(QUOTED.findall(m.group(1))) + names.discard("default") + return names + +def literal_policy_names(directory): + names = set() + for tf in sorted(directory.glob("*.tf")): + for block in re.finditer(r'resource\s+"vault_policy"\s+"[^"]+"\s*\{(.*?)\n\}', tf.read_text(), re.S): + n = re.search(r'^\s*name\s*=\s*"([^"$]+)"\s*$', block.group(1), re.M) + if n: + names.add(n.group(1)) + return names + +def called_module_dirs(directory): + dirs = [] + for tf in sorted(directory.glob("*.tf")): + for block in re.finditer(r'module\s+"[^"]+"\s*\{(.*?)\n\}', tf.read_text(), re.S): + s = re.search(r'^\s*source\s*=\s*"(\.[^"]+)"', block.group(1), re.M) + if s: + dirs.append((directory / s.group(1)).resolve()) + return dirs + +opentofu = root / "opentofu" +clouds = sorted(p.name for p in opentofu.iterdir() if p.is_dir() and p.name != "shared") if opentofu.is_dir() else [] +checked = 0 +for cloud in clouds: + configs = sorted((opentofu / cloud).glob("*/configure/openbao.tf")) + if not configs: + continue + mgmt = opentofu / cloud / "openbao" / "management" + if not mgmt.is_dir(): + fails.append(f"{cloud}: {configs[0].relative_to(root)} names OpenBao policies, but there is no opentofu/{cloud}/openbao/management") + continue + wanted = set().union(*(referenced(c) for c in configs)) + defined = literal_policy_names(mgmt) + for d in called_module_dirs(mgmt): + if d.is_dir(): + defined |= literal_policy_names(d) + checked += 1 + for name in sorted(wanted - defined): + fails.append(f'{cloud}: a JWT role names policy "{name}", but opentofu/{cloud}/openbao/management does not define it (directly or through a module it calls)') + +if checked == 0 and not fails: + fails.append("no opentofu//*/configure/openbao.tf found -- nothing to check, which is itself wrong") + +for f in fails: + print(f"FAIL: {f}") +if fails: + print(f"==> OpenBao policy parity: {len(fails)} problem(s). A role naming an undefined policy logs in and reads nothing.") + sys.exit(1) +print(f"==> OpenBao policy parity: every policy a JWT role names is defined ({checked} cloud(s)).") +PY From 9ae19f9ffee1972b07f7becd6c42bbe40b350223 Mon Sep 17 00:00:00 2001 From: Smana Date: Tue, 29 Sep 2026 09:28:17 +0200 Subject: [PATCH 2/8] feat(openbao): the Stage 2 store of record as a shared module, tested offline --- .../modules/openbao-store-of-record/apps.tf | 32 ++++++ .../modules/openbao-store-of-record/auth.tf | 31 ++++++ .../modules/openbao-store-of-record/mounts.tf | 19 ++++ .../modules/openbao-store-of-record/oidc.tf | 95 ++++++++++++++++ .../openbao-store-of-record/outputs.tf | 30 ++++++ .../openbao-store-of-record/policies.tf | 28 +++++ .../policies/admin.hcl | 101 ++++++++++++++++++ .../policies/app-prefix.hcl | 48 +++++++++ .../policies/external-secrets.hcl | 38 +++++++ .../policies/pki-admin.hcl | 67 ++++++++++++ .../policies/secrets-admin.hcl | 63 +++++++++++ .../tests/store_of_record.tftest.hcl | 75 +++++++++++++ .../openbao-store-of-record/variables.tf | 46 ++++++++ .../openbao-store-of-record/versions.tf | 16 +++ .../ci/tests/test-openbao-oidc-lifecycle.sh | 18 ++++ 15 files changed, 707 insertions(+) create mode 100644 opentofu/shared/modules/openbao-store-of-record/apps.tf create mode 100644 opentofu/shared/modules/openbao-store-of-record/auth.tf create mode 100644 opentofu/shared/modules/openbao-store-of-record/mounts.tf create mode 100644 opentofu/shared/modules/openbao-store-of-record/oidc.tf create mode 100644 opentofu/shared/modules/openbao-store-of-record/outputs.tf create mode 100644 opentofu/shared/modules/openbao-store-of-record/policies.tf create mode 100644 opentofu/shared/modules/openbao-store-of-record/policies/admin.hcl create mode 100644 opentofu/shared/modules/openbao-store-of-record/policies/app-prefix.hcl create mode 100644 opentofu/shared/modules/openbao-store-of-record/policies/external-secrets.hcl create mode 100644 opentofu/shared/modules/openbao-store-of-record/policies/pki-admin.hcl create mode 100644 opentofu/shared/modules/openbao-store-of-record/policies/secrets-admin.hcl create mode 100644 opentofu/shared/modules/openbao-store-of-record/tests/store_of_record.tftest.hcl create mode 100644 opentofu/shared/modules/openbao-store-of-record/variables.tf create mode 100644 opentofu/shared/modules/openbao-store-of-record/versions.tf create mode 100755 scripts/ci/tests/test-openbao-oidc-lifecycle.sh diff --git a/opentofu/shared/modules/openbao-store-of-record/apps.tf b/opentofu/shared/modules/openbao-store-of-record/apps.tf new file mode 100644 index 000000000..13676ab7a --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/apps.tf @@ -0,0 +1,32 @@ +# Per-app ownership of apps/ (ADR-0036). Gated on OIDC: without the OIDC mount +# there is no accessor to alias against. +locals { + secret_owning_apps = local.oidc_on == 1 ? var.secret_owning_apps : toset([]) +} + +resource "vault_policy" "app_prefix" { + for_each = local.secret_owning_apps + + name = "app-${each.value}" + policy = templatefile("${path.module}/policies/app-prefix.hcl", { + mount = vault_mount.apps.path + app = each.value + }) +} + +resource "vault_identity_group" "app" { + for_each = local.secret_owning_apps + + name = "openbao-app-${each.value}" + type = "external" + policies = [vault_policy.app_prefix[each.value].name] +} + +resource "vault_identity_group_alias" "app" { + for_each = local.secret_owning_apps + + # Must equal the ZITADEL project role name as it lands in the groups claim. + name = "app-${each.value}" + mount_accessor = vault_jwt_auth_backend.oidc[0].accessor + canonical_id = vault_identity_group.app[each.value].id +} diff --git a/opentofu/shared/modules/openbao-store-of-record/auth.tf b/opentofu/shared/modules/openbao-store-of-record/auth.tf new file mode 100644 index 000000000..983d3d564 --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/auth.tf @@ -0,0 +1,31 @@ +# The break-glass login. It stays alongside OIDC on purpose (ADR-0034): ZITADEL's +# own credential lives in platform/zitadel/envvars, so an OIDC-only login would +# have no way back in when ZITADEL is down. It MUST carry secrets-admin, or it +# authenticates and then reads nothing it exists to recover. +resource "vault_auth_backend" "userpass" { + type = "userpass" + path = "userpass" +} + +# Lands in state, as the root token already does. The caller publishes it to +# its own cloud's secret store from the `admin_password` output. +resource "random_password" "admin" { + length = 32 + special = true + override_special = "!#%*-_=+" +} + +resource "vault_generic_endpoint" "admin_user" { + path = "auth/${vault_auth_backend.userpass.path}/users/${var.admin_username}" + # The password is never readable back, so a read would always look like drift. + disable_read = true + disable_delete = false + ignore_absent_fields = true + + data_json = jsonencode({ + policies = [vault_policy.admin.name, vault_policy.pki_admin.name, vault_policy.secrets_admin.name] + password = random_password.admin.result + token_ttl = 3600 + token_max_ttl = 28800 + }) +} diff --git a/opentofu/shared/modules/openbao-store-of-record/mounts.tf b/opentofu/shared/modules/openbao-store-of-record/mounts.tf new file mode 100644 index 000000000..2ead88b65 --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/mounts.tf @@ -0,0 +1,19 @@ +# The Stage 2 store of record (ADR-0033). Root namespace, deliberately: a policy +# binds only within the namespace it is created in, and the oidc/ mount, the +# identity groups and every policy live in root. +# +# Grammar: platform//, one-to-one onto ADR-0023's dash names. +resource "vault_mount" "platform" { + path = "platform" + type = "kv-v2" + description = "Platform component secrets; store of record (ADR-0033 Stage 2)" +} + +# Separate from platform/ because External Secrets' vault provider takes ONE +# mount per store, so the split is what lets the two audiences carry different +# policies. Grammar: apps//. +resource "vault_mount" "apps" { + path = "apps" + type = "kv-v2" + description = "Application secrets, owned per app (ADR-0036)" +} diff --git a/opentofu/shared/modules/openbao-store-of-record/oidc.tf b/opentofu/shared/modules/openbao-store-of-record/oidc.tf new file mode 100644 index 000000000..24adb7414 --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/oidc.tf @@ -0,0 +1,95 @@ +# Human login through ZITADEL, authorised by project roles (ADR-0034). Everything +# is count-gated on the caller supplying a client id and an issuer, so a cluster +# whose ZITADEL is not bootstrapped yet applies cleanly with no OIDC method. +locals { + oidc_enabled = var.oidc_client_id != "" && var.oidc_issuer != "" ? 1 : 0 + + # The gate is derived from the caller's OIDC secret payload, so it may arrive + # marked sensitive -- and a sensitive value cannot be a for_each argument. + # The keys come from a plain list; only the gate needs unwrapping. try() + # covers the non-sensitive case, where nonsensitive() would error. + oidc_on = try(nonsensitive(local.oidc_enabled), local.oidc_enabled) + + # BOTH callbacks are required, and must match the `openbao` entry in + # scripts/provision/zitadel-oidc-clients.sh. The UI path embeds the mount path twice, + # which is why `path` below is pinned to "oidc". + oidc_redirect_uris = [ + "${var.openbao_address}/ui/vault/auth/oidc/oidc/callback", + "http://localhost:8250/oidc/callback", + ] +} + +resource "vault_jwt_auth_backend" "oidc" { + count = local.oidc_enabled + + path = "oidc" + type = "oidc" + description = "ZITADEL OIDC for human operators (ADR-0034)" + oidc_discovery_url = var.oidc_issuer + oidc_client_id = var.oidc_client_id + oidc_client_secret = var.oidc_client_secret + + # A LITERAL, not a reference to the role below: referencing it would order the + # role before the mount it lives in, and a fresh apply fails with + # `no handler for route "auth/oidc/role/default"`. + default_role = "default" + + # checkov:skip=CKV_SECRET_6:False positive on `token_type = "default-service"` -- an OpenBao token-type constant, not a base64 secret. + tune { + listing_visibility = "unauth" + default_lease_ttl = "1h" + max_lease_ttl = "8h" + token_type = "default-service" + } + + lifecycle { + # Same as opentofu/aws/openbao/management/oidc.tf (#2078): OpenBao never + # returns the secret on read, and every fresh ZITADEL issues a new client id, + # so an apply here would replay discovery while the IdP is still down. + # zitadel-oidc-clients.sh's reconcile_openbao_oidc rotates both fields. + ignore_changes = [oidc_client_id, oidc_client_secret] + } +} + +resource "vault_jwt_auth_backend_role" "oidc_default" { + count = local.oidc_enabled + + backend = vault_jwt_auth_backend.oidc[0].path + role_name = "default" + role_type = "oidc" + + allowed_redirect_uris = local.oidc_redirect_uris + user_claim = "email" + # Without these ZITADEL issues a token with neither `email` nor `groups`, and + # the login dies on `claim "email" not found in token`. + oidc_scopes = ["profile", "email", "groups"] + groups_claim = "groups" + bound_audiences = [var.oidc_client_id] + + # Authorisation comes from the external group below, never from the role. + token_policies = [] + token_ttl = 3600 + token_max_ttl = 28800 + + lifecycle { + # Tracks the client id, so it rotates with it (#2078). + ignore_changes = [bound_audiences] + } +} + +resource "vault_identity_group" "oidc_admin" { + count = local.oidc_enabled + + name = "openbao-admin" + type = "external" + policies = [vault_policy.admin.name, vault_policy.pki_admin.name, vault_policy.secrets_admin.name] +} + +resource "vault_identity_group_alias" "oidc_admin" { + count = local.oidc_enabled + + # Must equal the value in the token's `groups` array, exactly. + name = var.admin_group_alias + mount_accessor = vault_jwt_auth_backend.oidc[0].accessor + canonical_id = vault_identity_group.oidc_admin[0].id +} diff --git a/opentofu/shared/modules/openbao-store-of-record/outputs.tf b/opentofu/shared/modules/openbao-store-of-record/outputs.tf new file mode 100644 index 000000000..a3cb3875c --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/outputs.tf @@ -0,0 +1,30 @@ +output "admin_password" { + description = "The break-glass userpass password. The caller publishes it to its own cloud's secret store." + value = random_password.admin.result + sensitive = true +} + +output "platform_mount_path" { + description = "Path of the platform/ kv-v2 mount" + value = vault_mount.platform.path +} + +output "apps_mount_path" { + description = "Path of the apps/ kv-v2 mount" + value = vault_mount.apps.path +} + +output "policy_names" { + description = "The four policies this module defines, by the names other states reference" + value = [ + vault_policy.admin.name, + vault_policy.pki_admin.name, + vault_policy.secrets_admin.name, + vault_policy.external_secrets.name, + ] +} + +output "oidc_enabled" { + description = "1 when the OIDC login is configured, 0 otherwise" + value = local.oidc_on +} diff --git a/opentofu/shared/modules/openbao-store-of-record/policies.tf b/opentofu/shared/modules/openbao-store-of-record/policies.tf new file mode 100644 index 000000000..67d0920bc --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/policies.tf @@ -0,0 +1,28 @@ +# Platform administrators. Grants no secret path on its own; secrets-admin does. +resource "vault_policy" "admin" { + name = "admin" + policy = file("${path.module}/policies/admin.hcl") +} + +# PKI administration, templated so it tracks the mount it governs. +resource "vault_policy" "pki_admin" { + name = "pki-admin" + policy = templatefile("${path.module}/policies/pki-admin.hcl", { + pki_mount = var.pki_mount_path + }) +} + +# Full control of both Stage 2 mounts: the break-glass login and openbao-admin. +resource "vault_policy" "secrets_admin" { + name = "secrets-admin" + policy = file("${path.module}/policies/secrets-admin.hcl") +} + +# External Secrets' read-only identity over both mounts. Attached by NAME to +# the per-cluster JWT role in each cluster's configure stack -- a different +# state -- so the name must stay `external-secrets` in both places. +# scripts/ci/validate-openbao-policies.sh enforces that pairing. +resource "vault_policy" "external_secrets" { + name = "external-secrets" + policy = file("${path.module}/policies/external-secrets.hcl") +} diff --git a/opentofu/shared/modules/openbao-store-of-record/policies/admin.hcl b/opentofu/shared/modules/openbao-store-of-record/policies/admin.hcl new file mode 100644 index 000000000..f0e4faa3f --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/policies/admin.hcl @@ -0,0 +1,101 @@ +# The following doc has been taken as reference: https://learn.hashicorp.com/tutorials/vault/policies#write-a-policy +# Added the ability to manage identities + +# Read system health check +path "sys/health" +{ + capabilities = ["read", "sudo"] +} + +# Create and manage ACL policies broadly across Vault + +# List existing policies +path "sys/policies/acl" +{ + capabilities = ["list"] +} + +# Create and manage ACL policies +path "sys/policies/acl/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# Enable and manage authentication methods broadly across Vault + +# Manage auth methods broadly across Vault +path "auth/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# Manage identities broadly across Vault +path "identity/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# PKI is deliberately not granted here. It has its own policy, `pki-admin.hcl`, +# scoped to the real mount and templated from var.pki_mount_path. The operator +# login carries both. +# +# This policy used to grant `pki/*` and `int_pki/*`, and neither named a real +# mount. It was reachable, though — the repo docs documented a hand-created userpass +# admin, and both this policy and a stray `pki` mount created by the old +# `openbao-config.sh pki` step happened to land in root, so the paths lined up +# by accident. That mount held a duplicate copy of the root CA private key and +# had no `vault_pki_secret_backend_role`, so it could not issue anything anyway. +# +# Note for anyone adding paths here: this policy lives in the root namespace, so +# a path only matches mounts in root. It cannot reach into the `app` tenant +# namespace. Manage namespaces via `sys/namespaces/*` below; manage what is +# inside one from a policy created in that namespace. + +# Create, update, and delete auth methods +path "sys/auth/*" +{ + capabilities = ["create", "update", "delete", "sudo"] +} + +# List auth methods +path "sys/auth" +{ + capabilities = ["read"] +} + +# Manage tenant namespaces. Creating and deleting a namespace is a root-level +# operation; administering what is inside one needs a policy created in that +# namespace. No tenant namespace exists today -- the `app` one was removed with +# ADR-0036, because a policy on a root identity group could never reach into it. +path "sys/namespaces" +{ + capabilities = ["list"] +} + +path "sys/namespaces/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# NOTE: there is deliberately no grant on any secret PATH here. This policy +# administers OpenBao; reading and writing the kv mounts is `secrets-admin.hcl`, +# and the operator login carries both. +# +# The warning that used to sit here -- "any kv mount later created *in* root +# would need it back" -- was right, and was then missed: `platform/` and `apps/` +# were created in root and the grant went only to the OIDC admin GROUP, so the +# break-glass userpass login could authenticate and read neither. Fixed by adding +# secrets-admin to that login in auth.tf. If you add a mount, add it to +# secrets-admin.hcl AND check auth.tf carries the policy. + +# Manage secrets engines +path "sys/mounts/*" +{ + capabilities = ["create", "read", "update", "delete", "list", "sudo"] +} + +# List existing secrets engines. +path "sys/mounts" +{ + capabilities = ["read"] +} diff --git a/opentofu/shared/modules/openbao-store-of-record/policies/app-prefix.hcl b/opentofu/shared/modules/openbao-store-of-record/policies/app-prefix.hcl new file mode 100644 index 000000000..fd8508c65 --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/policies/app-prefix.hcl @@ -0,0 +1,48 @@ +# One app's own prefix under the apps/ mount. Rendered per app by apps.tf. +# +# kv-v2 splits the mount across several API paths; a grant on the bare app path +# alone matches nothing a client calls, because the client asks for +# `/data//`. +# +# `destroy` is deliberately absent: a compromised credential must not be able to +# erase secret history. Soft-delete and undelete are enough for day-to-day work, +# and permanent destruction is an administrative act (see secrets-admin.hcl). + +path "${mount}/data/${app}/*" { + capabilities = ["create", "read", "update", "patch", "delete", "list"] +} + +path "${mount}/metadata/${app}/*" { + capabilities = ["create", "read", "update", "list", "delete"] +} + +path "${mount}/delete/${app}/*" { + capabilities = ["update"] +} + +path "${mount}/undelete/${app}/*" { + capabilities = ["update"] +} + +# Listing the mount root, so the UI can show this app's folder. kv-v2 metadata +# listing at the top level is what renders the tree; without it the app's own +# prefix is reachable only by typing its full path. +path "${mount}/metadata" { + capabilities = ["list"] +} + +path "sys/mounts" { + capabilities = ["read"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/token/revoke-self" { + capabilities = ["update"] +} diff --git a/opentofu/shared/modules/openbao-store-of-record/policies/external-secrets.hcl b/opentofu/shared/modules/openbao-store-of-record/policies/external-secrets.hcl new file mode 100644 index 000000000..0adcefa31 --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/policies/external-secrets.hcl @@ -0,0 +1,38 @@ +# External Secrets reads both mounts. It writes NOTHING, anywhere. +# +# This is the property that makes per-app ownership mean something. The +# controller reads with its OWN identity, not the requester's, so if it could +# write, anything able to create a namespaced ExternalSecret could launder a +# value into another app's prefix and the per-app human policies would be +# bypassed entirely. +# +# `list` on metadata is what the provider needs to resolve a `dataFrom` extract; +# `read` on data is what serves an ordinary `remoteRef.key`. +# +# The token self-operations at the bottom are also granted by OpenBao's built-in +# `default` policy, which this role carries alongside this one. They are +# repeated here so the policy stands on its own if `default` is ever dropped. + +path "platform/data/*" { + capabilities = ["read"] +} + +path "platform/metadata/*" { + capabilities = ["read", "list"] +} + +path "apps/data/*" { + capabilities = ["read"] +} + +path "apps/metadata/*" { + capabilities = ["read", "list"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} diff --git a/opentofu/shared/modules/openbao-store-of-record/policies/pki-admin.hcl b/opentofu/shared/modules/openbao-store-of-record/policies/pki-admin.hcl new file mode 100644 index 000000000..a230fb788 --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/policies/pki-admin.hcl @@ -0,0 +1,67 @@ +# Administer the private PKI issuer. +# +# Created in the root namespace, alongside the mount. The `admin` policy used to +# carry `path "pki/*"` and `path "int_pki/*"`; neither named a real mount, and +# while the PKI lived in the `admin/pki` child namespace no path written from +# `admin` could have reached it — a policy binds only within its own namespace. +# Collapsing the platform into root makes co-location the answer. +# +# The mount path below is templated from var.pki_mount_path (escaped here as +# $${pki_mount}) so the policy cannot drift from the mount it governs. + +# Issue, sign, revoke, and manage roles, issuers and keys on the mount. +path "${pki_mount}" { + capabilities = ["read", "list"] +} + +path "${pki_mount}/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +# Rotating an issuer requires sudo on these two. +path "${pki_mount}/root/rotate/*" { + capabilities = ["create", "update", "sudo"] +} + +path "${pki_mount}/issuers/generate/*" { + capabilities = ["create", "update", "sudo"] +} + +# Read and tune the mount's lease TTLs, and list what is mounted here. +path "sys/mounts" { + capabilities = ["read", "list"] +} + +path "sys/mounts/${pki_mount}" { + capabilities = ["read"] +} + +path "sys/mounts/${pki_mount}/tune" { + capabilities = ["read", "update"] +} + +# Inspect and revoke the leases this mount hands out. +path "sys/leases/lookup/${pki_mount}/*" { + capabilities = ["read", "list"] +} + +path "sys/leases/revoke" { + capabilities = ["create", "update"] +} + +path "sys/health" { + capabilities = ["read"] +} + +# Let an operator see and manage their own token. +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/token/revoke-self" { + capabilities = ["update"] +} diff --git a/opentofu/shared/modules/openbao-store-of-record/policies/secrets-admin.hcl b/opentofu/shared/modules/openbao-store-of-record/policies/secrets-admin.hcl new file mode 100644 index 000000000..b71d8b55f --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/policies/secrets-admin.hcl @@ -0,0 +1,63 @@ +# Full control of both secret mounts, for platform administrators. +# +# kv-v2 splits one logical mount across several API paths: values under `data/`, +# version history and soft-deletes under `metadata/`, `delete/`, `undelete/` and +# `destroy/`. A grant on `platform/*` alone is the classic kv-v2 mistake -- it +# reads correctly and matches nothing a client actually calls, because the +# client asks for `platform/data/`. +# +# Unlike the per-app policies, this one DOES carry `destroy`: erasing a secret's +# history is an administrative act, and someone has to be able to do it. + +path "platform/data/*" { + capabilities = ["create", "read", "update", "patch", "delete", "list"] +} + +path "platform/metadata/*" { + capabilities = ["create", "read", "update", "list", "delete"] +} + +path "platform/delete/*" { + capabilities = ["update"] +} + +path "platform/undelete/*" { + capabilities = ["update"] +} + +path "platform/destroy/*" { + capabilities = ["update"] +} + +path "platform/config" { + capabilities = ["read", "update"] +} + +path "apps/data/*" { + capabilities = ["create", "read", "update", "patch", "delete", "list"] +} + +path "apps/metadata/*" { + capabilities = ["create", "read", "update", "list", "delete"] +} + +path "apps/delete/*" { + capabilities = ["update"] +} + +path "apps/undelete/*" { + capabilities = ["update"] +} + +path "apps/destroy/*" { + capabilities = ["update"] +} + +path "apps/config" { + capabilities = ["read", "update"] +} + +# Listing the mounts themselves, so the UI can render the secrets tree. +path "sys/mounts" { + capabilities = ["read"] +} diff --git a/opentofu/shared/modules/openbao-store-of-record/tests/store_of_record.tftest.hcl b/opentofu/shared/modules/openbao-store-of-record/tests/store_of_record.tftest.hcl new file mode 100644 index 000000000..d7ca58bf8 --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/tests/store_of_record.tftest.hcl @@ -0,0 +1,75 @@ +# Offline: mocked providers, so no OpenBao is needed. `command = apply` because +# the break-glass user's data_json embeds the generated password, which is only +# known after an apply -- and under mock providers an apply calls nothing real. +mock_provider "vault" {} +mock_provider "random" {} + +variables { + pki_mount_path = "pki_private_issuer" + openbao_address = "https://bao.priv.gcp.ogenki.io:8200" +} + +run "mounts_policies_and_break_glass_without_oidc" { + command = apply + + assert { + condition = vault_mount.platform.path == "platform" && vault_mount.apps.path == "apps" + error_message = "the two Stage 2 mounts must be platform/ and apps/" + } + assert { + condition = vault_mount.platform.type == "kv-v2" && vault_mount.apps.type == "kv-v2" + error_message = "both mounts must be kv-v2: the ClusterSecretStores read with version v2" + } + assert { + condition = vault_policy.external_secrets.name == "external-secrets" + error_message = "the JWT roles reference this policy BY NAME: it must be external-secrets" + } + assert { + condition = length(vault_jwt_auth_backend.oidc) == 0 && length(vault_identity_group.oidc_admin) == 0 + error_message = "OIDC must stay off without a client id and an issuer" + } + assert { + condition = length(vault_policy.app_prefix) == 0 + error_message = "per-app personas alias the OIDC mount, so they must not exist without it" + } + assert { + condition = contains(jsondecode(vault_generic_endpoint.admin_user.data_json).policies, "secrets-admin") + error_message = "the break-glass login must carry secrets-admin, or it cannot read what it exists to recover" + } +} + +run "oidc_group_and_personas_when_configured" { + command = apply + + variables { + oidc_client_id = "fixture-client" + oidc_client_secret = "fixture-value" # pragma: allowlist secret + oidc_issuer = "https://auth.gcp.cloud.ogenki.io" + secret_owning_apps = ["app-wizard", "image-gallery"] + } + + assert { + condition = length(vault_jwt_auth_backend.oidc) == 1 && vault_jwt_auth_backend.oidc[0].path == "oidc" + error_message = "the OIDC mount must exist, at the pinned path oidc/" + } + assert { + condition = vault_identity_group_alias.oidc_admin[0].name == "admin" + error_message = "the openbao-admin group must alias ZITADEL's admin project role" + } + assert { + condition = contains(vault_identity_group.oidc_admin[0].policies, "secrets-admin") + error_message = "openbao-admin must carry secrets-admin" + } + assert { + condition = length(vault_policy.app_prefix) == 2 + error_message = "one policy per secret-owning app" + } + assert { + condition = contains(vault_jwt_auth_backend_role.oidc_default[0].oidc_scopes, "groups") && contains(vault_jwt_auth_backend_role.oidc_default[0].oidc_scopes, "email") + error_message = "the default role must request groups and email, or the login dies on `claim \"email\" not found in token`" + } + assert { + condition = contains(vault_jwt_auth_backend_role.oidc_default[0].allowed_redirect_uris, "https://bao.priv.gcp.ogenki.io:8200/ui/vault/auth/oidc/oidc/callback") + error_message = "the UI callback must match what scripts/provision/zitadel-oidc-clients.sh registers" + } +} diff --git a/opentofu/shared/modules/openbao-store-of-record/variables.tf b/opentofu/shared/modules/openbao-store-of-record/variables.tf new file mode 100644 index 000000000..6108532a2 --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/variables.tf @@ -0,0 +1,46 @@ +variable "pki_mount_path" { + description = "The PKI mount this OpenBao issues from. Templated into the pki-admin policy so the policy cannot drift from the mount it governs." + type = string +} + +variable "openbao_address" { + description = "https://bao.:8200. Builds the OIDC UI callback, which must match what scripts/provision/zitadel-oidc-clients.sh registers." + type = string +} + +variable "admin_username" { + description = "Username of the userpass break-glass login" + type = string + default = "admin" +} + +variable "admin_group_alias" { + description = "The value in the token's `groups` claim that maps to openbao-admin: ZITADEL's `admin` project role (ZITADEL_PROJECT_ROLES in scripts/provision/zitadel-oidc-clients.sh)." + type = string + default = "admin" +} + +variable "secret_owning_apps" { + description = "Apps that own a prefix under apps/. Each gets a policy, an external identity group and an alias (ADR-0036). Only created when OIDC is configured, because the alias is bound to the OIDC mount." + type = set(string) + default = [] +} + +variable "oidc_client_id" { + description = "ZITADEL client id for OpenBao's OIDC login. Empty disables OIDC, so a first deploy converges before ZITADEL exists." + type = string + default = "" +} + +variable "oidc_client_secret" { + description = "ZITADEL client secret for OpenBao's OIDC login" + type = string + default = "" + sensitive = true +} + +variable "oidc_issuer" { + description = "ZITADEL issuer URL. Empty disables OIDC." + type = string + default = "" +} diff --git a/opentofu/shared/modules/openbao-store-of-record/versions.tf b/opentofu/shared/modules/openbao-store-of-record/versions.tf new file mode 100644 index 000000000..03e0e451d --- /dev/null +++ b/opentofu/shared/modules/openbao-store-of-record/versions.tf @@ -0,0 +1,16 @@ +terraform { + required_version = "~> 1.5" + + required_providers { + # The OpenBao API is Vault-compatible, so the vault provider drives it. + # `~> 5.0` admits both callers: AWS pins ~> 5.0, GCP ~> 5.4. + vault = { + source = "hashicorp/vault" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + } +} diff --git a/scripts/ci/tests/test-openbao-oidc-lifecycle.sh b/scripts/ci/tests/test-openbao-oidc-lifecycle.sh new file mode 100755 index 000000000..3b745f3b9 --- /dev/null +++ b/scripts/ci/tests/test-openbao-oidc-lifecycle.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# #2078 on every copy of the OpenBao OIDC login (GCP parity GP-2): tofu must never +# rewrite the rotating client fields. A management apply runs before ZITADEL is up +# on every rebuild, and rewriting them replays OIDC discovery against an IdP that +# is not there yet; zitadel-oidc-clients.sh's reconcile_openbao_oidc rotates them. +set -uo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +fails=0 +for f in opentofu/aws/openbao/management/oidc.tf opentofu/shared/modules/openbao-store-of-record/oidc.tf; do + [ -f "$ROOT/$f" ] || { echo "FAIL $f is missing"; fails=$((fails + 1)); continue; } + grep -Eq 'ignore_changes[[:space:]]*=[[:space:]]*\[oidc_client_id, oidc_client_secret\]' "$ROOT/$f" \ + || { echo "FAIL $f: the backend does not ignore oidc_client_id/oidc_client_secret"; fails=$((fails + 1)); } + grep -Eq 'ignore_changes[[:space:]]*=[[:space:]]*\[bound_audiences\]' "$ROOT/$f" \ + || { echo "FAIL $f: the default role does not ignore bound_audiences"; fails=$((fails + 1)); } +done +[ "$fails" -eq 0 ] || exit 1 +echo PASS From b4979c47bf89b1780caa129a45b9137145cc67ec Mon Sep 17 00:00:00 2001 From: Smana Date: Tue, 29 Sep 2026 09:36:42 +0200 Subject: [PATCH 3/8] feat(openbao): GCP's OpenBao gets the Stage 2 mounts, policies and logins --- .github/workflows/ci.yaml | 8 ++ opentofu/gcp/openbao/management/auth.tf | 5 +- opentofu/gcp/openbao/management/outputs.tf | 5 ++ opentofu/gcp/openbao/management/policies.tf | 7 +- .../gcp/openbao/management/store-of-record.tf | 80 +++++++++++++++++++ opentofu/gcp/openbao/management/variables.tf | 30 +++++++ .../gcp/openbao/management/variables.tfvars | 4 + opentofu/gcp/openbao/management/versions.tf | 4 + .../gcp/openbao/management/workflows.tm.hcl | 13 ++- .../tests/test-validate-openbao-policies.sh | 2 +- scripts/tasks.yaml | 4 + taskfile.yaml | 1 + 12 files changed, 153 insertions(+), 10 deletions(-) create mode 100644 opentofu/gcp/openbao/management/store-of-record.tf diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 45a33958d..b3e3d0758 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -83,6 +83,11 @@ jobs: # What the pre-commit-terraform hooks call instead of `terraform`. PCT_TFPATH: tofu + - name: Test the OpenBao store-of-record module + run: | + tofu -chdir=opentofu/shared/modules/openbao-store-of-record init -backend=false -input=false + tofu -chdir=opentofu/shared/modules/openbao-store-of-record test + security-scan: name: Security scanning 🔒 runs-on: ubuntu-latest @@ -322,6 +327,9 @@ jobs: - name: Check identity provider topology run: ./scripts/ci/validate-idp-topology.sh + - name: OpenBao policy parity + run: ./scripts/ci/validate-openbao-policies.sh + # --------------------------------------------------------------------------- # Post-merge only. A red run on `main` is already visible next to the commit, # but only to someone looking at it -- and the whole point of the `push` diff --git a/opentofu/gcp/openbao/management/auth.tf b/opentofu/gcp/openbao/management/auth.tf index 27a6ac97c..6780bb9b5 100644 --- a/opentofu/gcp/openbao/management/auth.tf +++ b/opentofu/gcp/openbao/management/auth.tf @@ -5,5 +5,6 @@ # pinned cert-manager role_id and the two Secret Manager entries, is gone: a # JWT login mints nothing long-lived. # -# No human auth method on GCP yet; operators use the root token from -# openbao-priv-gcp-root-token, as documented. +# Human logins -- the userpass break-glass and the ZITADEL OIDC method -- come +# from the shared module (store-of-record.tf). The break-glass password is +# published to openbao-priv-gcp-admin-credentials. diff --git a/opentofu/gcp/openbao/management/outputs.tf b/opentofu/gcp/openbao/management/outputs.tf index 6e6e574dd..d7c141768 100644 --- a/opentofu/gcp/openbao/management/outputs.tf +++ b/opentofu/gcp/openbao/management/outputs.tf @@ -7,3 +7,8 @@ output "cert_manager_role_name" { description = "PKI role name cert-manager's ClusterIssuer must reference" value = vault_pki_secret_backend_role.cert_manager.name } + +output "admin_credentials_secret_name" { + description = "GCP Secret Manager entry holding the break-glass userpass credentials" + value = google_secret_manager_secret.admin_credentials.secret_id +} diff --git a/opentofu/gcp/openbao/management/policies.tf b/opentofu/gcp/openbao/management/policies.tf index 0d02d3170..09b360a91 100644 --- a/opentofu/gcp/openbao/management/policies.tf +++ b/opentofu/gcp/openbao/management/policies.tf @@ -1,6 +1,7 @@ -# Two policies now. The AWS stack also carries admin, pki-admin and app -# policies; none of those has a consumer on GCP, and the design scopes this -# stack to PKI only. Add them when something needs them, not before. +# The cert-manager and snapshot policies below are GCP's own. The Stage 2 +# policies -- admin, pki-admin, secrets-admin, external-secrets and the per-app +# ones -- come from the shared module in store-of-record.tf, the same set AWS +# defines inline. # # `templatefile` rather than `file` so the granted path is derived from the mount # and role that actually exist, instead of being restated as a literal that diff --git a/opentofu/gcp/openbao/management/store-of-record.tf b/opentofu/gcp/openbao/management/store-of-record.tf new file mode 100644 index 000000000..df24486c5 --- /dev/null +++ b/opentofu/gcp/openbao/management/store-of-record.tf @@ -0,0 +1,80 @@ +# Stage 2 on GCP (docs/superpowers/specs/2026-09-11-openbao-stage2-gcp-design.md, +# ADR-0037): the same mounts, policies and logins AWS defines, from the shared +# module. Until this existed, jwt/gcp-0's external-secrets role named a policy +# no GCP OpenBao ever had, and the 14 OpenBao-backed ExternalSecrets on gcp-0 +# authenticated and then read nothing. + +# OIDC is gated on the `openbao-oidc` entry EXISTING, not on the variable +# being set. Reading a version of a missing secret is a hard error; listing +# first turns "ZITADEL not bootstrapped yet" into "OIDC off", so a first deploy +# converges and the second apply after zitadel-oidc-clients.sh picks it up -- +# the same two-pass shape as opentofu/aws/openbao/management/oidc.tf. +# +# The gate checks that the SECRET exists, not that it has a VERSION -- the +# provider has no version-listing data source. zitadel-oidc-clients.sh writes on +# GCP as create + versions add, so a failed add leaves a version-less secret and +# this plan then fails on "latest" not found. Recovery: add the version (re-run +# zitadel-oidc-clients.sh sync --apply) or delete the empty secret +# (gcloud secrets delete openbao-oidc --project ), then re-apply. +data "google_secret_manager_secrets" "openbao_oidc" { + count = var.openbao_oidc_secret_id == "" ? 0 : 1 + project = var.project_id + # Not an exact match in Secret Manager's list filter; contains() below is. + filter = "name:${var.openbao_oidc_secret_id}" +} + +locals { + oidc_secret_present = var.openbao_oidc_secret_id != "" && contains( + [for s in try(data.google_secret_manager_secrets.openbao_oidc[0].secrets, []) : s.secret_id], + var.openbao_oidc_secret_id + ) +} + +data "google_secret_manager_secret_version" "openbao_oidc" { + count = local.oidc_secret_present ? 1 : 0 + secret = var.openbao_oidc_secret_id + project = var.project_id +} + +locals { + oidc_raw = try(jsondecode(data.google_secret_manager_secret_version.openbao_oidc[0].secret_data), {}) + oidc_client_id = try(local.oidc_raw["client_id"], "") + oidc_client_secret = try(local.oidc_raw["client_secret"], "") + # Falls back to the `endpoint` the registration script stores alongside the + # credentials, so the IdP hostname is not configured twice. + oidc_issuer = var.openbao_oidc_issuer != "" ? var.openbao_oidc_issuer : try(local.oidc_raw["endpoint"], "") +} + +module "store_of_record" { + source = "../../../shared/modules/openbao-store-of-record" + + pki_mount_path = vault_mount.pki.path + openbao_address = local.openbao_address + admin_username = var.admin_username + admin_group_alias = "admin" + secret_owning_apps = var.secret_owning_apps + oidc_client_id = local.oidc_client_id + oidc_client_secret = local.oidc_client_secret + oidc_issuer = local.oidc_issuer +} + +# Published so an operator can retrieve the break-glass password -- the GCP +# twin of opentofu/aws/openbao/management/secrets.tf. Google-managed key, +# matching the AVD-AWS-0098 decision recorded in .trivyignore.yaml. +resource "google_secret_manager_secret" "admin_credentials" { + project = var.project_id + secret_id = var.admin_credentials_secret_name + + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "admin_credentials" { + secret = google_secret_manager_secret.admin_credentials.id + secret_data = jsonencode({ + username = var.admin_username + password = module.store_of_record.admin_password + address = local.openbao_address + }) +} diff --git a/opentofu/gcp/openbao/management/variables.tf b/opentofu/gcp/openbao/management/variables.tf index 24be156c3..ecf4031bd 100644 --- a/opentofu/gcp/openbao/management/variables.tf +++ b/opentofu/gcp/openbao/management/variables.tf @@ -122,3 +122,33 @@ variable "external_secrets_service_account" { type = string default = "external-secrets" } + +variable "admin_username" { + description = "Username of the userpass break-glass login, carrying admin, pki-admin and secrets-admin" + type = string + default = "admin" +} + +variable "admin_credentials_secret_name" { + description = "GCP Secret Manager entry this stack publishes the break-glass credentials to, as {username, password, address}" + type = string + default = "openbao-priv-gcp-admin-credentials" +} + +variable "openbao_oidc_secret_id" { + description = "GCP Secret Manager entry holding {client_id, client_secret, endpoint} for OpenBao's ZITADEL OIDC client, written by scripts/provision/zitadel-oidc-clients.sh. An entry that does not exist yet disables OIDC rather than failing." + type = string + default = "openbao-oidc" +} + +variable "openbao_oidc_issuer" { + description = "ZITADEL issuer URL. Defaults to the `endpoint` field of openbao_oidc_secret_id." + type = string + default = "" +} + +variable "secret_owning_apps" { + description = "Apps that own a prefix under apps/ (ADR-0036). Each gets a policy, an external identity group and an alias once OIDC is configured." + type = set(string) + default = [] +} diff --git a/opentofu/gcp/openbao/management/variables.tfvars b/opentofu/gcp/openbao/management/variables.tfvars index 70473b507..804596293 100644 --- a/opentofu/gcp/openbao/management/variables.tfvars +++ b/opentofu/gcp/openbao/management/variables.tfvars @@ -2,3 +2,7 @@ # project's real values (europe-west4, priv.gcp.ogenki.io, the Secret Manager # entry names the ceremony and init script create). project_id = "ogenki-435905" + +# The same apps as AWS (opentofu/aws/openbao/management/variables.tfvars), so a +# persona behaves identically on either cloud's OpenBao. +secret_owning_apps = ["app-wizard", "image-gallery"] diff --git a/opentofu/gcp/openbao/management/versions.tf b/opentofu/gcp/openbao/management/versions.tf index 255c608be..0dc5b4a11 100644 --- a/opentofu/gcp/openbao/management/versions.tf +++ b/opentofu/gcp/openbao/management/versions.tf @@ -11,5 +11,9 @@ terraform { source = "hashicorp/vault" version = "~> 5.4" } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } } } diff --git a/opentofu/gcp/openbao/management/workflows.tm.hcl b/opentofu/gcp/openbao/management/workflows.tm.hcl index 4b24cca6f..293ce2726 100644 --- a/opentofu/gcp/openbao/management/workflows.tm.hcl +++ b/opentofu/gcp/openbao/management/workflows.tm.hcl @@ -219,11 +219,16 @@ script "destroy" { # gated on TM_LINEAGE_DESTROY=true, so this path is only ever taken when # an operator has deliberately asked to tear the lineage down. # - # Only `vault_*` is dropped from state. The google_secret_manager_* and - # random_password resources here are real and do not match the prefix, - # so tofu still deletes them. + # Two prefixes are dropped from state: this stack's own `vault_*` + # resources, and the shared module's `module.store_of_record.vault_*` + # ones (matching is on the START of the state address, so `vault_` + # alone does not match a `module.` address -- see the script's header). + # The module's `random_password` and this stack's `google_secret_manager_*` + # resources are real and do not match either prefix, so tofu still + # deletes them. bash "${terramate.root.path.fs.absolute}/scripts/ops/teardown/tofu-destroy-contained.sh" \ - --contained-prefix vault_ -- \ + --contained-prefix vault_ \ + --contained-prefix module.store_of_record.vault_ -- \ -auto-approve -parallelism=1 -var-file=variables.tfvars BASH ], diff --git a/scripts/ci/tests/test-validate-openbao-policies.sh b/scripts/ci/tests/test-validate-openbao-policies.sh index f618ec262..70cf91b84 100755 --- a/scripts/ci/tests/test-validate-openbao-policies.sh +++ b/scripts/ci/tests/test-validate-openbao-policies.sh @@ -9,7 +9,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -VALIDATOR="${SCRIPT_DIR}/../validate-openbao-policies.sh" +VALIDATOR="$SCRIPT_DIR/../validate-openbao-policies.sh" failures=0 # role_policies -- a configure stack naming policies diff --git a/scripts/tasks.yaml b/scripts/tasks.yaml index 1acc84166..b3e310f44 100644 --- a/scripts/tasks.yaml +++ b/scripts/tasks.yaml @@ -33,6 +33,10 @@ tasks: desc: Assert exactly one cloud hosts ZITADEL (ADR-0027) cmds: ["{{.TASKFILE_DIR}}/ci/validate-idp-topology.sh"] + openbao-policies: + desc: Assert every policy an OpenBao JWT role names is defined by its cloud's management stack + cmds: ["{{.TASKFILE_DIR}}/ci/validate-openbao-policies.sh"] + vector-vrl: desc: Test the CNPG auto_explain VRL in the timberio/vector image (needs docker) cmds: ["{{.TASKFILE_DIR}}/ci/validate-vector-vrl.sh"] diff --git a/taskfile.yaml b/taskfile.yaml index eea0eef36..cd96ce9fd 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -30,4 +30,5 @@ tasks: - {task: "ci:links"} - {task: "ci:doc-claims"} - {task: "ci:idp-topology"} + - {task: "ci:openbao-policies"} - {task: "ci:doc-paths"} From d46d8b2b72f9220074f5b9c74dc06b71deebddb6 Mon Sep 17 00:00:00 2001 From: Smana Date: Tue, 29 Sep 2026 09:39:12 +0200 Subject: [PATCH 4/8] fix(openbao): anchor the OIDC lifecycle guard, restore the module's rationale, widen its tests --- .../modules/openbao-store-of-record/auth.tf | 8 ++++++- .../modules/openbao-store-of-record/oidc.tf | 5 ++++ .../tests/store_of_record.tftest.hcl | 23 +++++++++++++++++++ .../ci/tests/test-openbao-oidc-lifecycle.sh | 5 ++-- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/opentofu/shared/modules/openbao-store-of-record/auth.tf b/opentofu/shared/modules/openbao-store-of-record/auth.tf index 983d3d564..a08b8e15d 100644 --- a/opentofu/shared/modules/openbao-store-of-record/auth.tf +++ b/opentofu/shared/modules/openbao-store-of-record/auth.tf @@ -2,13 +2,19 @@ # own credential lives in platform/zitadel/envvars, so an OIDC-only login would # have no way back in when ZITADEL is down. It MUST carry secrets-admin, or it # authenticates and then reads nothing it exists to recover. +# +# Deliberately no token_bound_cidrs: the only route to this API is the private +# load balancer, so the network is already constrained, and a CIDR bind on the +# break-glass credential buys nothing against the risk of locking yourself out. resource "vault_auth_backend" "userpass" { type = "userpass" path = "userpass" } # Lands in state, as the root token already does. The caller publishes it to -# its own cloud's secret store from the `admin_password` output. +# its own cloud's secret store from the `admin_password` output. The write-only +# `password_wo` would keep it out of state, but it needs write-only attribute +# support, so it is deliberately not used here. resource "random_password" "admin" { length = 32 special = true diff --git a/opentofu/shared/modules/openbao-store-of-record/oidc.tf b/opentofu/shared/modules/openbao-store-of-record/oidc.tf index 24adb7414..877c08571 100644 --- a/opentofu/shared/modules/openbao-store-of-record/oidc.tf +++ b/opentofu/shared/modules/openbao-store-of-record/oidc.tf @@ -47,6 +47,11 @@ resource "vault_jwt_auth_backend" "oidc" { # returns the secret on read, and every fresh ZITADEL issues a new client id, # so an apply here would replay discovery while the IdP is still down. # zitadel-oidc-clients.sh's reconcile_openbao_oidc rotates both fields. + # + # Caveat: ANY other in-place update to this resource re-sends the whole + # config, oidc_client_secret taken from state (the creation-time value). + # After one, re-run the sync with --openbao-*. The real fix, + # oidc_client_secret_wo, is a follow-up. ignore_changes = [oidc_client_id, oidc_client_secret] } } diff --git a/opentofu/shared/modules/openbao-store-of-record/tests/store_of_record.tftest.hcl b/opentofu/shared/modules/openbao-store-of-record/tests/store_of_record.tftest.hcl index d7ca58bf8..4c54847fd 100644 --- a/opentofu/shared/modules/openbao-store-of-record/tests/store_of_record.tftest.hcl +++ b/opentofu/shared/modules/openbao-store-of-record/tests/store_of_record.tftest.hcl @@ -36,6 +36,29 @@ run "mounts_policies_and_break_glass_without_oidc" { condition = contains(jsondecode(vault_generic_endpoint.admin_user.data_json).policies, "secrets-admin") error_message = "the break-glass login must carry secrets-admin, or it cannot read what it exists to recover" } + assert { + condition = strcontains(vault_policy.pki_admin.policy, "path \"pki_private_issuer/*\"") + error_message = "pki-admin must be templated from pki_mount_path, or it governs a mount that does not exist" + } +} + +run "oidc_stays_off_with_a_client_id_but_no_issuer" { + command = apply + + variables { + oidc_client_id = "fixture-client" + oidc_client_secret = "fixture-value" # pragma: allowlist secret + secret_owning_apps = ["app-wizard"] + } + + assert { + condition = length(vault_jwt_auth_backend.oidc) == 0 && length(vault_jwt_auth_backend_role.oidc_default) == 0 && length(vault_identity_group.oidc_admin) == 0 + error_message = "OIDC needs both a client id and an issuer; a client id alone must not enable it" + } + assert { + condition = length(vault_policy.app_prefix) == 0 && length(vault_identity_group.app) == 0 + error_message = "per-app personas alias the OIDC mount, so they must not exist without it" + } } run "oidc_group_and_personas_when_configured" { diff --git a/scripts/ci/tests/test-openbao-oidc-lifecycle.sh b/scripts/ci/tests/test-openbao-oidc-lifecycle.sh index 3b745f3b9..b41c6a3ee 100755 --- a/scripts/ci/tests/test-openbao-oidc-lifecycle.sh +++ b/scripts/ci/tests/test-openbao-oidc-lifecycle.sh @@ -7,11 +7,12 @@ set -uo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" fails=0 +# Anchored to the line start so a commented-out `# ignore_changes` cannot pass. for f in opentofu/aws/openbao/management/oidc.tf opentofu/shared/modules/openbao-store-of-record/oidc.tf; do [ -f "$ROOT/$f" ] || { echo "FAIL $f is missing"; fails=$((fails + 1)); continue; } - grep -Eq 'ignore_changes[[:space:]]*=[[:space:]]*\[oidc_client_id, oidc_client_secret\]' "$ROOT/$f" \ + grep -Eq '^[[:space:]]*ignore_changes[[:space:]]*=[[:space:]]*\[oidc_client_id, oidc_client_secret\]' "$ROOT/$f" \ || { echo "FAIL $f: the backend does not ignore oidc_client_id/oidc_client_secret"; fails=$((fails + 1)); } - grep -Eq 'ignore_changes[[:space:]]*=[[:space:]]*\[bound_audiences\]' "$ROOT/$f" \ + grep -Eq '^[[:space:]]*ignore_changes[[:space:]]*=[[:space:]]*\[bound_audiences\]' "$ROOT/$f" \ || { echo "FAIL $f: the default role does not ignore bound_audiences"; fails=$((fails + 1)); } done [ "$fails" -eq 0 ] || exit 1 From 63c30ae41b4c4a3f8aa57ac1179cf3ce2ae02083 Mon Sep 17 00:00:00 2001 From: Smana Date: Tue, 29 Sep 2026 09:42:44 +0200 Subject: [PATCH 5/8] feat(secrets): migrate takes explicit keys, for a cluster already pointed at OpenBao Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011xuNsQcGpQPcEH8m1h3rDD --- .../tests/test-secret-store-migrate-keys.sh | 95 +++++++++++++++++++ scripts/provision/secret-store.sh | 38 +++++++- 2 files changed, 131 insertions(+), 2 deletions(-) create mode 100755 scripts/ci/tests/test-secret-store-migrate-keys.sh diff --git a/scripts/ci/tests/test-secret-store-migrate-keys.sh b/scripts/ci/tests/test-secret-store-migrate-keys.sh new file mode 100755 index 000000000..296fa5af9 --- /dev/null +++ b/scripts/ci/tests/test-secret-store-migrate-keys.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2034 +# (KEYS and KEYS_SET are read by the body of migrate_keys(), which is eval'd +# in from the script under test, so static analysis cannot see that use. Same +# reason as test-openbao-new-lineage.sh.) +# +# Which source keys does `secret-store.sh migrate` walk? +# +# WHY. Normally the keys this cluster's ExternalSecrets ask for. But once the +# shared ExternalSecrets were repointed at OpenBao (Stage 2), gcp-0's ask for +# OpenBao paths such as `zitadel/envvars`, which bao_target_for does not map -- +# so a cluster-derived walk skips every key and migrates nothing, while +# reporting success. --keys names the managed-store keys explicitly. +# +# `--keys ""` (or a KEYS_SET but blank/all-separators list) is NOT "no --keys": +# an operator interpolating an unset or empty shell variable into --keys would +# otherwise silently fall back to the same cluster-derived walk that gcp-0's +# migrate --keys is meant to bypass, and still report success. So --keys given +# but resolving to zero keys is refused (exit 2), never a fallback. +# +# migrate_keys() is lifted out of the script, so this tests the code that ships. +set -uo pipefail +cd "$(dirname "$0")/../../.." || exit +fail=0 +check() { + if [ "$2" = "$3" ]; then printf ' ok %s\n' "$1" + else printf ' FAIL %s: expected %q got %q\n' "$1" "$2" "$3"; fail=1; fi +} + +# check_refuses : asserts migrate_keys(), called with the KEYS/KEYS_SET +# already set by the caller, exits 2, prints nothing on stdout, and explains +# itself on stderr. +check_refuses() { + local name="$1" out err rc + out="$(migrate_keys 2>/dev/null)"; rc=$? + err="$(migrate_keys 2>&1 1>/dev/null)" + if [ "$rc" -eq 2 ] && [ -z "$out" ] && printf '%s' "$err" | grep -qF "refusing rather than falling back"; then + printf ' ok %s\n' "$name" + else + printf ' FAIL %s: rc=%s out=%q err=%q\n' "$name" "$rc" "$out" "$err" + fail=1 + fi +} + +body="$(sed -n '/^migrate_keys() {/,/^}/p' scripts/provision/secret-store.sh)" +[ -n "$body" ] || { echo "could not extract migrate_keys() from scripts/provision/secret-store.sh" >&2; exit 1; } +eval "$body" +migrate_source_keys() { printf 'from-the-cluster\n'; } + +KEYS_SET="false" +KEYS="" +check "no --keys: the cluster's ExternalSecrets decide" "from-the-cluster" "$(migrate_keys)" + +KEYS_SET="true" +KEYS="zitadel-envvars,harbor-oidc" +check "commas separate, output sorted" $'harbor-oidc\nzitadel-envvars' "$(migrate_keys)" +KEYS_SET="true" +KEYS="zitadel-envvars harbor-oidc" +check "spaces separate" $'harbor-oidc\nzitadel-envvars' "$(migrate_keys)" +KEYS_SET="true" +KEYS="a,, b ,a" +check "blanks dropped, duplicates collapsed" $'a\nb' "$(migrate_keys)" + +KEYS_SET="true" +KEYS="" +check_refuses "--keys \"\" refuses rather than falling back to the cluster" +KEYS_SET="true" +KEYS=" , ," +check_refuses "--keys of only separators/blanks refuses" + +fn="$(sed -n '/^cmd_migrate() {/,/^}/p' scripts/provision/secret-store.sh)" +if printf '%s' "$fn" | grep -qF 'keys="$(migrate_keys)" || exit 2'; then + printf ' ok cmd_migrate captures migrate_keys and exits 2 on failure\n' +else + printf ' FAIL cmd_migrate must capture migrate_keys and exit 2 on failure\n'; fail=1 +fi +if printf '%s' "$fn" | grep -qF 'done <<<"$keys"'; then + printf ' ok cmd_migrate loops over the captured keys\n' +else + printf ' FAIL cmd_migrate must loop over the captured $keys\n'; fail=1 +fi +if printf '%s' "$fn" | grep -qF 'migrate_source_keys'; then + printf ' FAIL cmd_migrate must not call migrate_source_keys directly\n'; fail=1 +else + printf ' ok cmd_migrate does not call migrate_source_keys directly\n' +fi +if grep -qE '^[[:space:]]+--keys\)[[:space:]]+KEYS="\$2"; KEYS_SET="true"; shift 2 ;;' scripts/provision/secret-store.sh; then + printf ' ok --keys is parsed and marks KEYS_SET\n' +else + printf ' FAIL --keys is not parsed correctly\n'; fail=1 +fi + +echo +if [ "$fail" -eq 0 ]; then echo "all checks passed"; else echo "FAILURES"; fi +exit "$fail" diff --git a/scripts/provision/secret-store.sh b/scripts/provision/secret-store.sh index 65eed81fb..55f17fe91 100755 --- a/scripts/provision/secret-store.sh +++ b/scripts/provision/secret-store.sh @@ -51,7 +51,7 @@ # secret, so it is safe to re-run and safe on a cluster whose secrets were # written by hand. Most of the store is not seedable -- see GENERATABLE. # -# migrate --cloud aws|gcp [--context CTX] [--apply] +# migrate --cloud aws|gcp [--context CTX] [--keys K1,K2] [--apply] # Copy every mapped key from the cloud managed store into OpenBao, as the # store of record (ADR-0033 Stage 2). Dry-run unless --apply. Additive: an # existing destination is left alone and the source is never deleted, so @@ -62,6 +62,14 @@ # writes a platform secret into an app's prefix, which is a privilege # boundary rather than a cosmetic mistake. # +# --keys K1,K2,... walks exactly these managed-store keys instead of the +# ones the cluster's ExternalSecrets name. Needed once those ExternalSecrets +# already point at OpenBao: they then name OpenBao paths, which the mapping +# does not know, and a cluster-derived walk copies nothing. Given with no +# usable keys (empty, all blank, or an unset variable substituted in), +# --keys refuses rather than silently falling back to the cluster's +# ExternalSecrets. +# # migrate-aws [--apply] # Copy AWS Secrets Manager entries from the old slash-separated names to # the portable dash-separated ones. Dry-run unless --apply. Additive: it @@ -85,6 +93,8 @@ CONTEXT="" REGION="${AWS_REGION:-${AWS_DEFAULT_REGION:-}}" PROJECT="" APPLY="false" +KEYS="" # migrate: explicit managed-store keys; empty = derive from the cluster +KEYS_SET="false" # migrate: was --keys given at all, vs. given but empty STORE="" # aws|gcp|openbao; defaults to $CLOUD, i.e. that cloud's managed store COMMAND="${1:-}" [ $# -gt 0 ] && shift @@ -97,6 +107,7 @@ while [ $# -gt 0 ]; do --project) PROJECT="$2"; shift 2 ;; --store) STORE="$2"; shift 2 ;; --apply) APPLY="true"; shift ;; + --keys) KEYS="$2"; KEYS_SET="true"; shift 2 ;; *) echo "unknown argument: $1" >&2; exit 2 ;; esac done @@ -465,6 +476,27 @@ migrate_source_keys() { sort -u } +# The source keys `migrate` walks: an explicit --keys list when given, else the +# keys this cluster's ExternalSecrets ask for. Commas or whitespace separate the +# list; blanks and duplicates are dropped. +# +# --keys given but resolving to zero keys (empty, all blank, or an unset +# variable substituted in) is refused, not silently treated as "no --keys" -- +# see the header comment for why. +migrate_keys() { + if [ "$KEYS_SET" = "true" ]; then + local keys + keys="$(printf '%s\n' "$KEYS" | tr ', ' '\n\n' | { grep -v '^$' || true; } | sort -u)" + if [ -z "$keys" ]; then + echo "--keys was given but names no key -- refusing rather than falling back to the cluster's ExternalSecrets" >&2 + return 2 + fi + printf '%s\n' "$keys" + else + migrate_source_keys + fi +} + # Copy every mapped key from the cloud managed store into OpenBao. # # Additive and idempotent: a destination that already exists is left alone and @@ -476,6 +508,8 @@ cmd_migrate() { exit 2 } local copied=0 exists=0 skipped=0 absent=0 key target payload + local keys + keys="$(migrate_keys)" || exit 2 printf '%-58s %-46s %s\n' "SOURCE KEY" "OPENBAO PATH" "ACTION" while IFS= read -r key; do @@ -510,7 +544,7 @@ cmd_migrate() { printf '%-58s %-46s %s\n' "$key" "$target" "would copy" fi copied=$((copied + 1)) - done <<<"$(migrate_source_keys)" + done <<<"$keys" echo echo "copied: ${copied}, exists: ${exists}, absent: ${absent}, skipped: ${skipped}" From bd5ddd37d4475b0a4bb432428afb9e8d1240a30e Mon Sep 17 00:00:00 2001 From: Smana Date: Tue, 29 Sep 2026 10:26:02 +0200 Subject: [PATCH 6/8] feat(openbao): an explicit switch lets a new GCP lineage boot beside a foreign-sealed mirror Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011xuNsQcGpQPcEH8m1h3rDD --- scripts/ci/tests/test-openbao-new-lineage.sh | 206 ++++++++++++++++++ scripts/provision/openbao-config.sh | 133 +++++++++++ .../guides/openbao-cross-cloud-failover.md | 41 ++++ 3 files changed, 380 insertions(+) create mode 100644 scripts/ci/tests/test-openbao-new-lineage.sh diff --git a/scripts/ci/tests/test-openbao-new-lineage.sh b/scripts/ci/tests/test-openbao-new-lineage.sh new file mode 100644 index 000000000..04f96ede0 --- /dev/null +++ b/scripts/ci/tests/test-openbao-new-lineage.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2034 +# (file-wide: CLOUD and SNAPSHOT_BUCKET are read by the body of +# latest_unsealed_snapshot(), which is eval'd in from the script under test, so +# static analysis cannot see those uses. Same reason as +# test-openbao-fallback-address.sh.) +# +# May OPENBAO_NEW_LINEAGE=true start a NEW lineage? And is the switch wired where +# it has to be? +# +# WHY THIS MATTERS. GCP's snapshot bucket also holds the AWS mirror, whose every +# object is AWS-sealed. A gcpckms node that finds no object under its own seal +# has no path in: rehydrate refuses the foreign seal, and -- with +# OPENBAO_SNAPSHOT_SKIP_FOREIGN_SEAL=true -- refuses a plain init, correctly, +# because that overwrites a lineage's stored keys on a guess. The switch is the +# operator saying "start this lineage" out loud. Getting its rule wrong in the +# permissive direction discards a lineage's history; in the strict direction a +# GCP-only platform cannot boot at all. +# +# new_lineage_verdict() is lifted out of the script rather than restated, so this +# tests the code that ships. Everything around it in rehydrate needs a live node. +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +fail=0 +check() { + if [ "$2" = "$3" ]; then printf ' ok %s\n' "$1" + else printf ' FAIL %s: expected %q got %q\n' "$1" "$2" "$3"; fail=1; fi +} +contains() { + if printf '%s' "$2" | grep -qF -- "$3"; then printf ' ok %s\n' "$1" + else printf ' FAIL %s: %q not found\n' "$1" "$3"; fail=1; fi +} +absent() { + if printf '%s' "$2" | grep -qF -- "$3"; then printf ' FAIL %s: %q unexpectedly present\n' "$1" "$3"; fail=1 + else printf ' ok %s\n' "$1"; fi +} +for _h in check contains absent; do + declare -F "$_h" >/dev/null || { echo "harness incomplete: $_h() is not defined" >&2; exit 2; } +done + +SRC="${OPENBAO_CONFIG_SCRIPT:-$HERE/../../provision/openbao-config.sh}" +body="$(sed -n '/^new_lineage_verdict() {/,/^}/p' "$SRC")" +[ -n "$body" ] || { echo "could not extract new_lineage_verdict() from $SRC" >&2; exit 1; } +eval "$body" + +echo "== the case the switch exists for: a GCP node beside an AWS-only mirror" +check "proceeds" "proceed" "$(new_lineage_verdict gcpckms awskms "" "" "")" + +echo "== an object under this node's seal exists: restore it, never start over" +check "refuses" "refuse-own-seal-exists" \ + "$(new_lineage_verdict gcpckms awskms "2026-09-01T000000Z-gcpckms.snap" "" "")" + +echo "== the newest object is already restorable here" +check "refuses" "refuse-same-seal" \ + "$(new_lineage_verdict gcpckms gcpckms "2026-09-01T000000Z-gcpckms.snap" "" "")" + +echo "== a named object contradicts a new lineage" +check "refuses" "refuse-named-key" \ + "$(new_lineage_verdict gcpckms awskms "" "2026-09-05T092947Z-awskms.snap" "")" + +echo "== a legacy object with no seal segment: its seal is UNKNOWN, not 'none' -- refuse" +check "refuses" "refuse-unsealed-object" \ + "$(new_lineage_verdict gcpckms "" "" "" "2026-09-02T041500Z.snap")" + +echo "== the newest object is a foreign mirror, but an older legacy object's seal is unknown too" +check "refuses" "refuse-unsealed-object" \ + "$(new_lineage_verdict gcpckms awskms "" "" "2026-09-02T041500Z.snap")" + +echo "== the newest object itself has no parseable seal (N2): unknown, not none -- refuse" +check "refuses" "refuse-unsealed-object" \ + "$(new_lineage_verdict gcpckms "" "" "" "")" + +echo "== wiring inside rehydrate_openbao" +fn="$(sed -n '/^rehydrate_openbao() {/,/^}/p' "$SRC")" +contains "reads the switch" "$fn" 'OPENBAO_NEW_LINEAGE:-false' +contains "asks the verdict with the node's own-seal listing" "$fn" \ + 'new_lineage_verdict "$node_seal" "$snap_seal" "$own_latest"' +contains "the call site never combines with a named OPENBAO_SNAPSHOT_KEY" "$fn" \ + 'new_lineage_verdict "$node_seal" "$snap_seal" "$own_latest" "${OPENBAO_SNAPSHOT_KEY:-}"' +contains "the call site passes the legacy (unsealed) listing as the 5th argument" "$fn" \ + 'new_lineage_verdict "$node_seal" "$snap_seal" "$own_latest" "${OPENBAO_SNAPSHOT_KEY:-}" "$unsealed_latest"' +contains "unsealed_latest is computed via latest_unsealed_snapshot" "$fn" \ + 'unsealed_latest=$(latest_unsealed_snapshot)' +absent "latest_snapshot_sealed \"\" no longer appears (N2: it only caught the legacy shape)" "$fn" \ + 'latest_snapshot_sealed ""' +arm="$(printf '%s\n' "$fn" | sed -n '/^[[:space:]]*proceed)/,/;;/p')" +contains "the proceed arm initialises" "$arm" 'init_openbao' +contains "the proceed arm returns before falling into the seal gate" "$arm" 'return 0' +for v in refuse-named-key refuse-same-seal refuse-own-seal-exists refuse-unsealed-object; do + varm="$(printf '%s\n' "$fn" | sed -n "/^[[:space:]]*${v})/,/;;/p")" + contains "the ${v} arm exits non-zero" "$varm" 'exit 1' +done +switch_line=$(printf '%s\n' "$fn" | grep -nF 'OPENBAO_NEW_LINEAGE:-false' | head -1 | cut -d: -f1) +gate_line=$(printf '%s\n' "$fn" | grep -nF '"${OPENBAO_SNAPSHOT_SKIP_FOREIGN_SEAL:-false}" != "true"' | head -1 | cut -d: -f1) +if [ -n "$switch_line" ] && [ -n "$gate_line" ] && [ "$switch_line" -lt "$gate_line" ]; then + printf ' ok the switch is consulted before the foreign-seal refusal\n' +else + printf ' FAIL the switch must be consulted before the foreign-seal refusal (switch=%s gate=%s)\n' "$switch_line" "$gate_line"; fail=1 +fi +cannot_list_line=$(printf '%s\n' "$fn" | grep -nF 'Refusing to initialise: cannot prove' | head -1 | cut -d: -f1) +if [ -n "$switch_line" ] && [ -n "$cannot_list_line" ] && [ "$cannot_list_line" -lt "$switch_line" ]; then + printf ' ok the switch comes after the cannot-list refusal\n' +else + printf ' FAIL the switch must come after the cannot-list refusal (cannot_list=%s switch=%s)\n' "$cannot_list_line" "$switch_line"; fail=1 +fi +recovery_preflight_line=$(printf '%s\n' "$fn" | grep -nF 'secret_read "$RECOVERY_KEYS_SECRET_NAME"' | head -1 | cut -d: -f1) +if [ -n "$switch_line" ] && [ -n "$recovery_preflight_line" ] && [ "$recovery_preflight_line" -lt "$switch_line" ]; then + printf ' ok the switch comes after the recovery-keys pre-flight\n' +else + printf ' FAIL the switch must come after the recovery-keys pre-flight (preflight=%s switch=%s)\n' "$recovery_preflight_line" "$switch_line"; fail=1 +fi + +echo "== latest_unsealed_snapshot(): what the switch's pre-flight actually lists" +# N1: the I1 fix (a legacy/manual snapshot blocking a new lineage) changed THIS +# function. Nothing exercised it directly -- the verdict tests above are all +# pure-function tests that never call it, so reverting its filter, or passing +# the wrong argument at the call site, left every test green. Lifted the same +# way new_lineage_verdict() is: sed out the function (and the SNAP_NAME_RE it +# depends on) from the script that ships, eval, drive it directly. +lat_body="$(sed -n '/^latest_unsealed_snapshot() {/,/^}/p' "$SRC")" +[ -n "$lat_body" ] || { echo "could not extract latest_unsealed_snapshot() from $SRC" >&2; exit 1; } +snap_re_line="$(grep -E '^SNAP_NAME_RE=' "$SRC" | head -1)" +[ -n "$snap_re_line" ] || { echo "could not extract SNAP_NAME_RE from $SRC" >&2; exit 1; } +log_err() { :; } # the function under test calls this on a listing failure; silence it +eval "$snap_re_line" +eval "$lat_body" +declare -f latest_unsealed_snapshot >/dev/null || { + echo "latest_unsealed_snapshot did not define after eval" >&2; exit 1; } + +echo "-- GCP branch (gcp_gcloud storage ls gs://bucket/, non-recursive) --" +CLOUD="gcp" +SNAPSHOT_BUCKET="test" +GCLOUD_RC=0 +GCLOUD_LISTING='' +gcp_gcloud() { + if [ "$GCLOUD_RC" != 0 ]; then return 1; fi + printf '%s\n' "$GCLOUD_LISTING" +} + +GCLOUD_LISTING='gs://test/2026-09-05T092947Z-awskms.snap' +check "only sealed (AWS-mirror) objects present: nothing unsealed" "" "$(latest_unsealed_snapshot)" + +GCLOUD_LISTING=$'gs://test/2026-09-05T092947Z-awskms.snap\ngs://test/README.md' +check "a non-.snap top-level object is not a snapshot, so it never blocks" "" "$(latest_unsealed_snapshot)" + +GCLOUD_LISTING=$'gs://test/2026-09-05T092947Z-awskms.snap\ngs://test/manual-backup.snap' +check "a hand-named object (manual-backup.snap) counts as unsealed" \ + "manual-backup.snap" "$(latest_unsealed_snapshot)" + +GCLOUD_LISTING=$'gs://test/2026-09-05T092947Z-awskms.snap\ngs://test/2026-09-02T041500Z.snap' +check "a legacy .snap object (no seal segment) counts as unsealed" \ + "2026-09-02T041500Z.snap" "$(latest_unsealed_snapshot)" + +GCLOUD_LISTING='gs://test/2026-09-05T092947Z-gcpckms.snap' +check "a SEALED name is not \"unsealed\"" "" "$(latest_unsealed_snapshot)" + +# A real non-recursive `gcloud storage ls` never prints the nested key under a +# prefix -- only the prefix marker itself (a trailing-slash "directory" line). +# So "moved aside" is invisible here by construction, the same way it already +# is for latest_snapshot_sealed(); the fixture below is what gcloud would +# actually emit for a bucket holding both a top-level mirror object and +# something moved aside under aside/. +GCLOUD_LISTING=$'gs://test/2026-09-05T092947Z-awskms.snap\ngs://test/aside/' +check "an object moved aside under a prefix is invisible (top-level only)" \ + "" "$(latest_unsealed_snapshot)" + +GCLOUD_RC=1 +latest_unsealed_snapshot >/dev/null; rc=$? +check "a GCP listing failure returns non-zero, not an empty (selectable) result" "1" "$rc" +GCLOUD_RC=0 + +echo "-- AWS branch (s3api list-objects-v2, newest by LastModified) --" +CLOUD="aws" +AWS_RC=0 +AWS_JSON='' +fixture_aws_cmd() { + if [ "$AWS_RC" != 0 ]; then return 1; fi + printf '%s' "$AWS_JSON" +} +get_aws_cmd() { printf 'fixture_aws_cmd'; } + +# Newest by LastModified must win, NOT lexical order (2026-09-06... sorts +# before manual-backup.snap lexically) -- and the prefixed key must be +# excluded even though it is the most recently modified of all four. +AWS_JSON='{"Contents":[ + {"Key":"2026-09-05T092947Z-awskms.snap","LastModified":"2026-09-05T09:29:47.000Z"}, + {"Key":"manual-backup.snap","LastModified":"2026-09-01T00:00:00.000Z"}, + {"Key":"2026-09-06T041500Z.snap","LastModified":"2026-09-06T04:15:00.000Z"}, + {"Key":"aside/2026-09-02T041500Z.snap","LastModified":"2026-09-07T00:00:00.000Z"}, + {"Key":"README.md","LastModified":"2026-09-08T00:00:00.000Z"} +]}' +check "the newest UNSEALED key by LastModified wins; sealed and prefixed keys excluded" \ + "2026-09-06T041500Z.snap" "$(latest_unsealed_snapshot)" + +AWS_RC=1 +latest_unsealed_snapshot >/dev/null; rc=$? +check "an AWS listing failure returns non-zero too" "1" "$rc" +AWS_RC=0 + +echo "== documented" +usage_fn="$(sed -n '/^usage() {/,/^}/p' "$SRC")" +contains "usage names the switch" "$usage_fn" 'OPENBAO_NEW_LINEAGE=true' + +echo +if [ "$fail" -eq 0 ]; then echo "all checks passed"; else echo "FAILURES"; fi +exit "$fail" diff --git a/scripts/provision/openbao-config.sh b/scripts/provision/openbao-config.sh index 0cfee7edf..47c30fe83 100755 --- a/scripts/provision/openbao-config.sh +++ b/scripts/provision/openbao-config.sh @@ -102,6 +102,15 @@ usage() { echo " skipping a newer snapshot discards every write" echo " after it. Only for a failback, where the" echo " foreign-sealed objects are not coming back." + echo " OPENBAO_NEW_LINEAGE=true Let 'rehydrate' start a NEW lineage (a plain init) on a" + echo " node whose seal no top-level object in the bucket carries" + echo " (objects moved aside under a prefix are not examined) --" + echo " the first boot of a GCP-only lineage beside the AWS mirror." + echo " Refused whenever an object under this node's seal exists," + echo " together with OPENBAO_SNAPSHOT_KEY, or when any top-level" + echo " snapshot's seal is unknown (a name with no '-'" + echo " segment -- legacy or otherwise)." + echo " REPLACES the stored root token and recovery keys." echo "" echo "Example:" echo " $0 init --url https://openbao:8200 --root-token-secret-name openbao/root-token \\" @@ -749,6 +758,68 @@ latest_snapshot_sealed() { fi } +# Newest TOP-LEVEL object whose name does NOT match SNAP_NAME_RE (the +# "-.snap" shape) -- covers both a legacy ".snap" and +# an arbitrary name like "manual-backup.snap". Its seal cannot be read from the +# name, so it is UNKNOWN, not "none": one of these might be this node's seal, +# and initialising over it would overwrite the lineage's stored keys on a +# guess. Same three-state contract as latest_snapshot_sealed(): stdout empty = +# none, return 1 = the listing itself failed. Top-level only, mirroring +# latest_snapshot()/latest_snapshot_sealed() -- an object moved aside under a +# prefix is invisible here too, the same "moved aside" semantics the rest of +# rehydrate uses. +latest_unsealed_snapshot() { + if [ "$CLOUD" = "gcp" ]; then + local listing + if ! listing=$(gcp_gcloud storage ls "gs://${SNAPSHOT_BUCKET}/"); then + log_err "could not list gs://${SNAPSHOT_BUCKET} -- a listing failure, NOT an empty bucket." + return 1 + fi + printf '%s\n' "$listing" | sed 's#.*/##' | { grep -Ev "$SNAP_NAME_RE" || true; } | { grep -E '\.snap$' || true; } | sort | tail -n1 + else + local aws_cmd; aws_cmd=$(get_aws_cmd) + local out + if ! out=$($aws_cmd s3api list-objects-v2 --bucket "$SNAPSHOT_BUCKET" --output json); then + log_err "could not list s3://${SNAPSHOT_BUCKET} -- a listing failure, NOT an empty bucket." + return 1 + fi + printf '%s' "$out" | jq -r --arg re "$SNAP_NAME_RE" ' + (.Contents // []) + | map(select((.Key | test("^[^/]+$")) and (.Key | test($re) | not) and (.Key | test("\\.snap$")))) + | if length == 0 then "" else (sort_by(.LastModified) | last | .Key) end' + fi +} + +# May OPENBAO_NEW_LINEAGE=true start a NEW lineage on this node? A pure decision, +# lifted into scripts/test-openbao-new-lineage.sh, so the rule is tested without a +# live node. +# +# $1 this node's seal type $2 the newest object's seal segment (may be empty) +# $3 the newest object carrying $1's seal, or empty when none does +# $4 OPENBAO_SNAPSHOT_KEY, empty when unset +# $5 the newest top-level object whose name carries NO seal segment at all +# (latest_unsealed_snapshot -- legacy ".snap" or an arbitrary name like +# "manual-backup.snap"), empty when none exists +# +# Prints exactly one verdict, checked in this order: +# refuse-named-key a named restore was asked for; a new lineage contradicts it +# refuse-same-seal the newest object IS restorable here; restore it instead +# refuse-own-seal-exists an older object carries this seal -- this lineage's own +# history; restore it with OPENBAO_SNAPSHOT_SKIP_FOREIGN_SEAL +# refuse-unsealed-object some top-level object's seal is UNKNOWN, not "none" -- either +# $5 is non-empty, or the newest object itself ($2) has no +# parseable seal; it might be this node's seal; retag and re-run +# proceed nothing in the bucket carries this node's seal, and no +# top-level object has an unknown seal: init +new_lineage_verdict() { + local node_seal=$1 snap_seal=$2 own_latest=$3 snapshot_key=$4 unsealed_latest=$5 + if [ -n "$snapshot_key" ]; then printf 'refuse-named-key'; return 0; fi + if [ "$snap_seal" = "$node_seal" ]; then printf 'refuse-same-seal'; return 0; fi + if [ -n "$own_latest" ]; then printf 'refuse-own-seal-exists'; return 0; fi + if [ -n "$unsealed_latest" ] || [ -z "$snap_seal" ]; then printf 'refuse-unsealed-object'; return 0; fi + printf 'proceed' +} + # The PKI mount's CA endpoint is unauthenticated, which makes it the one thing a # rehydrate can assert without a token: if it answers with a certificate that # chains to the CA we already trust, the barrier unwrapped and the mount came @@ -1097,6 +1168,68 @@ rehydrate_openbao() { snap_seal=$(snapshot_seal_segment "$latest") log_message "INFO" "This node's seal is '${node_seal}'; ${latest} carries '${snap_seal:-none}'." + # THE NEW-LINEAGE SWITCH (docs/superpowers/specs/2026-09-11-openbao-stage2-gcp-design.md). + # A GCP-sealed node whose bucket holds only AWS-mirrored objects has no path + # in: the gate below refuses the foreign seal, and with the skip it refuses a + # plain init -- correctly, since that overwrites a lineage's stored keys on a + # guess. OPENBAO_NEW_LINEAGE=true is the operator saying "start this lineage" + # out loud, and it is honoured ONLY when nothing in the bucket carries this + # node's seal. The moved-aside (rc 2) and could-not-list refusals above have + # already exited before this point, so the switch can never bypass them. + if [ "${OPENBAO_NEW_LINEAGE:-false}" = "true" ]; then + local own_latest unsealed_latest verdict + if ! own_latest=$(latest_snapshot_sealed "$node_seal"); then + log_message "ERROR" "OPENBAO_NEW_LINEAGE=true, but ${SNAPSHOT_BUCKET} cannot be listed for '${node_seal}' objects." + log_message "ERROR" "A new lineage is only safe once the bucket is PROVEN to hold none. Nothing has changed yet." + exit 1 + fi + if ! unsealed_latest=$(latest_unsealed_snapshot); then + log_message "ERROR" "OPENBAO_NEW_LINEAGE=true, but ${SNAPSHOT_BUCKET} cannot be listed for top-level objects" + log_message "ERROR" "with no seal segment (legacy or otherwise)." + log_message "ERROR" "A new lineage is only safe once the bucket is PROVEN to hold none. Nothing has changed yet." + exit 1 + fi + verdict=$(new_lineage_verdict "$node_seal" "$snap_seal" "$own_latest" "${OPENBAO_SNAPSHOT_KEY:-}" "$unsealed_latest") + case "$verdict" in + proceed) + log_message "WARN" "OPENBAO_NEW_LINEAGE=true -- STARTING A NEW '${node_seal}' LINEAGE." + log_message "WARN" " newest object : ${latest} (sealed '${snap_seal:-none}') -- NOT restored" + log_message "WARN" " no top-level '-${node_seal}' object in ${SNAPSHOT_BUCKET} (objects moved aside under a prefix are NOT examined)" + log_message "WARN" " ${ROOT_TOKEN_SECRET_NAME} and ${RECOVERY_KEYS_SECRET_NAME} are REPLACED with this node's new keys." + if [ "$CLOUD" = "gcp" ]; then + log_message "WARN" " those two entries are also where the awskms standby reads its pre-copied AWS keys -- re-copy them before relying on that standby. The previous versions remain in Secret Manager." + fi + init_openbao + return 0 ;; + refuse-named-key) + log_message "ERROR" "OPENBAO_NEW_LINEAGE=true and OPENBAO_SNAPSHOT_KEY=${OPENBAO_SNAPSHOT_KEY:-} contradict each other:" + log_message "ERROR" "one asks for a new lineage, the other for a named restore. Unset one. Nothing has changed yet." + exit 1 ;; + refuse-same-seal) + log_message "ERROR" "OPENBAO_NEW_LINEAGE=true, but the newest object ${latest} carries this node's seal" + log_message "ERROR" "'${node_seal}' and can be restored. A new lineage would discard it. Unset" + log_message "ERROR" "OPENBAO_NEW_LINEAGE and re-run. Nothing has changed yet." + exit 1 ;; + refuse-own-seal-exists) + log_message "ERROR" "OPENBAO_NEW_LINEAGE=true, but ${own_latest} carries this node's seal '${node_seal}':" + log_message "ERROR" "this lineage's own history. Restore it instead -- unset OPENBAO_NEW_LINEAGE and re-run" + log_message "ERROR" "with OPENBAO_SNAPSHOT_SKIP_FOREIGN_SEAL=true. Nothing has changed yet." + exit 1 ;; + refuse-unsealed-object) + log_message "ERROR" "OPENBAO_NEW_LINEAGE=true, but ${unsealed_latest:-$latest} carries NO seal segment: its seal is" + log_message "ERROR" "UNKNOWN, not 'none' -- it may be this node's seal '${node_seal}'. Confirm the seal," + log_message "ERROR" "then retag it to -.snap, or move it to ANOTHER bucket, then re-run." + log_message "ERROR" "Do NOT move it under a prefix in THIS bucket: that only hides it from this check --" + log_message "ERROR" "the AWS-mirror objects stay at the top level, so a prefix does not trip the" + log_message "ERROR" "moved-aside refusal -- and the switch would then start a new lineage over it." + log_message "ERROR" "Nothing has changed yet." + exit 1 ;; + *) + log_message "ERROR" "new_lineage_verdict returned '${verdict}' -- refusing. Nothing has changed yet." + exit 1 ;; + esac + fi + if [ "$snap_seal" != "$node_seal" ]; then local found if [ -n "$snap_seal" ]; then diff --git a/website/content/docs/guides/openbao-cross-cloud-failover.md b/website/content/docs/guides/openbao-cross-cloud-failover.md index d6fb2c7a8..2fea0e0f6 100644 --- a/website/content/docs/guides/openbao-cross-cloud-failover.md +++ b/website/content/docs/guides/openbao-cross-cloud-failover.md @@ -390,6 +390,47 @@ the newest object in the AWS history. 5. Revert the `openbao_target_ip` / overlay changes from step 4 of the failover. +## Starting a GCP-only lineage + +GCP's snapshot bucket also holds the AWS mirror, and every mirrored object is +AWS-sealed. A `gcpckms` node that finds no object under its own seal therefore +has no way in. `rehydrate` refuses the foreign seal, and even with +`OPENBAO_SNAPSHOT_SKIP_FOREIGN_SEAL=true` it refuses to initialise, because that +would overwrite a lineage's stored keys on a guess. The first boot of a GCP-only +lineage says so out loud instead: + +```bash +OPENBAO_NEW_LINEAGE=true TM_CLOUD=gcp \ + terramate -C opentofu/gcp/openbao/management script run deploy +``` + +The command above presumes the GCP cluster stack is already up, typically from +a full deploy that stopped at the seal refusal. The switch also sits behind the +recovery-keys pre-flight: on a project where `openbao-priv-gcp-recovery-keys` +has no readable version, the switch is unreachable and `rehydrate` refuses +before it ever reads `OPENBAO_NEW_LINEAGE`. + +The switch has five limits: + +- It is honoured only when no top-level object carries the node's own seal, and + every top-level snapshot's name carries a `-` segment. A snapshot whose + name has no seal segment (a legacy `.snap` or a hand-named one) is + refused, because its seal is unknown. +- Snapshots moved aside under a prefix are **not examined**. Never move one + under a prefix in this bucket to get past the refusal, because that only + hides it from the check. Retag it to `-.snap`, or move it to + another bucket. +- It is never honoured together with `OPENBAO_SNAPSHOT_KEY`. +- It **replaces** the stored root token and recovery keys. +- These are the same two entries the `awskms` standby reads its pre-copied AWS + keys from. After a GCP-only run, re-copy the AWS lineage's keys before + relying on the standby. The previous versions remain in Secret Manager + (`gcloud secrets versions list openbao-priv-gcp-recovery-keys --project ogenki-435905`). + +Every later boot restores the newest `-gcpckms` object. When a mirrored AWS +object is newer, set `OPENBAO_SNAPSHOT_SKIP_FOREIGN_SEAL=true`, as the refusal +message says. + ## Drill record Every executed failover is recorded with the snapshot object, the measured RPO, From ab0dc39986c83f39c500c0485c8339215d3b9a4c Mon Sep 17 00:00:00 2001 From: Smana Date: Tue, 29 Sep 2026 10:43:02 +0200 Subject: [PATCH 7/8] fix(openbao): name the break-glass replace step, drop dead citations, guard shared-policy parity A new-lineage init leaves vault_generic_endpoint.admin_user's disable_read=true hiding the fact that no apply will ever recreate the break-glass user again -- the proceed arm and the failover guide now both name the required tofu apply -replace=module.store_of_record.vault_generic_endpoint.admin_user. Also: drop two citations to a design doc and an ADR that do not apply here (salvage provenance is ac62abf2 instead), and add a cmp guard so the shared module's five policy files can never silently drift from AWS's copies. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011xuNsQcGpQPcEH8m1h3rDD --- .../gcp/openbao/management/store-of-record.tf | 6 +-- scripts/ci/tests/test-openbao-new-lineage.sh | 2 + .../tests/test-validate-openbao-policies.sh | 11 +++++ scripts/ci/validate-openbao-policies.sh | 47 ++++++++++++++++++- scripts/provision/openbao-config.sh | 4 +- .../guides/openbao-cross-cloud-failover.md | 17 +++++++ 6 files changed, 81 insertions(+), 6 deletions(-) diff --git a/opentofu/gcp/openbao/management/store-of-record.tf b/opentofu/gcp/openbao/management/store-of-record.tf index df24486c5..866fc62dd 100644 --- a/opentofu/gcp/openbao/management/store-of-record.tf +++ b/opentofu/gcp/openbao/management/store-of-record.tf @@ -1,6 +1,6 @@ -# Stage 2 on GCP (docs/superpowers/specs/2026-09-11-openbao-stage2-gcp-design.md, -# ADR-0037): the same mounts, policies and logins AWS defines, from the shared -# module. Until this existed, jwt/gcp-0's external-secrets role named a policy +# Stage 2 on GCP (salvaged from ac62abf2): the same mounts, policies and logins +# AWS defines, from the shared module. Until this existed, jwt/gcp-0's +# external-secrets role named a policy # no GCP OpenBao ever had, and the 14 OpenBao-backed ExternalSecrets on gcp-0 # authenticated and then read nothing. diff --git a/scripts/ci/tests/test-openbao-new-lineage.sh b/scripts/ci/tests/test-openbao-new-lineage.sh index 04f96ede0..c0cb24e90 100644 --- a/scripts/ci/tests/test-openbao-new-lineage.sh +++ b/scripts/ci/tests/test-openbao-new-lineage.sh @@ -86,6 +86,8 @@ absent "latest_snapshot_sealed \"\" no longer appears (N2: it only caught the le arm="$(printf '%s\n' "$fn" | sed -n '/^[[:space:]]*proceed)/,/;;/p')" contains "the proceed arm initialises" "$arm" 'init_openbao' contains "the proceed arm returns before falling into the seal gate" "$arm" 'return 0' +contains "the proceed arm names the break-glass replace address (disable_read hides the missing user otherwise)" "$arm" \ + 'tofu apply -replace=module.store_of_record.vault_generic_endpoint.admin_user' for v in refuse-named-key refuse-same-seal refuse-own-seal-exists refuse-unsealed-object; do varm="$(printf '%s\n' "$fn" | sed -n "/^[[:space:]]*${v})/,/;;/p")" contains "the ${v} arm exits non-zero" "$varm" 'exit 1' diff --git a/scripts/ci/tests/test-validate-openbao-policies.sh b/scripts/ci/tests/test-validate-openbao-policies.sh index 70cf91b84..9d1deb261 100755 --- a/scripts/ci/tests/test-validate-openbao-policies.sh +++ b/scripts/ci/tests/test-validate-openbao-policies.sh @@ -115,6 +115,17 @@ resource "vault_jwt_auth_backend_role" "r" { EOF expect "a literal token_policies list on a role is a reference too" 1 "$r" 'policy "undefined-x"' +r="$t/policy-diverge"; role_policies "$r" aws '"default"' +mkdir -p "$r/opentofu/aws/openbao/management/policies" "$r/opentofu/shared/modules/openbao-store-of-record/policies" +for name in admin app-prefix external-secrets pki-admin secrets-admin; do + echo "path \"x\" { capabilities = [\"read\"] }" >"$r/opentofu/aws/openbao/management/policies/${name}.hcl" + cp "$r/opentofu/aws/openbao/management/policies/${name}.hcl" "$r/opentofu/shared/modules/openbao-store-of-record/policies/${name}.hcl" +done +expect "identical shared policy copies pass" 0 "$r" +echo "path \"y\" { capabilities = [\"read\"] }" >"$r/opentofu/shared/modules/openbao-store-of-record/policies/admin.hcl" +expect "a diverged shared policy copy fails, naming the file" 1 "$r" \ + 'opentofu/aws/openbao/management/policies/admin.hcl and opentofu/shared/modules/openbao-store-of-record/policies/admin.hcl have diverged' + expect "the repository itself passes" 0 "$(cd "${SCRIPT_DIR}/../../.." && pwd)" if [ "$failures" -ne 0 ]; then echo "==> ${failures} failure(s)"; exit 1; fi diff --git a/scripts/ci/validate-openbao-policies.sh b/scripts/ci/validate-openbao-policies.sh index fb9d973b1..ddf2e6baf 100755 --- a/scripts/ci/validate-openbao-policies.sh +++ b/scripts/ci/validate-openbao-policies.sh @@ -23,14 +23,28 @@ # opentofu//*/configure/openbao.tf, so a role moved to a sibling file # goes unchecked; a `vault_policy` gated to `count = 0` still counts as # defined; and a policy name built from a variable, not a literal string, is -# not resolved either way. +# not resolved either way. `called_module_dirs()` resolves only a one-level +# `./`/`../` local module source -- a nested module (one local module calling +# another) or a registry source is read as "not defined" and the run fails +# loudly on a policy that in fact exists. That is a false failure, never a +# false pass, so it is safe by construction; widen it if a real layout needs it. +# +# A second, unrelated check runs after policy parity: the five policy files +# AWS's management stack and the shared openbao-store-of-record module both +# ship (admin, app-prefix, external-secrets, pki-admin, secrets-admin) must stay +# byte-identical. The module is GCP's copy of AWS's policies, kept as a second +# file rather than a shared source (D1, 2026-09-29 GCP parity design) so the AWS +# stack never depends on GCP's directory; nothing else enforces that the two +# copies do not quietly drift apart. A DELIBERATE divergence (for example G-5's +# agents/ grant) must land in both copies in the same change, not just one. # # Usage: validate-openbao-policies.sh [ROOT_DIR] set -euo pipefail ROOT="${1:-$(git rev-parse --show-toplevel)}" -exec python3 - "$ROOT" <<'PY' +python_rc=0 +python3 - "$ROOT" <<'PY' || python_rc=$? import pathlib import re import sys @@ -94,3 +108,32 @@ if fails: sys.exit(1) print(f"==> OpenBao policy parity: every policy a JWT role names is defined ({checked} cloud(s)).") PY + +# The five policy files the shared module copies from AWS's management stack +# (see SCOPE EDGES above) must stay byte-identical -- only checked when BOTH +# fixed directories exist, so fixtures that do not model this layout are unaffected. +AWS_POLICIES="$ROOT/opentofu/aws/openbao/management/policies" +SHARED_POLICIES="$ROOT/opentofu/shared/modules/openbao-store-of-record/policies" +policy_rc=0 +if [ -d "$AWS_POLICIES" ] && [ -d "$SHARED_POLICIES" ]; then + diverged=0 + for name in admin app-prefix external-secrets pki-admin secrets-admin; do + aws_file="$AWS_POLICIES/$name.hcl" + shared_file="$SHARED_POLICIES/$name.hcl" + if [ -f "$aws_file" ] && [ -f "$shared_file" ] && ! cmp -s "$aws_file" "$shared_file"; then + echo "FAIL: opentofu/aws/openbao/management/policies/${name}.hcl and opentofu/shared/modules/openbao-store-of-record/policies/${name}.hcl have diverged -- a deliberate divergence (e.g. G-5's agents/ grant) must land in both copies" + diverged=1 + fi + done + if [ "$diverged" -ne 0 ]; then + echo "==> shared policy parity: the copies diverged." + policy_rc=1 + else + echo "==> shared policy parity: AWS and the shared module's copies are byte-identical (5 file(s))." + fi +fi + +if [ "$python_rc" -ne 0 ] || [ "$policy_rc" -ne 0 ]; then + exit 1 +fi +exit 0 diff --git a/scripts/provision/openbao-config.sh b/scripts/provision/openbao-config.sh index 47c30fe83..a736a7da0 100755 --- a/scripts/provision/openbao-config.sh +++ b/scripts/provision/openbao-config.sh @@ -1168,7 +1168,7 @@ rehydrate_openbao() { snap_seal=$(snapshot_seal_segment "$latest") log_message "INFO" "This node's seal is '${node_seal}'; ${latest} carries '${snap_seal:-none}'." - # THE NEW-LINEAGE SWITCH (docs/superpowers/specs/2026-09-11-openbao-stage2-gcp-design.md). + # THE NEW-LINEAGE SWITCH (salvaged from ac62abf2). # A GCP-sealed node whose bucket holds only AWS-mirrored objects has no path # in: the gate below refuses the foreign seal, and with the skip it refuses a # plain init -- correctly, since that overwrites a lineage's stored keys on a @@ -1198,6 +1198,8 @@ rehydrate_openbao() { log_message "WARN" " ${ROOT_TOKEN_SECRET_NAME} and ${RECOVERY_KEYS_SECRET_NAME} are REPLACED with this node's new keys." if [ "$CLOUD" = "gcp" ]; then log_message "WARN" " those two entries are also where the awskms standby reads its pre-copied AWS keys -- re-copy them before relying on that standby. The previous versions remain in Secret Manager." + log_message "WARN" " NEXT STEP: the new OpenBao has no break-glass user yet. vault_generic_endpoint.admin_user has disable_read=true, so a plain apply sees no drift and never recreates it. Run:" + log_message "WARN" " tofu apply -replace=module.store_of_record.vault_generic_endpoint.admin_user (on the GCP management stack)" fi init_openbao return 0 ;; diff --git a/website/content/docs/guides/openbao-cross-cloud-failover.md b/website/content/docs/guides/openbao-cross-cloud-failover.md index 2fea0e0f6..2b60d0cf2 100644 --- a/website/content/docs/guides/openbao-cross-cloud-failover.md +++ b/website/content/docs/guides/openbao-cross-cloud-failover.md @@ -410,6 +410,23 @@ recovery-keys pre-flight: on a project where `openbao-priv-gcp-recovery-keys` has no readable version, the switch is unreachable and `rehydrate` refuses before it ever reads `OPENBAO_NEW_LINEAGE`. +**Immediately after the proceed arm prints its warning, force-replace the +break-glass user:** + +```bash +tofu -chdir=opentofu/gcp/openbao/management apply \ + -replace=module.store_of_record.vault_generic_endpoint.admin_user +``` + +`vault_generic_endpoint.admin_user` sets `disable_read = true` (the password is +never readable back, so a read would always look like drift). That also means a +plain apply against the new, empty OpenBao sees no drift on this resource and +never recreates `auth/userpass/users/admin` — the tofu state still has it +recorded from the lineage that just got discarded. Without the `-replace`, +`openbao-priv-gcp-admin-credentials` keeps publishing a password for a +break-glass user that no longer exists, and there is no way back in once OIDC +is also unconfigured. + The switch has five limits: - It is honoured only when no top-level object carries the node's own seal, and From b06d8340ed5db38eec1ffcb3ddfe7348b4598e27 Mon Sep 17 00:00:00 2001 From: Smana Date: Tue, 29 Sep 2026 10:45:56 +0200 Subject: [PATCH 8/8] fix(openbao): the break-glass replace keeps the stack's var-file The printed and documented -replace command omitted -var-file, so secret_owning_apps fell back to [] and the apply would destroy every app persona; it also lacked the stack's -parallelism=1. It now runs after the deploy finishes, not mid-deploy. The policy parity gate also treats a copy missing on one side as divergence, so deleting one no longer passes. --- scripts/ci/tests/test-openbao-new-lineage.sh | 2 +- scripts/ci/tests/test-validate-openbao-policies.sh | 4 ++++ scripts/ci/validate-openbao-policies.sh | 3 ++- scripts/provision/openbao-config.sh | 5 +++-- .../docs/guides/openbao-cross-cloud-failover.md | 11 ++++++++--- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/scripts/ci/tests/test-openbao-new-lineage.sh b/scripts/ci/tests/test-openbao-new-lineage.sh index c0cb24e90..326832aee 100644 --- a/scripts/ci/tests/test-openbao-new-lineage.sh +++ b/scripts/ci/tests/test-openbao-new-lineage.sh @@ -87,7 +87,7 @@ arm="$(printf '%s\n' "$fn" | sed -n '/^[[:space:]]*proceed)/,/;;/p')" contains "the proceed arm initialises" "$arm" 'init_openbao' contains "the proceed arm returns before falling into the seal gate" "$arm" 'return 0' contains "the proceed arm names the break-glass replace address (disable_read hides the missing user otherwise)" "$arm" \ - 'tofu apply -replace=module.store_of_record.vault_generic_endpoint.admin_user' + 'apply -parallelism=1 -var-file=variables.tfvars -replace=module.store_of_record.vault_generic_endpoint.admin_user' for v in refuse-named-key refuse-same-seal refuse-own-seal-exists refuse-unsealed-object; do varm="$(printf '%s\n' "$fn" | sed -n "/^[[:space:]]*${v})/,/;;/p")" contains "the ${v} arm exits non-zero" "$varm" 'exit 1' diff --git a/scripts/ci/tests/test-validate-openbao-policies.sh b/scripts/ci/tests/test-validate-openbao-policies.sh index 9d1deb261..dfda37f06 100755 --- a/scripts/ci/tests/test-validate-openbao-policies.sh +++ b/scripts/ci/tests/test-validate-openbao-policies.sh @@ -125,6 +125,10 @@ expect "identical shared policy copies pass" 0 "$r" echo "path \"y\" { capabilities = [\"read\"] }" >"$r/opentofu/shared/modules/openbao-store-of-record/policies/admin.hcl" expect "a diverged shared policy copy fails, naming the file" 1 "$r" \ 'opentofu/aws/openbao/management/policies/admin.hcl and opentofu/shared/modules/openbao-store-of-record/policies/admin.hcl have diverged' +cp "$r/opentofu/aws/openbao/management/policies/admin.hcl" "$r/opentofu/shared/modules/openbao-store-of-record/policies/admin.hcl" +rm "$r/opentofu/shared/modules/openbao-store-of-record/policies/secrets-admin.hcl" +expect "a shared policy copy deleted on one side fails" 1 "$r" \ + 'policies/secrets-admin.hcl have diverged' expect "the repository itself passes" 0 "$(cd "${SCRIPT_DIR}/../../.." && pwd)" diff --git a/scripts/ci/validate-openbao-policies.sh b/scripts/ci/validate-openbao-policies.sh index ddf2e6baf..043718ec7 100755 --- a/scripts/ci/validate-openbao-policies.sh +++ b/scripts/ci/validate-openbao-policies.sh @@ -120,7 +120,8 @@ if [ -d "$AWS_POLICIES" ] && [ -d "$SHARED_POLICIES" ]; then for name in admin app-prefix external-secrets pki-admin secrets-admin; do aws_file="$AWS_POLICIES/$name.hcl" shared_file="$SHARED_POLICIES/$name.hcl" - if [ -f "$aws_file" ] && [ -f "$shared_file" ] && ! cmp -s "$aws_file" "$shared_file"; then + # A copy missing on one side is divergence too, or deleting it would pass. + if ! [ -f "$aws_file" ] || ! [ -f "$shared_file" ] || ! cmp -s "$aws_file" "$shared_file"; then echo "FAIL: opentofu/aws/openbao/management/policies/${name}.hcl and opentofu/shared/modules/openbao-store-of-record/policies/${name}.hcl have diverged -- a deliberate divergence (e.g. G-5's agents/ grant) must land in both copies" diverged=1 fi diff --git a/scripts/provision/openbao-config.sh b/scripts/provision/openbao-config.sh index a736a7da0..ab084ec90 100755 --- a/scripts/provision/openbao-config.sh +++ b/scripts/provision/openbao-config.sh @@ -1198,8 +1198,9 @@ rehydrate_openbao() { log_message "WARN" " ${ROOT_TOKEN_SECRET_NAME} and ${RECOVERY_KEYS_SECRET_NAME} are REPLACED with this node's new keys." if [ "$CLOUD" = "gcp" ]; then log_message "WARN" " those two entries are also where the awskms standby reads its pre-copied AWS keys -- re-copy them before relying on that standby. The previous versions remain in Secret Manager." - log_message "WARN" " NEXT STEP: the new OpenBao has no break-glass user yet. vault_generic_endpoint.admin_user has disable_read=true, so a plain apply sees no drift and never recreates it. Run:" - log_message "WARN" " tofu apply -replace=module.store_of_record.vault_generic_endpoint.admin_user (on the GCP management stack)" + log_message "WARN" " NEXT STEP, once this deploy finishes: the new OpenBao has no break-glass user. vault_generic_endpoint.admin_user has disable_read=true, so the deploy's apply sees no drift and never recreates it. Run:" + log_message "WARN" " tofu -chdir=opentofu/gcp/openbao/management apply -parallelism=1 -var-file=variables.tfvars -replace=module.store_of_record.vault_generic_endpoint.admin_user" + log_message "WARN" " (without -var-file, secret_owning_apps falls back to [] and the apply destroys the app personas)" fi init_openbao return 0 ;; diff --git a/website/content/docs/guides/openbao-cross-cloud-failover.md b/website/content/docs/guides/openbao-cross-cloud-failover.md index 2b60d0cf2..f11638279 100644 --- a/website/content/docs/guides/openbao-cross-cloud-failover.md +++ b/website/content/docs/guides/openbao-cross-cloud-failover.md @@ -410,14 +410,19 @@ recovery-keys pre-flight: on a project where `openbao-priv-gcp-recovery-keys` has no readable version, the switch is unreachable and `rehydrate` refuses before it ever reads `OPENBAO_NEW_LINEAGE`. -**Immediately after the proceed arm prints its warning, force-replace the -break-glass user:** +**Once the deploy that printed the proceed arm's warning finishes, force-replace +the break-glass user:** ```bash -tofu -chdir=opentofu/gcp/openbao/management apply \ +tofu -chdir=opentofu/gcp/openbao/management apply -parallelism=1 \ + -var-file=variables.tfvars \ -replace=module.store_of_record.vault_generic_endpoint.admin_user ``` +Keep both flags. Without `-var-file`, `secret_owning_apps` falls back to `[]` +and the plan destroys every app persona; `-parallelism=1` is what the stack's +own deploy uses against the OpenBao 2.6 write deadlock. + `vault_generic_endpoint.admin_user` sets `disable_read = true` (the password is never readable back, so a read would always look like drift). That also means a plain apply against the new, empty OpenBao sees no drift on this resource and