From 4dbca56ad268429835f3f779746ca3f6ca3d6c82 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:03:27 -0700 Subject: [PATCH 1/3] ci: reference the upstream MetalLB issue from the workaround Records nebari-dev/nebari-infrastructure-core#612 next to the pool-pinning step so whoever revisits this knows the workaround has an expiry: once NIC fixes the InfraSettings/Deploy ordering, the step and the nic-config can both go and the action's own config template can be used again. --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6135eae..6e3bfb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,10 @@ jobs: # with: the pool gets NIC's unroutable 192.168.1.100-110 fallback # instead, so the gateway LoadBalancer IP has no route from the # runner and every request to it times out. An explicitly configured - # pool takes precedence over both, so pin it ourselves. + # pool takes precedence over both, so pin it ourselves. Upstream: + # nebari-dev/nebari-infrastructure-core#612. Once that is fixed and + # released, this step and the nic-config below can both be dropped in + # favour of the action's own config template. # # Derive it from the network kind will actually use rather than # hardcoding a range, so this self-corrects if Docker picks a From ca35e0719e02b4a2e4b92d338d6be1cb242cc356 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:09:27 -0700 Subject: [PATCH 2/3] feat(chart): add trustBundle, and exercise real auth in the e2e job The e2e job ran with auth.devMode=true, so the OIDC path was never tested. The blocker was CA trust: the pod must complete OIDC discovery against https://keycloak.nebari.local before /readyz turns 200, and the sandbox gateway's certificate is signed by a self-signed CA the pod has no reason to trust. The sandbox resolving *.nebari.local in-cluster removed the other half of that blocker. Trusting a private CA is a real deployment need, not a CI-only one - any Nebari behind a private CA or a TLS-terminating proxy has it - so this adds it to the chart rather than hacking around it in the workflow: trustBundle: configMapName: "" key: ca-certificates.crt When configMapName is set the chart mounts that key read-only at /etc/ssl/nebari and points SSL_CERT_FILE at it. No Go change is needed: oidc.NewProvider uses the default pool, which honors that variable. Only the configured key is projected, so an unrelated entry in the ConfigMap cannot appear beside the bundle. The ConfigMap must hold a COMPLETE bundle - public roots plus the private CA - because SSL_CERT_FILE replaces Go's default pool rather than extending it. NIC already produces exactly that on a cluster with trust_bundle configured: a nebari-trust-bundle ConfigMap projected into every namespace, public roots included. values.yaml and the chart README both say so, since a CA-only file is the easy mistake here. Three features now contribute volumes, and the gate was spelled out twice (volumeMounts and volumes). A third term repeated in both places is where that stops being readable, so the condition moves into a nebari-frames.hasVolumes helper. In the e2e job the sandbox CA is staged into the frames namespace from the action's ca-cert-path output, which avoids copying it out of kube-public (a ConfigMap cannot be mounted across namespaces) and lets the namespace be created up front carrying the operator's opt-in label. No new wait logic: Synced/Healthy already gates on the readiness probe, so it is itself the assertion that DNS, CA trust, and the operator's client provisioning all work. Two deliberate guards against a test that passes without testing anything. The chart job now asserts on rendered content rather than only that a render succeeded, and the e2e job asserts the pod is really in real-auth mode - FRAMES_DEV_MODE absent, OIDC_ISSUER_URL present, SSL_CERT_FILE pointing at the mount. Dev mode also reports ready, so a quiet fallback to it would otherwise look green. This is the failure mode that let action-nebari-sandbox#103 sit unnoticed. Note the sandbox ConfigMap holds the gateway CA alone, so the job does not cover the complete-bundle case a production cluster uses. Also corrects AGENTS.md, which still described the sandbox as k3d. --- .github/workflows/ci.yml | 110 ++++++++++++++++++++++++++++---- AGENTS.md | 5 +- chart/README.md | 28 ++++++++ chart/templates/_helpers.tpl | 25 ++++++++ chart/templates/deployment.yaml | 23 ++++++- chart/values.schema.json | 8 +++ chart/values.yaml | 14 ++++ 7 files changed, 195 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e3bfb0..e907051 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,12 +88,34 @@ jobs: run: | helm template frames chart/ --set auth.devMode=true \ > /tmp/render-devmode.yaml + - name: Template renders (trust bundle) + run: | + helm template frames chart/ --set auth.devMode=true \ + --set trustBundle.configMapName=nebari-trust-bundle \ + > /tmp/render-trustbundle.yaml + # Assert on content, not just that it rendered: the whole point of + # the block is the mount and the env var, and a silently-empty + # render would still pass kubeconform below. + for want in \ + 'name: SSL_CERT_FILE' \ + 'value: "/etc/ssl/nebari/ca-certificates.crt"' \ + 'mountPath: "/etc/ssl/nebari"' \ + 'name: "nebari-trust-bundle"'; do + grep -qF "${want}" /tmp/render-trustbundle.yaml \ + || { echo "::error::trust-bundle render is missing: ${want}"; exit 1; } + done + # ...and that it stays opt-in: no trust-bundle wiring by default. + if grep -q 'SSL_CERT_FILE' /tmp/render-devmode.yaml; then + echo "::error::SSL_CERT_FILE rendered without trustBundle.configMapName set" + exit 1 + fi - name: Validate rendered manifests (kubeconform) run: | # -ignore-missing-schemas is required because the NebariApp CRD has # no published JSON schema anywhere; do not remove this flag without # publishing one first. - for f in /tmp/render-nebariapp.yaml /tmp/render-devmode.yaml; do + for f in /tmp/render-nebariapp.yaml /tmp/render-devmode.yaml \ + /tmp/render-trustbundle.yaml; do echo "::group::kubeconform ${f}" # Pinned for the same reason golangci-lint is: a new release must # not change gate behavior without a deliberate bump. v0.8+ needs @@ -189,6 +211,37 @@ jobs: # cluster-name output (its kubectl context is `kind-`). run: kind load docker-image nebari-frames:e2e --name ${{ steps.sandbox.outputs.cluster-name }} + - name: Provision the sandbox CA for the frames pod + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + CA_CERT_PATH: ${{ steps.sandbox.outputs.ca-cert-path }} + run: | + # Real auth needs the pod to trust the sandbox's self-signed gateway + # CA, or OIDC discovery against https://keycloak.nebari.local fails + # and /readyz never turns 200. The action hands us the CA as a file on + # the runner, so build the ConfigMap from that rather than copying the + # in-cluster `nebari-sandbox-ca` out of kube-public (a ConfigMap + # cannot be mounted across namespaces). + [ -s "${CA_CERT_PATH}" ] \ + || { echo "::error::sandbox ca-cert-path is empty or missing: '${CA_CERT_PATH}'"; exit 1; } + + # Create the namespace up front, carrying the opt-in label the + # operator requires. ArgoCD's CreateNamespace=true tolerates an + # existing namespace, and the ConfigMap is untracked by ArgoCD so + # `prune: true` leaves it alone. + kubectl create namespace nebari-frames --dry-run=client -o yaml \ + | kubectl apply -f - + kubectl label namespace nebari-frames nebari.dev/managed=true --overwrite + + # NOTE: this is the gateway CA alone, not public roots + CA as the + # chart's trustBundle contract asks for. Fine here (frames only talks + # to in-cluster Keycloak), but it means SSL_CERT_FILE narrows the + # pod's trust to this one CA, so the job does not exercise the + # complete-bundle case a production cluster would use. + kubectl create configmap nebari-sandbox-ca \ + --from-file=ca-certificates.crt="${CA_CERT_PATH}" \ + -n nebari-frames --dry-run=client -o yaml | kubectl apply -f - + - name: Deploy frames via GitOps env: GITOPS_DIR: ${{ steps.sandbox.outputs.gitops-dir }} @@ -216,23 +269,21 @@ jobs: image: repository: nebari-frames tag: e2e - # Auth runs in dev mode: the app's OIDC readiness check - # needs to reach the issuer (https://keycloak.nebari.local) - # from inside the pod. The sandbox now resolves - # *.nebari.local in-cluster, but its gateway CA is - # self-signed and the pod does not trust it, so /readyz - # stays 503 forever with real auth. Enabling real auth means - # mounting the sandbox CA into the pod (the action exposes - # it as ConfigMap `nebari-sandbox-ca` in `kube-public`). - # Routing, TLS, and the gateway path are still fully - # exercised via the NebariApp. + # Real auth, not dev mode. The operator provisions the + # OIDC clients and the pod must complete discovery against + # https://keycloak.nebari.local before /readyz turns 200, so + # the Synced/Healthy wait below is itself the assertion that + # in-cluster DNS, CA trust, and client provisioning all work. nebariapp: enabled: true hostname: frames.nebari.local auth: - enabled: false - auth: - devMode: true + enabled: true + # Trust the sandbox gateway CA staged into the namespace by + # the step above; without it discovery fails on an unknown + # authority. + trustBundle: + configMapName: nebari-sandbox-ca seed: orgSlug: e2e-org orgDisplayName: E2E Org @@ -316,6 +367,37 @@ jobs: || true exit 1 + - name: Auth is really on (not silently dev mode) + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + run: | + # Guard against a vacuous pass. Healthy above proves /readyz is 200, + # but dev mode also reports ready - so without this, a chart change + # that quietly fell back to devMode would still look green and the + # OIDC path would go untested. (Exactly how the sandbox action's + # gateway-ip check passed for months without connecting to anything: + # nebari-dev/action-nebari-sandbox#103.) + ENV_JSON=$(kubectl -n nebari-frames get deploy nebari-frames-nebari-frames \ + -o jsonpath='{.spec.template.spec.containers[0].env}') + echo "container env: ${ENV_JSON}" + case "${ENV_JSON}" in + *FRAMES_DEV_MODE*) + echo "::error::FRAMES_DEV_MODE is set - the pod is in dev mode, so real auth was not exercised" + exit 1 ;; + esac + case "${ENV_JSON}" in + *OIDC_ISSUER_URL*) ;; + *) echo "::error::OIDC_ISSUER_URL is absent - auth is not wired at all"; exit 1 ;; + esac + # Readiness proves discovery succeeded; this proves it succeeded via + # the mounted bundle rather than by some other route, so a silently + # dropped volume shows up here instead of passing unnoticed. + kubectl -n nebari-frames get deploy nebari-frames-nebari-frames \ + -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="SSL_CERT_FILE")].value}' \ + | grep -qF '/etc/ssl/nebari/ca-certificates.crt' \ + || { echo "::error::SSL_CERT_FILE is not pointing at the mounted trust bundle"; exit 1; } + echo "OK: real auth, OIDC wired, trust bundle mounted" + - name: Endpoint answers through the gateway env: GATEWAY_IP: ${{ steps.sandbox.outputs.gateway-ip }} diff --git a/AGENTS.md b/AGENTS.md index 3ce851f..b6982b6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,8 +44,9 @@ go run ./tools/docs-gen CI (`.github/workflows/ci.yml`) gates on five jobs: `proto` (buf lint plus a stale-codegen check on `gen/`), `go` (golangci-lint pinned to v2.12 plus race tests), `web` (lint, typecheck, vitest), `chart` (helm lint, template renders, kubeconform pinned to v0.7.0), and `e2e-sandbox` (deploys the -built image onto a k3d Nebari sandbox via ArgoCD). `docs.yml` also fails if the generated CLI -reference is stale. Run the local equivalents before pushing. +built image onto a kind Nebari sandbox via ArgoCD and exercises real Keycloak auth through the +gateway). `docs.yml` also fails if the generated CLI reference is stale. Run the local +equivalents before pushing. **Generated code is checked in.** After touching `proto/frames/v1/*.proto`, run `make proto` and commit `gen/`. After touching `cli/cmd/*`, run `go run ./tools/docs-gen` and commit diff --git a/chart/README.md b/chart/README.md index 9a87d45..15fac33 100644 --- a/chart/README.md +++ b/chart/README.md @@ -78,6 +78,32 @@ The chart renders these into the `-nebari-frames-config` ConfigMap, mou Every field is optional and falls back to its built-in Nebari default, so an unbranded install renders exactly the manifests it did before - no ConfigMap, no volume, no env var. Overriding `theme.*.primary` also rebrands button hover states and focus rings, which are derived from it. See [Configuration → Branding](https://nebari-dev.github.io/nebari-frames/configuration/#branding) for the full token list, the `BRANDING_*` environment variables used outside Kubernetes, and the value validation rules. +## Private CA trust + +When the OIDC issuer's certificate is signed by a private CA, the pod has to +trust that CA or startup OIDC discovery fails and `/readyz` stays `503`. Point +`trustBundle.configMapName` at a ConfigMap holding the bundle: + +```yaml +trustBundle: + configMapName: nebari-trust-bundle + key: ca-certificates.crt +``` + +The chart mounts that key read-only at `/etc/ssl/nebari` and sets +`SSL_CERT_FILE` to it. + +**The ConfigMap must contain a complete bundle - public roots plus the private +CA, not the CA alone.** `SSL_CERT_FILE` *replaces* Go's default CA pool rather +than extending it, so a CA-only file leaves the pod unable to reach anything +publicly signed. + +On a Nebari cluster with `trust_bundle` configured, NIC does this for you: it +projects a ready-made `nebari-trust-bundle` ConfigMap - public roots included - +into every namespace via trust-manager, so the snippet above is all you need. On +a cluster without `trust_bundle` configured that ConfigMap does not exist, and +you supply your own. + ## Telemetry The server writes structured JSON logs to stdout, ready for collection by the platform's log stack (e.g. the LGTM stack on Nebari). There is no `ServiceMonitor` in this chart because the server does not expose a metrics endpoint yet. When metrics land, the chart will grow a `ServiceMonitor` gated behind a `metrics.enabled` value so clusters without the Prometheus operator are unaffected. @@ -102,6 +128,8 @@ The server writes structured JSON logs to stdout, ready for collection by the pl | `persistence.size` | PVC size. Defaults to `1Gi`. | | `persistence.storageClass` | StorageClass for the PVC. Defaults to `""`, which uses the cluster default class. Set this explicitly in production so you control which class backs the volume. | | `persistence.accessMode` | PVC access mode. Defaults to `ReadWriteOnce`. | +| `trustBundle.configMapName` | ConfigMap holding a CA bundle to trust for outbound TLS, e.g. an OIDC issuer signed by a private CA. Empty (the default) uses the image's own trust store. See [Private CA trust](#private-ca-trust). | +| `trustBundle.key` | Key within that ConfigMap. Defaults to `ca-certificates.crt`, which is what NIC's `nebari-trust-bundle` uses. | | `branding.title` | Browser-tab title (also the logo's alt text). Empty keeps `Nebari Frames`. | | `branding.logoUrl` | Header and sign-in logo (light mode / default). Empty keeps the built-in Nebari wordmark. | | `branding.logoUrlDark` | Dark-mode logo. Empty falls back to `branding.logoUrl`, then the built-in dark wordmark. | diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index 300c2f6..cc1a969 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -82,3 +82,28 @@ NebariApp hostname. Empty string when neither is available (endpoint stays off). {{- printf "https://%s" (required "nebariapp.hostname is required when mcp.enabled and mcp.publicUrl is unset" .Values.nebariapp.hostname) -}} {{- end -}} {{- end -}} + +{{/* +Mount path and file for the trust bundle. Consumers point SSL_CERT_FILE at the +file, which REPLACES Go's default CA pool rather than extending it, so the +ConfigMap must carry a complete bundle (public roots plus the private CA). +*/}} +{{- define "nebari-frames.trustBundleMountPath" -}} +/etc/ssl/nebari +{{- end -}} + +{{- define "nebari-frames.trustBundleFilePath" -}} +{{- printf "%s/%s" (include "nebari-frames.trustBundleMountPath" .) .Values.trustBundle.key -}} +{{- end -}} + +{{/* +"true" when the pod needs a volumes/volumeMounts block at all, empty otherwise. +Three independent features contribute volumes now, and the gate is repeated for +volumeMounts and volumes, so the condition lives here rather than being spelled +out twice. +*/}} +{{- define "nebari-frames.hasVolumes" -}} +{{- if or .Values.persistence.enabled (include "nebari-frames.hasBranding" .) .Values.trustBundle.configMapName -}} +true +{{- end -}} +{{- end -}} diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 039a8f1..a80945f 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -99,6 +99,10 @@ spec: value: {{ .Values.auth.oidc.deviceClientId | quote }} {{- end }} {{- end }} + {{- if .Values.trustBundle.configMapName }} + - name: SSL_CERT_FILE + value: {{ include "nebari-frames.trustBundleFilePath" . | quote }} + {{- end }} livenessProbe: httpGet: path: /healthz @@ -116,7 +120,7 @@ spec: path: /readyz port: http periodSeconds: 10 - {{- if or .Values.persistence.enabled (include "nebari-frames.hasBranding" .) }} + {{- if include "nebari-frames.hasVolumes" . }} volumeMounts: {{- if .Values.persistence.enabled }} - name: data @@ -127,10 +131,15 @@ spec: mountPath: {{ include "nebari-frames.configMountPath" . | quote }} readOnly: true {{- end }} + {{- if .Values.trustBundle.configMapName }} + - name: trust-bundle + mountPath: {{ include "nebari-frames.trustBundleMountPath" . | quote }} + readOnly: true + {{- end }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{- if or .Values.persistence.enabled (include "nebari-frames.hasBranding" .) }} + {{- if include "nebari-frames.hasVolumes" . }} volumes: {{- if .Values.persistence.enabled }} - name: data @@ -142,4 +151,14 @@ spec: configMap: name: {{ include "nebari-frames.configMapName" . }} {{- end }} + {{- if .Values.trustBundle.configMapName }} + - name: trust-bundle + configMap: + name: {{ .Values.trustBundle.configMapName | quote }} + {{- /* Project only the configured key, so an unrelated entry in + the ConfigMap cannot appear alongside the bundle. */}} + items: + - key: {{ .Values.trustBundle.key | quote }} + path: {{ .Values.trustBundle.key | quote }} + {{- end }} {{- end }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 9992aaf..ee74c31 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -28,6 +28,14 @@ "accessMode": { "type": "string", "enum": ["ReadWriteOnce", "ReadWriteOncePod"] } } }, + "trustBundle": { + "type": "object", + "additionalProperties": false, + "properties": { + "configMapName": { "type": "string" }, + "key": { "type": "string", "minLength": 1 } + } + }, "branding": { "type": "object", "additionalProperties": false, diff --git a/chart/values.yaml b/chart/values.yaml index 9b88c04..8d805bf 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -118,6 +118,20 @@ nebariapp: - profile - email +# CA bundle for outbound TLS, e.g. an OIDC issuer whose certificate is signed by +# a private CA. Leave configMapName empty (the default) to use the image's own +# trust store, which renders exactly as it did before this block existed. +# +# The ConfigMap must carry a COMPLETE bundle - public roots plus the private CA. +# The pod consumes it via SSL_CERT_FILE, which REPLACES Go's default CA pool +# rather than extending it, so a CA-only file breaks TLS to anything publicly +# signed. On a Nebari cluster with `trust_bundle` configured, NIC projects a +# ready-made `nebari-trust-bundle` ConfigMap (key `ca-certificates.crt`, +# public roots included) into every namespace - point this at that. +trustBundle: + configMapName: "" + key: ca-certificates.crt + # MCP endpoint. Enabled by default; mounts only when a public URL is derivable # (mcp.publicUrl, else https:// when nebariapp.enabled). mcp: From 827e638c2f88e65e929e42765f8b360e9216751f Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:08:36 -0700 Subject: [PATCH 3/3] ci: make *.nebari.local resolve inside the kind cluster Real auth failed at DNS, before TLS was ever attempted: auth: OIDC discovery not ready; retrying error: ... lookup keycloak.nebari.local on 10.96.0.10:53: server misbehaving The sandbox action's in-cluster DNS step is a no-op on kind. It writes a `coredns-custom` ConfigMap, which is a k3s convention - k3s CoreDNS mounts it and does `import /etc/coredns/custom/*.server`. kind's CoreDNS comes from kubeadm with a plain `.:53` Corefile and no such import, so the ConfigMap is created, CoreDNS is restarted, and the file is never read. The step still logs "In-cluster DNS ready", which is what makes it hard to spot. Filed as nebari-dev/action-nebari-sandbox#104. Keycloak advertises the external issuer in its discovery document, so the frames pod must resolve keycloak.nebari.local to complete discovery. Add the zone to the Corefile kind actually reads, via a merge patch on just that key so nothing else in the ConfigMap is disturbed, skipping when the zone is already present so a re-run is not additive. Then prove it, with a throwaway pod that resolves the name and must see the gateway IP. Without that the failure surfaces fifteen minutes later as an unexplained unhealthy Application - which is precisely how the upstream no-op went unnoticed, since nothing in the action's suite ever resolves a hostname from inside a pod. This is the third workaround in this job for a v3 regression, all from the same k3d-to-kind migration: the MetalLB pool (#612) and now DNS, plus the gateway-ip test gap (#103) that let both ship green. Each carries the issue reference and comes out when upstream lands. --- .github/workflows/ci.yml | 67 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e907051..f16ba31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -211,6 +211,73 @@ jobs: # cluster-name output (its kubectl context is `kind-`). run: kind load docker-image nebari-frames:e2e --name ${{ steps.sandbox.outputs.cluster-name }} + - name: Make *.nebari.local resolve inside the cluster + env: + KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }} + GATEWAY_IP: ${{ steps.sandbox.outputs.gateway-ip }} + DOMAIN: nebari.local + run: | + # The sandbox action's own in-cluster DNS step is a no-op on kind. It + # writes a `coredns-custom` ConfigMap, which is a k3s convention (k3s + # CoreDNS mounts it and does `import /etc/coredns/custom/*.server`); + # kind's CoreDNS comes from kubeadm with a plain `.:53` Corefile and + # no such import, so the ConfigMap is never read and pods get SERVFAIL + # for nebari.local. The action still logs "In-cluster DNS ready". + # Upstream: nebari-dev/action-nebari-sandbox#104. + # + # Keycloak advertises the external issuer in its discovery document, + # so the frames pod has to resolve keycloak.nebari.local to complete + # OIDC discovery. Add the zone to the Corefile that kind actually + # reads. Remove this once the action handles kind. + [ -n "${GATEWAY_IP}" ] || { echo "::error::gateway-ip is empty"; exit 1; } + CORE=$(kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}') + [ -n "${CORE}" ] || { echo "::error::could not read kind's coredns Corefile"; exit 1; } + + # Merge-patch only the Corefile key so nothing else in the ConfigMap + # is disturbed, and skip if the zone is already there so a re-run is + # not additive. + PATCH=$(COREFILE="${CORE}" python3 <<'PY' + import json, os + domain, ip, core = os.environ["DOMAIN"], os.environ["GATEWAY_IP"], os.environ["COREFILE"] + if f"{domain}:53" in core: + block = "" + else: + # `template` is the only stock plugin that answers a wildcard, and + # AAAA/ANY return empty NOERROR so dual-stack resolvers do not + # stall and other qtypes do not SERVFAIL in an authoritative zone. + block = ( + f"{domain}:53 {{\n" + " errors\n" + f" template IN A {domain} {{\n" + ' answer "{{ .Name }} 60 IN A ' + ip + '"\n' + " }\n" + f" template IN AAAA {domain} {{\n" + " rcode NOERROR\n" + " }\n" + f" template IN ANY {domain} {{\n" + " rcode NOERROR\n" + " }\n" + "}\n\n" + ) + print(json.dumps({"data": {"Corefile": block + core}})) + PY + ) + kubectl -n kube-system patch configmap coredns --type merge --patch "${PATCH}" + kubectl -n kube-system rollout restart deployment/coredns + kubectl -n kube-system rollout status deployment/coredns --timeout=120s + + # Prove it from inside the cluster. Without this the failure surfaces + # 15 minutes later as an unexplained unhealthy Application, which is + # how the upstream no-op went unnoticed in the first place. + kubectl delete pod dns-probe --ignore-not-found >/dev/null + OUT=$(kubectl run dns-probe --image=busybox:1.36 --restart=Never \ + --attach --rm --quiet --image-pull-policy=IfNotPresent \ + --command -- nslookup "keycloak.${DOMAIN}" 2>&1) || true + echo "${OUT}" + echo "${OUT}" | grep -qF "${GATEWAY_IP}" \ + || { echo "::error::keycloak.${DOMAIN} does not resolve to ${GATEWAY_IP} from inside the cluster"; exit 1; } + echo "OK: keycloak.${DOMAIN} resolves to ${GATEWAY_IP} for pods" + - name: Provision the sandbox CA for the frames pod env: KUBECONFIG: ${{ steps.sandbox.outputs.kubeconfig }}