From 599a4af7d8790b8c229e983ae384bda0917bd940 Mon Sep 17 00:00:00 2001 From: mjnong Date: Fri, 28 Aug 2026 03:28:27 +0200 Subject: [PATCH] feat(deploy-backend): present a client certificate to buildkitd (CEL-1513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 1 of 3 of the shared buildkitd mTLS cutover. The server half is CellarNode/crossplane-gcloud#78; its runbook is docs/buildkit-mtls.md there. buildkitd applies TLS per listener and one daemon cannot serve plaintext and TLS on the same TCP port, so the server cutover is atomic and spans two repos. This lands first and is correct on both sides of it: the TLS builder is used when the ARC container-hook extension has mounted the client leaf at /run/buildkit/certs, and the current plaintext create runs otherwise. `docker buildx create` records an endpoint without dialling it, so both paths are keyed on `inspect --bootstrap` rather than create's exit status. The fallback also bootstraps, so a build that can reach buildkitd on neither transport fails at this step instead of at the first `docker buildx build`. Adds .github/tests/buildkit-mtls-client.test.rb as the client half of the invariant (the server half is scripts/verify_buildkit_mtls.py in crossplane-gcloud). Its PLAINTEXT_FALLBACK_EXPECTED constant is the merge-order gate: true here, flipped to false in step 3 when the fallback is deleted. The regression it exists for is a silent downgrade — dropping the --driver-opt flags, dead-coding the TLS branch, or letting the fallback swallow a failed handshake once the server is TLS-only all leave every build green while the connection reverts to unauthenticated plaintext. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- .github/tests/buildkit-mtls-client.test.rb | 132 ++++++++++++++++++ .github/workflows/deploy-backend.yaml | 72 +++++++++- .../workflows/validate-backend-deploy.yaml | 32 +++++ 3 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 .github/tests/buildkit-mtls-client.test.rb create mode 100644 .github/workflows/validate-backend-deploy.yaml diff --git a/.github/tests/buildkit-mtls-client.test.rb b/.github/tests/buildkit-mtls-client.test.rb new file mode 100644 index 0000000..9e96173 --- /dev/null +++ b/.github/tests/buildkit-mtls-client.test.rb @@ -0,0 +1,132 @@ +# Client half of the buildkitd mutual-TLS invariant (CEL-1513). +# +# The server half lives in CellarNode/crossplane-gcloud +# (scripts/verify_buildkit_mtls.py). That one proves buildkitd demands a client +# certificate; this one proves the only caller actually presents one, over the +# exact contract documented in crossplane-gcloud docs/buildkit-mtls.md: +# +# docker buildx create --name k8s-buildkit --driver remote \ +# --driver-opt cacert=/run/buildkit/certs/ca.crt \ +# --driver-opt cert=/run/buildkit/certs/tls.crt \ +# --driver-opt key=/run/buildkit/certs/tls.key \ +# tcp://buildkitd.buildkit.svc.cluster.local:1234 --use +# +# The regression it exists for is a silent downgrade. Dropping the --driver-opt +# flags, or letting the fallback swallow a failed handshake once the server is +# TLS-only, leaves every build green while the connection goes back to +# unauthenticated plaintext. Nothing else in CI notices. + +require "yaml" + +# Merge-order gate. The rollout is three steps and this constant is which one +# we are on: +# +# true - step 1 of 3. This repo lands FIRST, before crossplane-gcloud#78. +# The certificate does not exist in the cluster yet, so the step must +# still be able to reach a plaintext buildkitd. +# false - step 3 of 3. #78 has soaked; the fallback is deleted so a +# server-side TLS regression fails the build instead of downgrading +# it. Flip this in the same commit that deletes the branch. +PLAINTEXT_FALLBACK_EXPECTED = true + +CERT_DIR = "/run/buildkit/certs".freeze +BUILDKITD_ADDR = "tcp://buildkitd.buildkit.svc.cluster.local:1234".freeze + +workflow_path = File.expand_path("../workflows/deploy-backend.yaml", __dir__) +workflow = YAML.safe_load(File.read(workflow_path), aliases: true) + +build = workflow.fetch("jobs").fetch("build") +connect = build.fetch("steps").find { |step| step.fetch("name", "") == "Connect to in-cluster buildkitd" } +abort "deploy-backend must have a 'Connect to in-cluster buildkitd' step" if connect.nil? + +run = connect.fetch("run", "") +env = connect.fetch("env", {}) + +# --- endpoint and certificate paths are data, not inlined literals ----------- + +abort "buildkitd address must be passed as env data" unless env.fetch("BUILDKITD_ADDR", nil) == BUILDKITD_ADDR +abort "buildkitd certificate directory must be passed as env data" unless env.fetch("BUILDKITD_CERT_DIR", nil) == CERT_DIR +abort "buildkitd endpoint must not be inlined in the script" if run.include?(BUILDKITD_ADDR) + +# --- the mTLS invocation itself --------------------------------------------- + +abort "buildkitd connection must use the remote driver" unless run.include?("--driver remote") + +{ + "cacert" => "ca.crt", + "cert" => "tls.crt", + "key" => "tls.key", +}.each do |opt, file| + # cert-manager's key names, absolute paths. buildx rejects relative ones and + # renaming a key silently produces a builder that cannot handshake. + abort "buildkitd client must pass --driver-opt #{opt}=/#{file}" unless + run.include?(%(--driver-opt "#{opt}=${BUILDKITD_CERT_DIR}/#{file}")) +end + +# All three files, or none. A partial mount must not select the TLS path and +# then fail the handshake on a missing key. +["ca.crt", "tls.crt", "tls.key"].each do |file| + abort "buildkitd client must require #{file} to be readable before choosing TLS" unless + run.include?(%([ -r "${BUILDKITD_CERT_DIR}/#{file}" ])) +end + +# The condition is pinned whole, not just scanned for its parts. Certificate +# presence must be the ONLY thing gating the TLS branch: an extra conjunct +# (`if false && [ -r ... ]`, `... && [ "$SOMETHING" = 1 ]`) leaves every string +# this file greps for intact while making the branch unreachable, and the +# fallback below then quietly carries every build over plaintext. Reformatting +# the guard is expected to fail here — re-read it, then update this regex. +tls_guard = / + ^[ ]*if[ ]\[[ ]-r[ ]"\$\{BUILDKITD_CERT_DIR\}\/ca\.crt"[ ]\][ ]\\\n + [ ]*&&[ ]\[[ ]-r[ ]"\$\{BUILDKITD_CERT_DIR\}\/tls\.crt"[ ]\][ ]\\\n + [ ]*&&[ ]\[[ ]-r[ ]"\$\{BUILDKITD_CERT_DIR\}\/tls\.key"[ ]\];[ ]then$ +/x +abort "the TLS branch must be gated on certificate presence and nothing else" unless run.match?(tls_guard) + +# `docker buildx create` records an endpoint without dialling it, so create +# exiting 0 proves nothing. Only bootstrap performs the handshake. +abort "buildkitd connection must be proven with 'inspect --bootstrap'" unless run.include?("docker buildx inspect --bootstrap k8s-buildkit") + +# --- merge-order gate -------------------------------------------------------- + +fallback_markers = [ + "connected=false", + %(if [ "$connected" != true ]; then), +] + +if PLAINTEXT_FALLBACK_EXPECTED + fallback_markers.each do |marker| + abort "step 1 of 3 must keep the plaintext fallback (missing: #{marker})" unless run.include?(marker) + end + + # The fallback exists so a plaintext server still works. It must never be + # the only path: the TLS attempt has to come first. + tls_at = run.index("--driver-opt") + fallback_at = run.index(%(if [ "$connected" != true ]; then)) + abort "the mTLS attempt must precede the plaintext fallback" unless tls_at && fallback_at && tls_at < fallback_at + + # An unbootstrapped fallback would defer a total connection failure to the + # first `docker buildx build`, where it reads as a build error. + abort "the plaintext fallback must bootstrap too, so a dead endpoint fails here" unless + run.match?(/if \[ "\$connected" != true \]; then\s+recreate\s+docker buildx inspect --bootstrap/) +else + # Step 3 of 3: no branch may reach buildkitd without the client credential. + fallback_markers.each do |marker| + abort "step 3 of 3 must delete the plaintext fallback (still present: #{marker})" if run.include?(marker) + end + abort "step 3 of 3 must not leave a bare plaintext create" if + run.match?(/recreate\s*$/) || run.include?("recreate\n") +end + +# --- fork PRs must never see a client certificate --------------------------- + +# The credential is mounted by ARC into job pods in the cluster. Fork PRs are +# untrusted and build on GitHub-hosted runners, which have no cluster access +# and therefore no certificate; if that job ever moved to self-hosted it would +# hand arbitrary PR code a working buildkitd credential. +build_fork = workflow.fetch("jobs").fetch("build-fork") +abort "fork builds must stay on GitHub-hosted runners" unless build_fork.fetch("runs-on") == "ubuntu-latest" +abort "fork builds must not connect to in-cluster buildkitd" if + build_fork.fetch("steps").any? { |step| step.fetch("run", "").include?("--driver remote") } + +puts "BuildKit mTLS client contract passed (plaintext fallback: #{PLAINTEXT_FALLBACK_EXPECTED})" diff --git a/.github/workflows/deploy-backend.yaml b/.github/workflows/deploy-backend.yaml index 186f8fa..3c18a5c 100644 --- a/.github/workflows/deploy-backend.yaml +++ b/.github/workflows/deploy-backend.yaml @@ -94,12 +94,74 @@ jobs: fi fi - # Plain TCP is accepted here: private single-tenant cluster, and the - # legacy GKE dataplane doesn't enforce NetworkPolicy. Revisit (TLS via - # --driver-opt cacert/cert/key, or DPv2 + NetworkPolicy) if the cluster - # ever becomes multi-tenant. + # CEL-1513 step 1 of 3 — mutual TLS on the shared buildkitd, with a + # plaintext fallback. Server side is CellarNode/crossplane-gcloud#78 + # (runbook: docs/buildkit-mtls.md there). + # + # buildkitd applies TLS per listener and one daemon cannot serve + # plaintext and TLS on the same TCP port, so the server cutover is + # atomic and it spans two repos. This step lands FIRST and is correct on + # both sides of it: + # + # - server plaintext (today): the client leaf is not in the cluster + # yet, /run/buildkit/certs is absent, the guard is false, and the + # plaintext create runs exactly as before. + # - server TLS (after #78 syncs): ARC's container-hook extension mounts + # the client leaf into every job pod at /run/buildkit/certs, the + # guard is true, and the mTLS builder bootstraps. + # - mid-sync, either order: whichever path fails the handshake falls + # through to the other, and if neither connects the step fails here + # rather than at the first build. + # + # `docker buildx create` does not dial the daemon — it only records the + # endpoint. `inspect --bootstrap` is what performs the handshake, so + # both paths are keyed on it and not on create's exit status. + # + # Certificate file names are cert-manager's (ca.crt / tls.crt / tls.key) + # and the paths must be absolute. No `servername` override: the server + # leaf's SAN list covers the FQDN dialled below. + # + # STEP 3 of 3 removes the fallback branch, leaving only the mTLS path. + # Until it does, a server-side regression that dropped --tlscacert would + # silently downgrade CI to plaintext instead of failing the build. + # .github/tests/buildkit-mtls-client.test.rb pins that transition: flip + # PLAINTEXT_FALLBACK_EXPECTED there in the same commit. - name: Connect to in-cluster buildkitd - run: docker buildx create --name k8s-buildkit --driver remote tcp://buildkitd.buildkit.svc.cluster.local:1234 --use + env: + BUILDKITD_ADDR: tcp://buildkitd.buildkit.svc.cluster.local:1234 + BUILDKITD_CERT_DIR: /run/buildkit/certs + run: | + set -euo pipefail + + recreate() { + docker buildx rm k8s-buildkit >/dev/null 2>&1 || true + docker buildx create --name k8s-buildkit --driver remote "$@" \ + "$BUILDKITD_ADDR" --use + } + + connected=false + + if [ -r "${BUILDKITD_CERT_DIR}/ca.crt" ] \ + && [ -r "${BUILDKITD_CERT_DIR}/tls.crt" ] \ + && [ -r "${BUILDKITD_CERT_DIR}/tls.key" ]; then + echo "buildkitd client certificate found - connecting with mutual TLS" + recreate \ + --driver-opt "cacert=${BUILDKITD_CERT_DIR}/ca.crt" \ + --driver-opt "cert=${BUILDKITD_CERT_DIR}/tls.crt" \ + --driver-opt "key=${BUILDKITD_CERT_DIR}/tls.key" + if docker buildx inspect --bootstrap k8s-buildkit; then + connected=true + else + echo "::warning title=buildkitd mTLS handshake failed::Falling back to plaintext (CEL-1513 step 1 of 3). If buildkitd is already TLS-only this build will fail at connect - re-run it." + fi + else + echo "no buildkitd client certificate at ${BUILDKITD_CERT_DIR} - using plaintext (CEL-1513 step 1 of 3)" + fi + + if [ "$connected" != true ]; then + recreate + docker buildx inspect --bootstrap k8s-buildkit + fi - name: Authenticate to Google Cloud id: auth diff --git a/.github/workflows/validate-backend-deploy.yaml b/.github/workflows/validate-backend-deploy.yaml new file mode 100644 index 0000000..0705049 --- /dev/null +++ b/.github/workflows/validate-backend-deploy.yaml @@ -0,0 +1,32 @@ +name: Validate shared backend deploy workflow + +on: + pull_request: + paths: + - .github/tests/buildkit-mtls-client.test.rb + - .github/workflows/deploy-backend.yaml + - .github/workflows/validate-backend-deploy.yaml + push: + branches: [main] + paths: + - .github/tests/buildkit-mtls-client.test.rb + - .github/workflows/deploy-backend.yaml + - .github/workflows/validate-backend-deploy.yaml + +permissions: + contents: read + +jobs: + buildkit-mtls-client: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + + # Client half of the CEL-1513 invariant. The server half is + # scripts/verify_buildkit_mtls.py in CellarNode/crossplane-gcloud; both + # halves have to hold for the control to exist. + - name: Test BuildKit mTLS client contract + run: ruby .github/tests/buildkit-mtls-client.test.rb