diff --git a/.gitignore b/.gitignore index 041ace3a270..e4803f75e4f 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,6 @@ tmp/ # HLS config for haskell-tools plugin (Neovim) hls.json + +# Envoy AWS-LC build output (hack/envoy-aws-lc) +envoy-build.log diff --git a/Makefile b/Makefile index ccce920f5fe..2cb28ae3f3e 100644 --- a/Makefile +++ b/Makefile @@ -326,6 +326,30 @@ upload-hoogle-image: nix -v --show-trace -L build ".#wireServer.hoogleImage" --out-link $(HOOGLE_IMAGE_DIR)/image --fallback ./hack/bin/upload-image.sh $(HOOGLE_IMAGE_DIR)/image +# Envoy proxy image linked against AWS-LC instead of BoringSSL, so the Gateway +# can offer the SecP256r1MLKEM768 / SecP384r1MLKEM1024 post-quantum key +# agreement groups named by BSI TR-02102-2, which BoringSSL does not implement. +# +# Unlike every other image here this is not built by nix: the AWS-LC genrule +# needs Bazel's own downloaded LLVM toolchain plus pinned cmake/ninja/go, which +# nixpkgs' Envoy derivation patches out. It drives Envoy's build container +# instead — expect a multi-hour build and ~60G of disk. +# +# ENVOY_VERSION must match the Envoy your Envoy Gateway ships, since Envoy +# Gateway generates bootstrap config for a specific version. +# See hack/envoy-aws-lc/README.md and charts/wire-ingress/README.md. +# +# make build-envoy-aws-lc-image +# make build-envoy-aws-lc-image PUSH=1 ENVOY_VERSION=v1.39.0 +ENVOY_VERSION ?= v1.38.3 + +.PHONY: build-envoy-aws-lc-image +build-envoy-aws-lc-image: + ENVOY_VERSION=$(ENVOY_VERSION) \ + IMAGE=$(DOCKER_USER)/envoy-aws-lc \ + PUSH=$(PUSH) \ + ./hack/envoy-aws-lc/build.sh + ################################# ## cassandra / postgres management diff --git a/changelog.d/2-features/wire-ingress-tr-02102-2-tls.md b/changelog.d/2-features/wire-ingress-tr-02102-2-tls.md new file mode 100644 index 00000000000..422228182c4 --- /dev/null +++ b/changelog.d/2-features/wire-ingress-tr-02102-2-tls.md @@ -0,0 +1,18 @@ +The `wire-ingress` chart now constrains the TLS parameters Envoy negotiates +(`gateway.tls.*`: TLS versions, TLS 1.2 cipher suites, key agreement groups, +signature algorithms), restoring the BSI TR-02102-2 conformance the nginx +ingress provided via `ssl-protocols` / `ssl-ciphers`. Key agreement now prefers +the hybrid post-quantum group X25519MLKEM768, falling back to P-256/P-384/P-521. + +ALPN, TLS and PROXY protocol settings are now rendered into a single +Gateway-wide `ClientTrafficPolicy`, because Envoy Gateway rejects a second +policy targeting the same Gateway as `Conflicted` instead of merging it. + +`gateway.tls.ecdhCurves` is validated against the crypto library named in the +new `gateway.tls.sslLibrary`, so a group the proxy image cannot offer fails at +template time rather than silently breaking the listener at runtime. + +See the chart README for the conformance gaps that remain: TLS 1.3 cipher suites +are fixed by the crypto library and cannot be restricted by Envoy, and the +hybrid groups TR-02102-2 intends to recommend (SecP256r1MLKEM768 / +SecP384r1MLKEM1024) need an Envoy built against AWS-LC rather than BoringSSL. diff --git a/charts/wire-ingress/README.md b/charts/wire-ingress/README.md index 73433734acc..5aa99e24c78 100644 --- a/charts/wire-ingress/README.md +++ b/charts/wire-ingress/README.md @@ -82,6 +82,13 @@ name overrides, etc.) can be found in `values.yaml`. | `gateway.className` | `""` | **Required.** Name of the `GatewayClass` installed by the Envoy Gateway controller (e.g. `envoy`). Must match the `GatewayClass` object whose `spec.controllerName` is `gateway.envoyproxy.io/gatewayclass-controller`. | | `gateway.alpn.enabled` | `true` | Enables ALPN configuration via `ClientTrafficPolicy` to support HTTP/2 despite overlapping certificate SANs across multiple service listeners. When disabled, ALPN defaults to HTTP/1.1 only. | | `gateway.alpn.protocols` | `[h2, http/1.1]` | List of ALPN protocols to advertise to clients. Defaults to HTTP/2 with HTTP/1.1 fallback. | +| `gateway.tls.enabled` | `true` | Constrains the TLS parameters Envoy will negotiate on every HTTPS listener. Set to `false` to fall back to Envoy's defaults — see [TR-02102-2 conformance](#tr-02102-2-conformance). | +| `gateway.tls.minVersion` | `"1.2"` | Minimum TLS version. One of `Auto`, `"1.0"`, `"1.1"`, `"1.2"`, `"1.3"`. | +| `gateway.tls.maxVersion` | `"1.3"` | Maximum TLS version. Same value set as `minVersion`. | +| `gateway.tls.ciphers` | `[ECDHE-ECDSA-AES256-GCM-SHA384, ECDHE-RSA-AES256-GCM-SHA384]` | Cipher suites offered for **TLS 1.0-1.2 only**. Not rendered when `minVersion` is `"1.3"`, because Envoy Gateway rejects that combination. | +| `gateway.tls.ecdhCurves` | `[X25519MLKEM768, P-256, P-384, P-521]` | Key agreement groups in server preference order, as **BoringSSL** names (not IANA names). The hybrid post-quantum group comes first — see [Post-quantum key agreement](#post-quantum-key-agreement). | +| `gateway.tls.signatureAlgorithms` | `[ecdsa_secp*, rsa_pss_rsae_*]` | Signature algorithms, dropping Envoy's `rsa_pkcs1_*` defaults. Also constrains the federator mTLS handshake — see `values.yaml`. | +| `gateway.tls.sslLibrary` | `boringssl` | Crypto library the proxy image links against (`boringssl`, `aws-lc`, `openssl`). Chart-side validation of `ecdhCurves` only; does not change the rendered policy. See [Getting SecP256r1MLKEM768 / SecP384r1MLKEM1024](#getting-secp256r1mlkem768--secp384r1mlkem1024). | | `gateway.listeners.http.enabled` | `false` | Enables the HTTP listener on port 80. Required for HTTP01 ACME challenges via cert-manager's `gatewayHTTPRoute` solver — see [HTTP01 certificate challenges](#http01-certificate-challenges). | | `gateway.envoyProxy.create` | `true` | If `false`, no `EnvoyProxy` resource is created. Set `gateway.envoyProxy.name` to reference an existing one, or leave it empty to inherit the GatewayClass-level `EnvoyProxy`. | | `gateway.envoyProxy.name` | _(derived)_ | When `create: true` — name of the created resource. When `create: false` — name of an existing `EnvoyProxy` to reference via `infrastructure.parametersRef`. | @@ -89,7 +96,7 @@ name overrides, etc.) can be found in `values.yaml`. | `gateway.manageServiceType` | `true` | Shorthand that sets `envoyService.type` to `gateway.serviceType`. Disable when managing the service type via `gateway.envoyProxy.spec` directly. | | `gateway.serviceType` | `LoadBalancer` | Service type for the Envoy proxy service. Only used when `gateway.manageServiceType: true`. | | `gateway.infrastructure.annotations` | `{}` | Annotations forwarded to the LoadBalancer Service provisioned by Envoy Gateway — see [Gateway API docs](https://gateway-api.sigs.k8s.io/reference/spec/#gateway.networking.k8s.io/v1.GatewayInfrastructure). Use for cloud-specific LB settings (e.g. AWS NLB). | -| `gateway.proxyProtocol.enabled` | `false` | Creates a `ClientTrafficPolicy` enabling PROXY protocol on all listeners. Required when the upstream load balancer is configured to send PROXY protocol headers. | +| `gateway.proxyProtocol.enabled` | `false` | Enables PROXY protocol on all listeners (via the Gateway-wide `ClientTrafficPolicy`). Required when the upstream load balancer is configured to send PROXY protocol headers. | | `gateway.patchPolicies.enabled` | `true` | Controls whether `EnvoyPatchPolicy` resources are created — see [EnvoyPatchPolicy](#envoypatchpolicy). | | `gateway.patchPolicies.targetGatewayClass` | `false` | When `true`, `EnvoyPatchPolicy` targets the `GatewayClass` instead of the `Gateway`. **Required when `gateway.envoyProxy.spec.mergeGateways: true`**: with merged Gateways, policies targeting a `Gateway` are not applied — they must target the `GatewayClass`. Leave `false` for single-Gateway deployments (e.g. integration tests). | | `gateway.controllerNamespace` | `envoy-gateway-system` | Can be ignored, relevant only for integration tests. Namespace where Envoy Gateway runs its proxy pods. Change only if Envoy Gateway was installed into a non-default namespace. | @@ -308,17 +315,243 @@ federator: --- -### HTTP/2 support with ALPN ClientTrafficPolicy +### One Gateway-wide ClientTrafficPolicy -The chart creates a `ClientTrafficPolicy` resource that explicitly configures ALPN protocols when `gateway.alpn.enabled: true`. This is necessary because when a single certificate with multiple SANs is used across multiple listeners on the same port, Envoy would otherwise disable HTTP/2 as a safety measure to prevent connection coalescing attacks. +ALPN, TLS parameters and PROXY protocol are all rendered into a *single* +`ClientTrafficPolicy` (`-client-traffic`), not one resource per concern. + +Envoy Gateway attaches at most one `ClientTrafficPolicy` per target. A second +policy targeting the same `Gateway` is not merged — it is rejected with a +`Conflicted` status condition, and whichever policy lost the race is silently +dropped from the data plane. Section-scoped policies follow the same rule: the +federator listener has its own policy, so for that listener the Gateway-wide one +is marked `Overridden` and does **not** apply. That is why +`clienttrafficpolicy-federator.yaml` repeats the ALPN and TLS settings. + +#### ALPN + +`gateway.alpn.enabled: true` sets `spec.tls.alpnProtocols` explicitly: -The policy sets `spec.tls.alpnProtocols` to: ```yaml - h2 # HTTP/2 - http/1.1 # HTTP/1.1 fallback ``` -This allows HTTP/2 to be negotiated while maintaining support for older clients via HTTP/1.1 fallback. +This is necessary because when a single certificate with multiple SANs is used +across multiple listeners on the same port, Envoy would otherwise disable HTTP/2 +as a safety measure to prevent connection coalescing attacks. Setting it +explicitly allows HTTP/2 to be negotiated while keeping HTTP/1.1 fallback for +older clients. + +### TR-02102-2 conformance + +The nginx ingress restricted TLS to the mechanisms recommended by +[BSI TR-02102-2](https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/Publications/TechGuidelines/TG02102/BSI-TR-02102-2.html) +through `controller.config` in `charts/ingress-nginx-controller`. Under Envoy +Gateway the equivalent knobs live in `gateway.tls` and are rendered into +`spec.tls` of the `ClientTrafficPolicy` resources: + +| nginx setting | Envoy Gateway equivalent | +|---|---| +| `ssl-protocols: "TLSv1.2 TLSv1.3"` | `gateway.tls.minVersion` / `gateway.tls.maxVersion` | +| `ssl-ciphers: "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384"` | `gateway.tls.ciphers` (same two suites) | +| `server-snippet: ssl_conf_command Ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384` | **no equivalent** — see the caveat below | +| _(not restricted in nginx)_ | `gateway.tls.ecdhCurves`, `gateway.tls.signatureAlgorithms` | + +The defaults come from these tables of TR-02102-2 (2026 edition): + +- Table 2 — recommended TLS versions: 1.3 (2032+) and 1.2 (until end of 2031) +- Tables 3/4 — recommended TLS 1.2 cipher suites +- Tables 6/10 — recommended Diffie-Hellman groups. Note X25519, which Envoy + offers by default, is *not* on the BSI list, hence the explicit `ecdhCurves`. +- Tables 11/12 — recommended signature algorithms. Not restricted by default: + Envoy's defaults still include `rsa_pkcs1_*`, which TR-02102-2 only recommended + until the end of 2025. `values.yaml` contains the conformant list to opt into; + make sure it covers your certificate's key type before enabling it. + +#### Caveat: TLS 1.3 cipher suites cannot be restricted + +`gateway.tls.ciphers` maps onto Envoy's +[`TlsParameters.cipher_suites`](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#extensions-transport-sockets-tls-v3-tlsparameters), +which — like OpenSSL's `ssl_ciphers` — "has no effect when negotiating TLS 1.3". +Unlike nginx, Envoy exposes no counterpart to `ssl_conf_command Ciphersuites`: +it only ever calls `SSL_CTX_set_strict_cipher_list`, never BoringSSL's +`SSL_CTX_set_ciphersuites`, so the TLS 1.3 suite list is whatever the linked +BoringSSL offers. Envoy therefore also offers `TLS_CHACHA20_POLY1305_SHA256` +alongside `TLS_AES_128_GCM_SHA256` and `TLS_AES_256_GCM_SHA384` (the two on +TR-02102-2 Table 13). Upstream issue +[envoyproxy/envoy#19548](https://github.com/envoyproxy/envoy/issues/19548) asks +for exactly this and went stale without a resolution. + +A BoringSSL **FIPS** build does *not* help here, despite what the FIPS-specific +defaults suggest: `DEFAULT_CIPHER_SUITES_FIPS` and `DEFAULT_CURVES_FIPS` in +Envoy only change the TLS 1.2 cipher list and the curve list, and BoringSSL's +SSL layer has no FIPS conditional around the TLS 1.3 suite table. ChaCha20 +remains on offer. + +That leaves two real options: + +1. **Accept it and document the deviation.** ChaCha20-Poly1305 is not broken or + deprecated; it is simply absent from the BSI recommendation. This is the + default, and the recommended choice. +2. **Disable TLS 1.3** with `gateway.tls.maxVersion: "1.2"`. Fully conformant on + the cipher list, but gives up TLS 1.3 — which TR-02102-2 itself says "should + be used in preference" — and forecloses post-quantum key agreement, which + exists only in TLS 1.3. Not recommended. + +Closing the gap properly would mean patching BoringSSL's cipher table or +teaching Envoy to call `SSL_CTX_set_ciphersuites`, and running a self-built +proxy image via +`gateway.envoyProxy.spec.provider.kubernetes.envoyDeployment.container.image` +(set `gateway.manageServiceType: false` when doing so — it overwrites the whole +`provider` block). That means owning a fork of Envoy's TLS stack and a Bazel +build across every Envoy Gateway bump; it is not worth it for this one suite. + +### Post-quantum key agreement + +`gateway.tls.ecdhCurves` puts **X25519MLKEM768** ahead of the NIST curves, so +the handshake is hybrid post-quantum wherever the client supports it. This +matters for harvest-now-decrypt-later: a hybrid group derives the shared secret +from both X25519 and ML-KEM-768, so it is at least as strong as X25519 alone +and additionally resists a future quantum attacker. Current Chrome and Firefox +send an X25519MLKEM768 key share in the first ClientHello, so there is no extra +round trip for them; clients that do not offer the group fall back to P-256 / +P-384 / P-521. + +This has to be set explicitly — Envoy's own default is `X25519:P-256`, which is +purely classical, even though the BoringSSL it links against would offer the +hybrid group by default. + +Strictly speaking X25519MLKEM768 is a deviation from TR-02102-2 as written +today, since its classical half is X25519 and no PQ group is listed yet. It is +a deviation in the direction the BSI has already announced it is going, and it +is strictly stronger than the classical fallback. For a deployment that must +match the current text exactly, drop it and keep only the NIST curves: + +```yaml +gateway: + tls: + ecdhCurves: + - P-256 + - P-384 + - P-521 +``` + +#### Getting `SecP256r1MLKEM768` / `SecP384r1MLKEM1024` + +The BSI note under Table 10 says it intends to recommend `SecP256r1MLKEM768` and +`SecP384r1MLKEM1024` (from +[draft-ietf-tls-ecdhe-mlkem](https://datatracker.ietf.org/doc/draft-ietf-tls-ecdhemlkem/)) +once the RFC is adopted. **BoringSSL implements neither**, so they are not +available on the stock `envoyproxy/envoy` image. Its whole group table is +`P-256`, `P-384`, `P-521`, `X25519`, `X25519Kyber768Draft00`, `X25519MLKEM768` +and `MLKEM1024` — the brainpool and `ffdhe*` groups from TR Tables 6/10 are +missing too. + +They *are* available in **AWS-LC**, which Envoy supports as an alternative +crypto library. AWS-LC's group table includes both, under exactly those names, +and Envoy has an upstream build config for it (see +[bazel/SSL.md](https://github.com/envoyproxy/envoy/blob/main/bazel/SSL.md)): + +```bash +bazel build --config=aws-lc-fips //source/exe:envoy-static +``` + +To use it: + +**1. Build the image.** There is no published AWS-LC Envoy image — +`envoyproxy/envoy` has no FIPS or AWS-LC tags at all, and the vendor FIPS images +that do exist (Tetrate, Chainguard) are BoringSSL-FIPS, which does not help +here. [`hack/envoy-aws-lc/build.sh`](../../hack/envoy-aws-lc) drives Envoy's own +build container and asserts that the resulting binary reports `AWS-LC` in +`envoy --version`: + +```bash +cd hack/envoy-aws-lc +ENVOY_VERSION=v1.38.3 PUSH=1 IMAGE=quay.io/wire/envoy-aws-lc ./build.sh +``` + +`ENVOY_VERSION` must match the Envoy your Envoy Gateway ships — it generates +bootstrap config for a specific version. Envoy Gateway v1.8.3 ships +`envoyproxy/envoy:distroless-v1.38.3`. + +**2. Point the Gateway at it and enable the groups.** Note +`manageServiceType: false`: that shorthand overwrites the whole `provider` +block, so once you set `provider` yourself the service type has to move into +`envoyProxy.spec` alongside it. + +```yaml +gateway: + manageServiceType: false + envoyProxy: + create: true + spec: + provider: + type: Kubernetes + kubernetes: + envoyService: + type: LoadBalancer + envoyDeployment: + container: + image: quay.io/wire/envoy-aws-lc:v1.38.3-aws-lc + tls: + sslLibrary: aws-lc + ecdhCurves: + - SecP256r1MLKEM768 + - SecP384r1MLKEM1024 + - P-256 + - P-384 + - P-521 +``` + +If the GatewayClass owns a shared `EnvoyProxy` (`gateway.envoyProxy.create: +false`, as in the `envoy-shared` / `mergeGateways` setup), put the image +override on that cluster-level `EnvoyProxy` instead — it applies to every +Gateway using the class. + +**3. Check what was actually negotiated.** The proxy access log format set by +this chart includes `requested_server_name` but not the cipher, so confirm from +the outside: + +```bash +openssl s_client -connect nginz-https.example.com:443 -groups SecP256r1MLKEM768 &1 \ + | grep -E 'Negotiated TLS1.3 group|Cipher|Protocol' +``` + +(needs OpenSSL 3.5+, which is the client side that knows the group name). The +proxy also exports `envoy_listener_ssl_curves_*` counters on the metrics port, +which is the more practical way to see the mix across real clients. + +`gateway.tls.sslLibrary` only drives this validation — it changes nothing in the +rendered `ClientTrafficPolicy`. It exists because Envoy rejects the entire +listener when `ecdh_curves` holds a name the linked library does not know, and +the only trace is a line in the proxy log; failing at `helm template` time with +an explanation is considerably easier to debug. + +What this costs, before you commit to it: + +- **The Envoy project does not test it.** `bazel/SSL.md` states plainly that + "only the BoringSSL FIPS build on x86_64 is supported and tested by the Envoy + project", and that maintenance of other combinations "remains with downstream + projects". +- **HTTP/3 is disabled** in AWS-LC builds. Irrelevant for this Gateway, which + serves h2 and http/1.1, but worth knowing. +- **You own the build.** Envoy is a multi-hour Bazel build needing ~60G of + disk, and the AWS-LC genrule wants the Bazel-downloaded LLVM toolchain plus + pinned cmake/ninja/go — the very things a Nix derivation strips out, so + `pkgs.envoy` is not a shortcut and this is not part of the nix image set + (nixpkgs builds Envoy with `--config=gcc` and `--repository_disable_download`). +- **You own the version treadmill.** Envoy Gateway v1.8.3 ships + `envoyproxy/envoy:distroless-v1.38.3` and generates bootstrap config for that + version; a custom image has to track it across every Envoy Gateway bump. + +A third option is `--config=openssl` (OpenSSL 3.5 also has both groups), but it +loads OpenSSL dynamically at runtime and is explicitly *not* covered by the +Envoy security policy. Set `sslLibrary: openssl` to skip curve-name validation +if you go that way. + +Note that neither AWS-LC nor OpenSSL fixes the TLS 1.3 ChaCha20 gap above: +AWS-LC has no FIPS conditional around its TLS 1.3 cipher table either. ### Federator mTLS uses Envoy Gateway policies diff --git a/charts/wire-ingress/templates/_helpers.tpl b/charts/wire-ingress/templates/_helpers.tpl index f780f7f35c5..dc7941776c4 100644 --- a/charts/wire-ingress/templates/_helpers.tpl +++ b/charts/wire-ingress/templates/_helpers.tpl @@ -206,3 +206,88 @@ Call with a dict: {https, ssl, base, websockets (bool)}. {{- $csp = printf "%s upgrade-insecure-requests" $csp -}} {{- $csp -}} {{- end -}} + +{{/* +TLS parameters shared by every ClientTrafficPolicy this chart renders. + +These constrain what Envoy will negotiate with clients, and exist so the +deployment can stay conformant with BSI TR-02102-2 ("Cryptographic Mechanisms: +Recommendations and Key Lengths — Part 2: Use of Transport Layer Security"), +which the nginx-based ingress used to enforce via the `ssl-protocols`, +`ssl-ciphers` and `ssl_conf_command Ciphersuites` settings of +`charts/ingress-nginx-controller`. + +Emits the `minVersion` / `maxVersion` / `ciphers` / `ecdhCurves` / +`signatureAlgorithms` keys of an Envoy Gateway `ClientTrafficPolicy` +`spec.tls`, unindented. Call with the root context and `nindent` the result. + +Renders nothing when `gateway.tls.enabled` is false; callers guard on that so +they do not emit a stray blank line. +*/}} +{{- define "wire-ingress.tlsParameters" -}} +{{- $tls := .Values.gateway.tls -}} +{{- if $tls.enabled -}} +{{- $minVersion := $tls.minVersion | default "" | toString -}} +{{- $lib := $tls.sslLibrary | default "boringssl" -}} +{{- if not (has $lib (list "boringssl" "aws-lc" "openssl")) -}} +{{- fail (printf "gateway.tls.sslLibrary: %q is not one of boringssl, aws-lc, openssl" $lib) -}} +{{- end -}} +{{- if $minVersion }} +minVersion: {{ $minVersion | quote }} +{{- end }} +{{- if $tls.maxVersion }} +maxVersion: {{ $tls.maxVersion | toString | quote }} +{{- end }} +{{- /* +`ciphers` only applies to TLS 1.0-1.2 — TLS 1.3 suites are fixed by BoringSSL +and cannot be selected. Envoy Gateway rejects the resource outright (CEL +validation) if `ciphers` is set alongside `minVersion: "1.3"`, so drop it. +*/ -}} +{{- if and $tls.ciphers (ne $minVersion "1.3") }} +ciphers: + {{- range $tls.ciphers }} + - {{ . | quote }} + {{- end }} +{{- end }} +{{- if $tls.ecdhCurves }} +{{- /* +Envoy joins this list with ":" and hands it to SSL_CTX_set1_curves_list; if the +linked crypto library does not know a name, the whole listener is rejected and +the only trace is a line in the proxy log. Which names exist depends on how the +proxy image was built, so validate against the library named in +gateway.tls.sslLibrary. Unknown names are passed through — new groups appear +faster than this chart is updated — but a name that some OTHER library supports +is almost certainly a mismatch between the value and the running image. +*/ -}} +{{- $groups := dict + "boringssl" (list "P-224" "P-256" "P-384" "P-521" "X25519" "X25519Kyber768Draft00" "X25519MLKEM768" "MLKEM1024") + "aws-lc" (list "P-224" "P-256" "P-384" "P-521" "X25519" "SecP256r1MLKEM768" "X25519MLKEM768" "SecP384r1MLKEM1024" "MLKEM512" "MLKEM768" "MLKEM1024") -}} +{{- if hasKey $groups $lib }} +{{- $ok := index $groups $lib }} +{{- $anyLib := concat (index $groups "boringssl") (index $groups "aws-lc") }} +{{- range $curve := $tls.ecdhCurves }} +{{- if and (has $curve $anyLib) (not (has $curve $ok)) }} +{{- fail (printf "gateway.tls.ecdhCurves: %q is not implemented by %s, which gateway.tls.sslLibrary says the proxy image links against — Envoy would reject the listener at config load. SecP256r1MLKEM768 and SecP384r1MLKEM1024, the hybrid groups TR-02102-2 intends to recommend, exist only in AWS-LC: run an Envoy built with `--config=aws-lc-fips` and set gateway.tls.sslLibrary: aws-lc. On the stock BoringSSL image the only hybrid available is X25519MLKEM768. See the chart README." $curve $lib) }} +{{- end }} +{{- if not (has $curve $anyLib) }} +{{- range $known := $anyLib }} +{{- if eq (lower $curve) (lower $known) }} +{{- fail (printf "gateway.tls.ecdhCurves: %q is spelled wrong — these are crypto-library group names and are case sensitive. Use %q. (Note the NIST curves are P-256 / P-384 / P-521, not secp256r1 / secp384r1 / secp521r1.)" $curve $known) }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +ecdhCurves: + {{- range $tls.ecdhCurves }} + - {{ . | quote }} + {{- end }} +{{- end }} +{{- if $tls.signatureAlgorithms }} +signatureAlgorithms: + {{- range $tls.signatureAlgorithms }} + - {{ . | quote }} + {{- end }} +{{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/wire-ingress/templates/clienttrafficpolicy-alpn.yaml b/charts/wire-ingress/templates/clienttrafficpolicy-alpn.yaml deleted file mode 100644 index 4a63f7060a3..00000000000 --- a/charts/wire-ingress/templates/clienttrafficpolicy-alpn.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- if .Values.gateway.alpn.enabled }} -{{/* Envoy Gateway-specific (gateway.envoyproxy.io/v1alpha1). - Configures ALPN to allow HTTP/2 despite overlapping certificate SANs - when using multiple listeners with the same certificate. */}} -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: ClientTrafficPolicy -metadata: - name: {{ include "wire-ingress.gatewayName" . }}-alpn - namespace: {{ .Release.Namespace }} - labels: - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -spec: - targetRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: {{ include "wire-ingress.gatewayName" . | quote }} - tls: - alpnProtocols: - {{- range .Values.gateway.alpn.protocols }} - - {{ . }} - {{- end }} -{{- end }} diff --git a/charts/wire-ingress/templates/clienttrafficpolicy-federator.yaml b/charts/wire-ingress/templates/clienttrafficpolicy-federator.yaml index f51a72058cb..5db6aee5abd 100644 --- a/charts/wire-ingress/templates/clienttrafficpolicy-federator.yaml +++ b/charts/wire-ingress/templates/clienttrafficpolicy-federator.yaml @@ -1,6 +1,13 @@ {{- if .Values.federator.enabled }} {{/* Envoy Gateway-specific (gateway.envoyproxy.io/v1alpha1). - Enforces mTLS client certificate validation on the federator listener only. */}} + Enforces mTLS client certificate validation on the federator listener only. + + A section-scoped policy fully REPLACES the Gateway-wide one for its + listener (Envoy Gateway marks the Gateway-wide policy `Overridden` for this + section rather than merging field-by-field), so the ALPN and TLS parameters + from clienttrafficpolicy-gateway.yaml are repeated here. Without this the + federator listener would silently fall back to Envoy's defaults, which + include cipher suites that BSI TR-02102-2 does not recommend. */}} apiVersion: gateway.envoyproxy.io/v1alpha1 kind: ClientTrafficPolicy metadata: @@ -25,4 +32,13 @@ spec: caCertificateRefs: - name: federator-ca kind: ConfigMap + {{- if .Values.gateway.alpn.enabled }} + alpnProtocols: + {{- range .Values.gateway.alpn.protocols }} + - {{ . }} + {{- end }} + {{- end }} + {{- if .Values.gateway.tls.enabled }} + {{- include "wire-ingress.tlsParameters" . | trim | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/wire-ingress/templates/clienttrafficpolicy-gateway.yaml b/charts/wire-ingress/templates/clienttrafficpolicy-gateway.yaml new file mode 100644 index 00000000000..d3615404955 --- /dev/null +++ b/charts/wire-ingress/templates/clienttrafficpolicy-gateway.yaml @@ -0,0 +1,48 @@ +{{- $alpn := .Values.gateway.alpn.enabled -}} +{{- $tls := .Values.gateway.tls.enabled -}} +{{- $proxyProtocol := .Values.gateway.proxyProtocol.enabled -}} +{{- if or $alpn $tls $proxyProtocol }} +{{/* Envoy Gateway-specific (gateway.envoyproxy.io/v1alpha1). + + Gateway-wide client-side settings: ALPN, TLS parameters and PROXY protocol. + + These all live in ONE resource on purpose. Envoy Gateway attaches at most a + single ClientTrafficPolicy per target: a second policy targeting the same + Gateway is rejected with `Conflicted` rather than merged. Splitting these + into separate resources would silently drop whichever one lost the race. + + Listeners that have their own section-scoped ClientTrafficPolicy (the + federator listener) do NOT inherit this one — Envoy Gateway marks this + policy `Overridden` for those sections — so that policy repeats the same + ALPN and TLS settings. */}} +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: ClientTrafficPolicy +metadata: + name: {{ include "wire-ingress.gatewayName" . }}-client-traffic + namespace: {{ .Release.Namespace }} + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: {{ include "wire-ingress.gatewayName" . | quote }} + {{- if $proxyProtocol }} + proxyProtocol: + optional: {{ .Values.gateway.proxyProtocol.optional }} + {{- end }} + {{- if or $alpn $tls }} + tls: + {{- if $alpn }} + alpnProtocols: + {{- range .Values.gateway.alpn.protocols }} + - {{ . }} + {{- end }} + {{- end }} + {{- if $tls }} + {{- include "wire-ingress.tlsParameters" . | trim | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/wire-ingress/templates/clienttrafficpolicy-proxy-protocol.yaml b/charts/wire-ingress/templates/clienttrafficpolicy-proxy-protocol.yaml deleted file mode 100644 index ed856304088..00000000000 --- a/charts/wire-ingress/templates/clienttrafficpolicy-proxy-protocol.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if .Values.gateway.proxyProtocol.enabled }} -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: ClientTrafficPolicy -metadata: - name: {{ include "wire-ingress.gatewayName" . }}-proxy-protocol - namespace: {{ .Release.Namespace }} - labels: - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -spec: - targetRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: {{ include "wire-ingress.gatewayName" . | quote }} - proxyProtocol: - optional: {{ .Values.gateway.proxyProtocol.optional }} -{{- end }} diff --git a/charts/wire-ingress/values.yaml b/charts/wire-ingress/values.yaml index bcd14ac911b..9d407a39503 100644 --- a/charts/wire-ingress/values.yaml +++ b/charts/wire-ingress/values.yaml @@ -85,6 +85,100 @@ gateway: # Optional hostname restriction for the HTTP listener. Set alongside # listeners.https.hostname when using mergeGateways. hostname: "" + # TLS parameters applied to every HTTPS listener of the Gateway (including the + # federator listener). Rendered into the ClientTrafficPolicy resources. + # + # The defaults restrict Envoy to the mechanisms recommended by BSI TR-02102-2 + # ("Cryptographic Mechanisms: Recommendations and Key Lengths — Part 2: Use of + # Transport Layer Security"), matching what the nginx ingress enforced via + # ssl-protocols / ssl-ciphers in charts/ingress-nginx-controller. + # + # CAVEAT: Envoy cannot restrict TLS 1.3 cipher suites — they are fixed by + # BoringSSL and `ciphers` below only applies to TLS 1.0-1.2. A stock Envoy + # build therefore also offers TLS_CHACHA20_POLY1305_SHA256, which is not on + # the TR-02102-2 list. See the "TR-02102-2 conformance" section of the README + # for the two ways to close that gap. + tls: + # Set to false to leave all TLS parameters at Envoy's defaults. + enabled: true + # TR-02102-2 Table 2: TLS 1.3 (2032+) and TLS 1.2 (until end of 2031) are + # recommended; TLS 1.0/1.1 are not. Valid: Auto, "1.0", "1.1", "1.2", "1.3". + minVersion: "1.2" + maxVersion: "1.3" + # TR-02102-2 Table 3/4: recommended cipher suites for TLS 1.2. + # Ignored when minVersion is "1.3" (Envoy Gateway rejects the combination). + # The AES-128 counterparts (ECDHE-{ECDSA,RSA}-AES128-GCM-SHA256) are equally + # conformant and can be added here for broader client compatibility. + ciphers: + - ECDHE-ECDSA-AES256-GCM-SHA384 + - ECDHE-RSA-AES256-GCM-SHA384 + # Key agreement groups, in server preference order. These are BoringSSL + # group names, NOT IANA names — Envoy passes the list straight to + # SSL_CTX_set1_curves_list and refuses to start the listener if any name is + # unknown. + # + # X25519MLKEM768 first makes the handshake post-quantum: it is a hybrid of + # X25519 and ML-KEM-768, so it is at least as strong as X25519 alone and + # protects today's traffic against harvest-now-decrypt-later. PQ-capable + # clients (current Chrome and Firefox) send an X25519MLKEM768 key share in + # the first ClientHello, so this costs no extra round trip for them; clients + # that do not offer it simply fall back to the NIST curves below. + # + # The two groups TR-02102-2 names as its intended future recommendation — + # SecP256r1MLKEM768 and SecP384r1MLKEM1024 — are NOT available on the stock + # envoyproxy/envoy image, because BoringSSL does not implement them. They do + # exist in AWS-LC, which Envoy supports as an alternative crypto library + # (`bazel build --config=aws-lc-fips`). To use them, build that image, point + # gateway.envoyProxy.spec at it, and set sslLibrary: aws-lc below. See the + # README for what that costs. Otherwise the chart fails rendering with an + # explanatory message if you put those names here. + # + # For strict TR-02102-2 conformance today, drop X25519MLKEM768 and keep only + # the NIST curves: Tables 6 and 10 list secp256r1/384r1/521r1 (and the + # brainpool and ffdhe groups, which BoringSSL also does not implement), and + # no PQ group. X25519 — an Envoy default — is not on the BSI list either, + # which is why it is absent below as a standalone group. + ecdhCurves: + - X25519MLKEM768 + - P-256 + - P-384 + - P-521 + # Crypto library the Envoy proxy image is built against. Purely a chart-side + # validation hint: it decides which ecdhCurves names are accepted, and has + # no effect on the rendered ClientTrafficPolicy. Leave at boringssl unless + # you deploy a custom proxy image. + # boringssl - stock envoyproxy/envoy (default) + # aws-lc - built with --config=aws-lc-fips; adds SecP256r1MLKEM768, + # SecP384r1MLKEM1024, MLKEM512/768; note HTTP/3 is disabled in + # AWS-LC builds (irrelevant here, the Gateway serves h2 and + # http/1.1) and that this build is not covered by the Envoy + # project's own test matrix + # openssl - built with --config=openssl; curve names are not validated + sslLibrary: boringssl + # TR-02102-2 Tables 11 and 12, intersected with what BoringSSL accepts. + # Dropping Envoy's rsa_pkcs1_* defaults is the point: TR-02102-2 only + # recommended PKCS #1 v1.5 padding until the end of 2025. + # + # This list covers both certificates this chart can issue — ECDSA P-384 for + # the main cert (tls.privateKey below) and ECDSA P-256 for the federator + # cert — plus RSA-PSS for externally supplied RSA certificates. + # + # Only these names are valid; anything else makes Envoy reject the listener + # config at load time. In particular the TR also lists rsa_pss_pss_* and the + # ecdsa_brainpool* algorithms, which BoringSSL does not implement. + # + # Two things to check before changing this list: + # - it must cover the key type of your server certificate; + # - it also constrains the federator mTLS handshake, so remote backends + # presenting RSA client certificates over TLS 1.2 need rsa_pkcs1_sha256 + # added back unless their certificates support PSS. + signatureAlgorithms: + - ecdsa_secp256r1_sha256 + - ecdsa_secp384r1_sha384 + - ecdsa_secp521r1_sha512 + - rsa_pss_rsae_sha256 + - rsa_pss_rsae_sha384 + - rsa_pss_rsae_sha512 # ALPN configuration for HTTP/2 support with multiple listeners. # When using multiple listeners with overlapping certificate SANs, # Envoy Gateway may downgrade to HTTP/1.1. This ClientTrafficPolicy diff --git a/hack/envoy-aws-lc/Dockerfile b/hack/envoy-aws-lc/Dockerfile new file mode 100644 index 00000000000..00ef805b65d --- /dev/null +++ b/hack/envoy-aws-lc/Dockerfile @@ -0,0 +1,24 @@ +# Envoy proxy image linked against AWS-LC instead of BoringSSL. +# +# Built by ./build.sh, which produces the `envoy-static` binary in this +# directory first. See charts/wire-ingress/README.md for why. +# +# Layout mirrors what Envoy Gateway expects of the stock +# envoyproxy/envoy:distroless-* image: the binary must be reachable as `envoy` +# on PATH (Envoy Gateway sets `command: ["envoy"]`), and the container runs as +# uid/gid 65532 (`runAsNonRoot: true`, `runAsUser: 65532`). +# +# distroless `base` gives glibc (envoy-static is static apart from libc), +# ca-certificates and tzdata. The `nonroot` tag already defaults to 65532. +ARG BASE_IMAGE=gcr.io/distroless/base-debian12:nonroot +FROM ${BASE_IMAGE} + +COPY --chown=65532:65532 envoy-static /usr/local/bin/envoy + +# Set explicitly rather than relying on the base image's PATH: Envoy Gateway +# invokes the bare name `envoy`, so an unset PATH would be a CrashLoopBackOff +# with a confusing message. +ENV PATH=/usr/local/bin:/usr/bin:/bin + +USER 65532:65532 +ENTRYPOINT ["/usr/local/bin/envoy"] diff --git a/hack/envoy-aws-lc/README.md b/hack/envoy-aws-lc/README.md new file mode 100644 index 00000000000..98bc286d00c --- /dev/null +++ b/hack/envoy-aws-lc/README.md @@ -0,0 +1,96 @@ +# Envoy proxy image with AWS-LC + +Builds an Envoy proxy image linked against [AWS-LC](https://github.com/aws/aws-lc) +instead of BoringSSL, so that the Gateway can offer the hybrid post-quantum key +agreement groups `SecP256r1MLKEM768` and `SecP384r1MLKEM1024`. + +Those are the two groups BSI TR-02102-2 names as its intended recommendation +once [draft-ietf-tls-ecdhe-mlkem](https://datatracker.ietf.org/doc/draft-ietf-tls-ecdhemlkem/) +becomes an RFC. BoringSSL implements neither, so on a stock +`envoyproxy/envoy` image Envoy rejects the listener if they appear in +`ecdh_curves`. AWS-LC implements both, and Envoy supports it upstream through +`bazel build --config=aws-lc-fips` (see +[bazel/SSL.md](https://github.com/envoyproxy/envoy/blob/main/bazel/SSL.md)). + +```bash +./build.sh # build and tag locally +PUSH=1 ./build.sh # build, then push +``` + +Or from the repo root: `make build-envoy-aws-lc-image [PUSH=1]`. + +`build-in-container.sh` is the part that runs inside Envoy's build container. It +is a separate file on purpose: `ci/run_envoy_docker.sh` forwards the command +through an unquoted `exec ${DOCKER_COMMAND}`, so anything containing spaces is +word-split into argv and `&&` reaches bazel as a literal target — which surfaces +as `ERROR: no such target '//: '`. Only a single-token script path survives. + +| Variable | Default | Purpose | +|---|---|---| +| `ENVOY_VERSION` | `v1.38.3` | Envoy tag to build; must match your Envoy Gateway | +| `IMAGE` / `TAG` | `quay.io/wire/envoy-aws-lc` / `-aws-lc` | Image coordinates | +| `PUSH` | `0` | Push after a successful build and verification | +| `WORK_DIR` | `/var/tmp/envoy-aws-lc` | Checkout and Bazel output base; keep it off tmpfs | +| `MIN_DISK_GB` | `60` | Free-space precheck; `0` disables it | +| `BAZEL_BUILD_EXTRA_OPTIONS` | _(empty)_ | Extra bazel flags. Use if the build OOMs: `--jobs=8 --local_ram_resources=HOST_RAM*.5` | + +## Iterating on the build + +Keep a log so failures can be read back: + +```bash +set -o pipefail +make build-envoy-aws-lc-image 2>&1 | tee envoy-build.log +``` + +For a hands-off loop, let `entr` rebuild whenever the build scripts change. `-r` +kills an in-flight build as soon as a fix lands, so you never wait on a run that +is already known-bad: + +```bash +ls hack/envoy-aws-lc/build.sh \ + hack/envoy-aws-lc/build-in-container.sh \ + hack/envoy-aws-lc/Dockerfile \ + Makefile \ + | entr -cr bash -c 'set -o pipefail; make build-envoy-aws-lc-image 2>&1 | tee envoy-build.log' +``` + +Bazel state lives under `WORK_DIR` and survives between runs, so only the first +cycle pays the full build cost. `envoy-build.log` is gitignored. + +Two caveats: `entr -r` SIGTERMs the child, which can leave an orphaned +`ci-envoy-build-run-*` container behind — `docker ps` and clean up if a rerun +complains. And `entr` stops watching if the file list itself changes, so restart +it after adding a file here. + +The full rationale, the chart wiring, and what this costs you in maintenance is +in the [wire-ingress chart README](../../charts/wire-ingress/README.md#getting-secp256r1mlkem768--secp384r1mlkem1024). + +## Before you run it + +- **Linux, x86_64 / aarch64 / ppc64le.** The Envoy build container is Linux-only + and AWS-LC builds are limited to those architectures. +- **Hours, and roughly 60G of disk.** This is a full Bazel build of Envoy. + Override `WORK_DIR` if `/var/tmp` is small. +- **`ENVOY_VERSION` must match your Envoy Gateway.** Envoy Gateway generates + bootstrap config for a specific Envoy version. Read it off the running proxy: + + ```bash + kubectl -n envoy-gateway-system get deploy \ + -l gateway.envoyproxy.io/owning-gateway-name \ + -o jsonpath='{.items[0].spec.template.spec.containers[?(@.name=="envoy")].image}' + ``` + + Envoy Gateway v1.8.3 ships `envoyproxy/envoy:distroless-v1.38.3`, which is the + script's default. You own this pin from here on: it has to be rebuilt for + every Envoy Gateway bump. +- **This build is not covered by the Envoy project's test matrix.** + `bazel/SSL.md` states that only the BoringSSL FIPS build on x86_64 is + "supported and tested by the Envoy project", and that other combinations are + the responsibility of downstream projects. HTTP/3 is also disabled in AWS-LC + builds — irrelevant for this Gateway, which serves h2 and http/1.1. + +Deliberately not a nix image: the AWS-LC genrule wants Bazel's own downloaded +LLVM toolchain plus pinned cmake/ninja/go, which is precisely what nixpkgs' +Envoy derivation patches out (it builds with `--config=gcc` and +`--repository_disable_download`). diff --git a/hack/envoy-aws-lc/build-in-container.sh b/hack/envoy-aws-lc/build-in-container.sh new file mode 100755 index 00000000000..6e4684fd864 --- /dev/null +++ b/hack/envoy-aws-lc/build-in-container.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Runs INSIDE Envoy's build container, invoked by ../build.sh. Not meant to be +# run directly on a host. +# +# This exists as its own file rather than as a command string passed to +# ci/run_envoy_docker.sh because that script forwards the command through +# `exec ${DOCKER_COMMAND}` — an UNQUOTED expansion (see ci/docker-compose.yml). +# The result is word-split into argv, and shell operators are never re-parsed +# from an expansion, so a `cmd1 && cmd2` string is handed to cmd1 with `&&` as a +# literal argument. Bazel then fails with a baffling +# ERROR: no such target '//: ': target ' ' not declared in package '' +# Passing a single-token script path is the only thing that survives intact. +# +set -euo pipefail + +# ci/docker-compose.yml starts us in ENVOY_DOCKER_SOURCE_DIR (default /source), +# which is the mounted Envoy checkout. +OUT="${PWD}/envoy-static" + +# Escape hatch for tuning the build without editing this file. Envoy's +# ci/docker-compose.yml already forwards BAZEL_BUILD_EXTRA_OPTIONS into the +# container, so exporting it on the host is enough. Typical use is capping +# resources when a big machine OOMs partway through: +# BAZEL_BUILD_EXTRA_OPTIONS='--jobs=8 --local_ram_resources=HOST_RAM*.5' +read -r -a extra_flags <<<"${BAZEL_BUILD_EXTRA_OPTIONS:-}" + +# @aws_lc//:ninja_bin bootstraps ninja from source, and ninja's configure.py +# compiles it with ${CXX:-c++}. Bazel runs actions with a restricted PATH +# (/bin:/usr/bin:/usr/local/bin), and the Envoy build image has no `c++` at all: +# it installs g++-13 but only aliases `gcc` via update-alternatives, and keeps +# clang in /opt/llvm/bin, which is off the action PATH. The bootstrap therefore +# dies with "c++: not found" long before anything of ours compiles. +# +# Bazel reports this genrule as "[for tool]", so it runs in the EXEC +# configuration. That makes --host_action_env the right knob, and it must be the +# ONLY one: do not also pass --action_env=CXX. +# +# --action_env applies to target-configuration actions, which is where +# rules_foreign_cc builds libevent and friends. Those run under Envoy's Clang +# toolchain (libc++, lld, a sysroot), and injecting a GNU compiler there makes +# CMake's try_compile fail to link: +# Foreign Cc - CMake: Building event failed ... make: *** [cmTC_xxxx/fast] Error 2 +# The exec configuration has no such toolchain expectations, so scoping the +# override to it fixes ninja without disturbing anything else. The AWS-LC build +# genrule is unaffected either way — bazel/external/aws_lc.genrule_cmd pins its +# compilers with a CMake toolchain file pointing at the Bazel-provided LLVM. +resolve_cxx() { + local c + for c in c++ g++ clang++; do + if command -v "$c" >/dev/null 2>&1; then command -v "$c"; return 0; fi + done + for c in "${LLVM_ROOT:-/opt/llvm}/bin/clang++" /usr/bin/g++-*; do + if [[ -x "$c" ]]; then printf '%s\n' "$c"; return 0; fi + done + return 1 +} + +cxx_bin="${CXX:-}" +if [[ -z "$cxx_bin" ]]; then + cxx_bin="$(resolve_cxx)" || { + echo "error: no C++ compiler found in the build container; set CXX explicitly" >&2 + exit 1 + } +fi +echo "using CXX=${cxx_bin} for the ninja bootstrap" + +# --config=clang is NOT optional, even though bazel/SSL.md's example omits it. +# Per bazel/README.md, with no toolchain config Bazel "uses system default +# compiler settings". That half-works: Envoy's own C++ still compiles with the +# hermetic LLVM, but two things from --config=clang go missing — +# clang-common: --linkopt=-fuse-ld=lld +# libc++: --action_env=LDFLAGS="-stdlib=libc++ -fuse-ld=lld" +# and without them rules_foreign_cc's CMake probe links with the system /bin/ld +# instead of lld. That mixes the hermetic sysroot's pre-2.34 Scrt1.o with the +# host's newer glibc and dies in CMake's compiler test, long before libevent +# itself is built: +# undefined reference to `__libc_csu_init' / `__libc_csu_fini' +# Foreign Cc - CMake: Building event failed ... make: *** [cmTC_xxxxx/fast] Error 2 +bazel build --config=clang --config=aws-lc-fips -c opt \ + --host_action_env=CXX="$cxx_bin" \ + "${extra_flags[@]}" //source/exe:envoy-static + +# bazel-bin is a symlink into the /build mount; cp dereferences it, so the +# binary lands on the host through the /source bind mount. +cp -f bazel-bin/source/exe/envoy-static "$OUT" +chmod 0755 "$OUT" + +echo "wrote ${OUT}" diff --git a/hack/envoy-aws-lc/build.sh b/hack/envoy-aws-lc/build.sh new file mode 100755 index 00000000000..7974aebabba --- /dev/null +++ b/hack/envoy-aws-lc/build.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# +# Build an Envoy proxy image linked against AWS-LC instead of BoringSSL. +# +# WHY: BSI TR-02102-2 names SecP256r1MLKEM768 and SecP384r1MLKEM1024 as the +# hybrid post-quantum key agreement groups it intends to recommend. BoringSSL — +# what the stock envoyproxy/envoy image links against — implements neither, so +# Envoy rejects the listener if you put those names in ecdhCurves. AWS-LC +# implements both, and Envoy supports it upstream via `--config=aws-lc-fips`. +# See the "Post-quantum key agreement" section of charts/wire-ingress/README.md. +# +# This is deliberately NOT part of the nix image set: the AWS-LC genrule wants +# Bazel's own downloaded LLVM toolchain plus pinned cmake/ninja/go, which is +# exactly what nixpkgs' Envoy derivation patches out. So we drive Envoy's own +# build container instead, the same way the Envoy project does. +# +# Expect a multi-hour build and tens of GB of disk on first run. +# +# Usage: +# ./build.sh # build and tag locally +# PUSH=1 ./build.sh # build, then push +# ENVOY_VERSION=v1.39.0 ./build.sh +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# MUST match the Envoy version your Envoy Gateway ships, because Envoy Gateway +# generates bootstrap config for a specific Envoy version. To find it: +# kubectl -n envoy-gateway-system get deploy -l gateway.envoyproxy.io/owning-gateway-name \ +# -o jsonpath='{.items[0].spec.template.spec.containers[?(@.name=="envoy")].image}' +# Envoy Gateway v1.8.3 ships envoyproxy/envoy:distroless-v1.38.3. +ENVOY_VERSION="${ENVOY_VERSION:-v1.38.3}" + +IMAGE="${IMAGE:-quay.io/wire/envoy-aws-lc}" +TAG="${TAG:-${ENVOY_VERSION}-aws-lc}" +PUSH="${PUSH:-0}" + +# Both of these get large (the Bazel output base alone is tens of GB), so keep +# them off tmpfs. Override if /var/tmp is small on your machine. +WORK_DIR="${WORK_DIR:-/var/tmp/envoy-aws-lc}" +CHECKOUT_DIR="${CHECKOUT_DIR:-${WORK_DIR}/src}" +export ENVOY_DOCKER_BUILD_DIR="${ENVOY_DOCKER_BUILD_DIR:-${WORK_DIR}/build}" + +# Forwarded into the build container by Envoy's ci/docker-compose.yml and +# applied to the bazel invocation by build-in-container.sh. Use it to cap +# resources if the build OOMs, e.g. +# BAZEL_BUILD_EXTRA_OPTIONS='--jobs=8 --local_ram_resources=HOST_RAM*.5' +export BAZEL_BUILD_EXTRA_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS:-}" + +log() { printf '\n\033[1m==> %s\033[0m\n' "$*"; } +die() { printf '\033[1;31merror:\033[0m %s\n' "$*" >&2; exit 1; } + +# --- preflight --------------------------------------------------------------- + +command -v docker >/dev/null || die "docker is required" +command -v git >/dev/null || die "git is required" +docker info >/dev/null 2>&1 || die "cannot talk to the docker daemon" +# run_envoy_docker.sh drives `docker compose`, not the standalone docker-compose. +docker compose version >/dev/null 2>&1 || die "the docker compose plugin is required (ci/run_envoy_docker.sh uses 'docker compose')" + +arch="$(uname -m)" +case "$arch" in + x86_64|aarch64|ppc64le) ;; + *) die "AWS-LC builds of Envoy support x86_64, aarch64 and ppc64le only (found: $arch)" ;; +esac + +[[ "$(uname -s)" == Linux ]] || die "the Envoy build container only runs on Linux (found: $(uname -s))" + +mkdir -p "$ENVOY_DOCKER_BUILD_DIR" + +# A rough estimate of the Bazel output base plus the container image. Override +# MIN_DISK_GB=0 to skip the check (e.g. network storage that df misreports). +MIN_DISK_GB="${MIN_DISK_GB:-60}" +avail_gb="$(df -BG --output=avail "$ENVOY_DOCKER_BUILD_DIR" | tail -1 | tr -dc '0-9')" +if [[ -n "$avail_gb" && "$avail_gb" -lt "$MIN_DISK_GB" ]]; then + die "only ${avail_gb}G free at ${ENVOY_DOCKER_BUILD_DIR}; the build needs roughly ${MIN_DISK_GB}G. Set WORK_DIR to a bigger volume, or MIN_DISK_GB=0 to skip this check." +fi + +# --- source ------------------------------------------------------------------ + +if [[ -d "${CHECKOUT_DIR}/.git" ]]; then + log "Reusing checkout at ${CHECKOUT_DIR}" + git -C "$CHECKOUT_DIR" fetch --depth 1 origin "refs/tags/${ENVOY_VERSION}:refs/tags/${ENVOY_VERSION}" 2>/dev/null || true + git -C "$CHECKOUT_DIR" checkout --force "$ENVOY_VERSION" +else + log "Cloning envoy ${ENVOY_VERSION} into ${CHECKOUT_DIR}" + mkdir -p "$(dirname "$CHECKOUT_DIR")" + git clone --depth 1 --branch "$ENVOY_VERSION" https://github.com/envoyproxy/envoy "$CHECKOUT_DIR" +fi + +# Guard against a silent no-op: if upstream ever renames the config, bazel would +# fall back to a plain BoringSSL build and we would ship a normal Envoy. +grep -q 'common:aws-lc-fips' "${CHECKOUT_DIR}/.bazelrc" \ + || die "envoy ${ENVOY_VERSION} has no 'aws-lc-fips' config in .bazelrc — check bazel/SSL.md for the current flag" + +# --- build ------------------------------------------------------------------- + +log "Building envoy-static with --config=aws-lc-fips (this takes hours)" +# The command MUST be a single token: ci/run_envoy_docker.sh forwards it via an +# unquoted `exec ${DOCKER_COMMAND}`, so anything with spaces gets word-split +# into argv and `&&` is passed to bazel as a literal target. Hence the separate +# script rather than an inline `cmd1 && cmd2`. +install -m 0755 "${SCRIPT_DIR}/build-in-container.sh" "${CHECKOUT_DIR}/wire-build-aws-lc.sh" +( + cd "$CHECKOUT_DIR" + ./ci/run_envoy_docker.sh ./wire-build-aws-lc.sh +) + +[[ -f "${CHECKOUT_DIR}/envoy-static" ]] || die "build finished but ${CHECKOUT_DIR}/envoy-static is missing" + +# --- image ------------------------------------------------------------------- + +log "Building image ${IMAGE}:${TAG}" +cp -f "${CHECKOUT_DIR}/envoy-static" "${SCRIPT_DIR}/envoy-static" +trap 'rm -f "${SCRIPT_DIR}/envoy-static"' EXIT +docker build -t "${IMAGE}:${TAG}" "$SCRIPT_DIR" + +# --- verify ------------------------------------------------------------------ + +log "Verifying the crypto library" +version_output="$(docker run --rm --entrypoint /usr/local/bin/envoy "${IMAGE}:${TAG}" --version)" +echo "$version_output" +grep -q 'AWS-LC' <<<"$version_output" \ + || die "expected 'AWS-LC' in 'envoy --version', got: ${version_output}. The build silently fell back to another SSL library." + +if [[ "$PUSH" == "1" ]]; then + log "Pushing ${IMAGE}:${TAG}" + docker push "${IMAGE}:${TAG}" +fi + +cat <