From 4cf2eeea78616e40d7cbbd97e6d49dec3621404b Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Tue, 1 Sep 2026 18:02:42 -0500 Subject: [PATCH 1/4] e2e: adapt suites to the agentgateway dataplane Add e2e.RouterIsAgentgateway, detected from the atenet-router Deployment's containers, and gate the Envoy-only assertions on it: the h2-to-h1 downgrade contract is Envoy's protocol mirroring to atunnel, which agentgateway does not implement. --- internal/e2e/dataplane.go | 56 +++++++++++++++++++ .../e2e/suites/networking/grpcingress_test.go | 3 + 2 files changed, 59 insertions(+) create mode 100644 internal/e2e/dataplane.go diff --git a/internal/e2e/dataplane.go b/internal/e2e/dataplane.go new file mode 100644 index 0000000000..39657171da --- /dev/null +++ b/internal/e2e/dataplane.go @@ -0,0 +1,56 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "sync" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + routerDataplaneOnce sync.Once + routerIsAgentgateway bool + routerDataplaneErr error +) + +// RouterIsAgentgateway reports whether the atenet-router Deployment runs the +// agentgateway dataplane. In that mode the pod has no Envoy and no +// atenet-router ext_proc process, so router-internal surfaces — the statusz +// page, atenet_router_* metrics, and Envoy's protocol mirroring to atunnel — +// do not exist. Suites gate assertions on those surfaces with this instead of +// a per-lane env knob: the deployed containers are the source of truth. +func RouterIsAgentgateway(ctx context.Context, t *testing.T) bool { + t.Helper() + routerDataplaneOnce.Do(func() { + deploy, err := GetClients().K8s.AppsV1().Deployments(routerNamespace).Get(ctx, routerService, metav1.GetOptions{}) + if err != nil { + routerDataplaneErr = err + return + } + for _, c := range deploy.Spec.Template.Spec.Containers { + if c.Name == "agentgateway" { + routerIsAgentgateway = true + return + } + } + }) + if routerDataplaneErr != nil { + t.Fatalf("detecting the router dataplane: %v", routerDataplaneErr) + } + return routerIsAgentgateway +} diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index 331cfdd61a..f9031e534c 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -56,6 +56,9 @@ var grpcEchoFixtureManifests = e2e.SubstrateFixtureManifests{ // actor that really does speak gRPC. func TestIngressProtocolDowngrade(t *testing.T) { ctx := context.Background() + if e2e.RouterIsAgentgateway(ctx, t) { + t.Skip("the downgrade contract is Envoy's protocol mirroring to atunnel (xds.go); agentgateway does not implement it") + } actorName, _ := createAndResumeSubstrateActor(t, ctx, "protodowngrade", e2e.SubstrateCounterFixture()) actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} From a7074e7277cce8dd3e16206f30fb47aafc28d815 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Tue, 1 Sep 2026 18:02:42 -0500 Subject: [PATCH 2/4] agentgateway: authorize egress at CONNECT-accept via frontend policy Bump agentgateway to a nightly that authorizes the actor identity before terminating any CONNECT tunnel and retries stale worker assignments on the CONNECT leg by evicting and re-resolving through ResumeActor. substrateEgress accordingly moves from a policy on the internal HTTP route to frontendPolicies in the egress ConfigMaps (manifests component, MITM overlay, and the Helm chart): the frontend check covers HTTP, TLS, and opaque TCP tunnels alike, where the old route policy silently exempted the TLS and TCP passthrough routes. This closes the hole where a certificate for an unknown actor could open a tunnel, and makes suspended actors resumable through the router again. --- charts/substrate/templates/atenet-egress.yaml | 20 ++++++----- charts/substrate/values.yaml | 2 +- .../kustomization.yaml | 34 +++++++++---------- .../components/agentgateway/configmap.yaml | 20 ++++++----- .../agentgateway/kustomization.yaml | 4 +-- 5 files changed, 41 insertions(+), 39 deletions(-) diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 86cc2271ca..7c2751ac82 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -32,6 +32,16 @@ data: accessLog: add: substrate.connect.authority: source.connectHeaders["host"] + # Authorize the actor identity at CONNECT-accept, before any tunnel + # (HTTP, TLS, or opaque TCP) is terminated. Fails closed when the + # control plane is unreachable. + substrateEgress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem binds: - port: 8443 @@ -54,15 +64,7 @@ data: target: source.connectHeaders["host"] - protocol: HTTP routes: - - policies: - substrateEgress: - host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 - policies: - backendTLS: - cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem - key: /run/podidentity.podcert.ate.dev/credential-bundle.pem - root: /run/servicedns.podcert.ate.dev/trust-bundle.pem - backends: + - backends: - dynamic: target: source.connectHeaders["host"] - protocol: TCP diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 6bb83019ed..58fa14e021 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -70,6 +70,6 @@ images: postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 - agentgateway: ghcr.io/kagent-dev/substrate/agentgateway:c0f5597c7cb8 + agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8cbb254d coredns: coredns/coredns:1.11.1 busybox: busybox:1.36 diff --git a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml index 6512099dc5..073252aeb7 100644 --- a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml @@ -33,6 +33,15 @@ patches: accessLog: add: substrate.connect.authority: source.connectHeaders["host"] + # Authorize the actor identity at CONNECT-accept, before any tunnel + # is terminated. Fails closed when the control plane is unreachable. + substrateEgress: + host: api.ate-system.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem binds: - port: 8443 @@ -53,29 +62,18 @@ patches: cert: /run/egress-mitm/tls.crt key: /run/egress-mitm/tls.key routes: - - policies: - substrateEgress: - host: api.ate-system.svc:443 - policies: - backendTLS: - cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem - key: /run/podidentity.podcert.ate.dev/credential-bundle.pem - root: /run/servicedns-ca/trust-bundle.pem - backends: + - backends: - dynamic: {} policies: backendTLS: {} - protocol: HTTP routes: - - policies: - substrateEgress: - host: api.ate-system.svc:443 - policies: - backendTLS: - cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem - key: /run/podidentity.podcert.ate.dev/credential-bundle.pem - root: /run/servicedns-ca/trust-bundle.pem - backends: + - backends: + - dynamic: + target: source.connectHeaders["host"] + - protocol: TCP + tcpRoutes: + - backends: - dynamic: target: source.connectHeaders["host"] - protocol: TCP diff --git a/manifests/ate-install/components/agentgateway/configmap.yaml b/manifests/ate-install/components/agentgateway/configmap.yaml index 4bfd3089b0..78caf4a5dc 100644 --- a/manifests/ate-install/components/agentgateway/configmap.yaml +++ b/manifests/ate-install/components/agentgateway/configmap.yaml @@ -181,6 +181,16 @@ data: accessLog: add: substrate.connect.authority: source.connectHeaders["host"] + # Authorize the actor identity at CONNECT-accept, before any tunnel + # (HTTP, TLS, or opaque TCP) is terminated. Fails closed when the + # control plane is unreachable. + substrateEgress: + host: api.ate-system.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem binds: # Authenticate the actor before accepting CONNECT. @@ -207,15 +217,7 @@ data: target: source.connectHeaders["host"] - protocol: HTTP routes: - - policies: - substrateEgress: - host: api.ate-system.svc:443 - policies: - backendTLS: - cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem - key: /run/podidentity.podcert.ate.dev/credential-bundle.pem - root: /run/servicedns-ca/trust-bundle.pem - backends: + - backends: - dynamic: target: source.connectHeaders["host"] - protocol: TCP diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index af8c4496be..775148822e 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -42,7 +42,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: ghcr.io/kagent-dev/substrate/agentgateway:c0f5597c7cb8 + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8cbb254d args: - -f - /etc/agentgateway/config.yaml @@ -115,7 +115,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: ghcr.io/kagent-dev/substrate/agentgateway:c0f5597c7cb8 + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8cbb254d args: - -f - /etc/agentgateway/config.yaml From bd8a4a6ceb3ed8b54ccc4a5a4f527b6bd8039073 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Tue, 1 Sep 2026 18:21:45 -0500 Subject: [PATCH 3/4] ci: extend install rollout timeout for cold image pulls The e2e-test job's install step waits for each ate-system workload with the script's default 60s rollout timeout. That default assumes warm image caches; on a fresh CI runner every image (postgres, the agentgateway proxy for both router and egress, ...) is cold-pulled from public registries concurrently, and the first wait in line regularly exceeds 60s on pull latency alone. Give the install a 300s per-workload budget, matching the tolerance the helm-e2e workflow already has. --- .github/workflows/pr-workflow.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index b0a1e30690..2592ebc6ec 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -87,7 +87,11 @@ jobs: - name: Create cluster run: hack/create-kind-cluster.sh - name: Install Agent Substrate - run: hack/install-ate-kind.sh --deploy-ate-system --atenet-router=agentgateway + # The default 60s per-workload rollout wait assumes warm image caches; on a + # fresh runner every image (postgres, the agentgateway proxy for router and + # egress, ...) is cold-pulled from public registries concurrently, and the + # first wait in line regularly exceeds 60s on pull latency alone. + run: hack/install-ate-kind.sh --deploy-ate-system --atenet-router=agentgateway --rollout-timeout=300s - name: Enable NFS # Load NFS kernel modules so in-cluster NFS server and CSI driver can run. run: | From b06dc432b9cbb7bde312f77b083bea8f42e9f214 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Tue, 1 Sep 2026 18:54:45 -0500 Subject: [PATCH 4/4] ci: drop helm-e2e's wait on the deleted ActorTemplate CRD helm-e2e waited for condition=Ready on the counter-microvm ActorTemplate CRD after deploying the demo. The cutover to the substrate ActorTemplate proto removed the CRD golden flow and then the CRD itself, so the wait can never succeed. The demo deploy now creates the substrate template and waits for its golden snapshot internally, so drop the separate wait step and point the micro-VM test lane at the substrate fixtures with the E2E_SANDBOX_CLASS knob pr-workflow already uses. --- .github/workflows/helm-e2e.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index dfc5df2757..81b3a46103 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -92,6 +92,9 @@ jobs: - name: Install CSI NFS driver run: hack/install-ate-kind.sh --setup-csi=nfs - name: Deploy micro-VM counter demo + # The deploy creates the substrate ActorTemplate and waits for its golden + # snapshot internally; the ActorTemplate CRD (and its Ready condition) + # no longer exists to wait on. run: hack/run-microvm-demo-kind.sh --skip-control-plane - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter @@ -101,9 +104,7 @@ jobs: run: hack/run-e2e-kind.sh -v -args --no-color - name: Run E2E tests (micro-VM) env: - E2E_TEMPLATE_NAMESPACE: ate-demo-counter-microvm - E2E_TEMPLATE_NAME: counter-microvm - E2E_TEMPLATE_READY_TIMEOUT: 600s + E2E_SANDBOX_CLASS: microvm run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color - name: Dump diagnostics on failure if: failure()