trident-acl-agent: add storm trident-acl-agent E2E test harness, test images, and pipeline

.pipelines/templates/e2e-template.yml

@@ -270,6 +270,11 @@ stages:
       dependsOnStage: ${{ parameters.baseImageArtifactStage }}
       testSecureBoot: ${{ parameters.testSecureBoot }}
 
+  # Validate trident-acl-agent (storm A/B update scenario against a real tridentd)
+  - template: stages/testing_acl_agent/trident-acl-agent-test.yml
+    parameters:
+      dependsOnStage: ${{ parameters.baseImageArtifactStage }}
+
   # TESTING stages for PRERELEASE
   - ${{ if eq(parameters.stageType, 'pre') }}:
       # Functional Testing

.pipelines/templates/stages/testing_acl_agent/trident-acl-agent-test.yml

@@ -0,0 +1,140 @@
+parameters:
+  - name: dependsOnStage
+    type: string
+    default: ""
+
+  - name: micBuildType
+    displayName: MIC Build Type
+    type: string
+    values:
+      - dev
+      - preview
+      - release
+    default: release
+
+  - name: micVersion
+    displayName: MIC Version
+    type: string
+    default: "*.*.*"
+
+  - name: baseimgAzlVersion
+    displayName: Base Image AZL version
+    type: string
+    default: "3.0"
+
+  - name: verboseLogging
+    displayName: "Enable verbose logging"
+    type: boolean
+    default: false
+
+stages:
+  - stage: BuildImagesAclAgent
+    displayName: Build Base and Update Images for trident-acl-agent
+    dependsOn:
+      - PrepareSSHKeys
+      - GetTridentBinaries_rpms_amd64
+      - ${{ if ne(parameters.dependsOnStage, '') }}:
+          - ${{ parameters.dependsOnStage }}
+
+    jobs:
+      - template: ../trident_images/build-image.yml
+        parameters:
+          label: "acl-agent-base"
+          makeTarget: "artifacts/trident-vm-acl-agent-testimage.qcow2"
+          baseimgType: qemu_guest
+          baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }}
+          micBuildType: ${{ parameters.micBuildType }}
+          micVersion: ${{ parameters.micVersion }}
+          useStagedSshKeys: true
+
+      - template: ../trident_images/build-image.yml
+        parameters:
+          label: "acl-agent-update"
+          makeTarget: "artifacts/trident-vm-acl-agent-update-testimage.cosi"
+          baseimgType: qemu_guest
+          baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }}
+          micBuildType: ${{ parameters.micBuildType }}
+          micVersion: ${{ parameters.micVersion }}
+          useStagedSshKeys: true
+
+  - stage: TridentAclAgentTest
+    displayName: Validate trident-acl-agent
+    dependsOn:
+      - BuildingTools
+      - BuildImagesAclAgent
+
+    jobs:
+      - job: AclAgentStormTest
+        displayName: Run storm aclagent scenario
+        timeoutInMinutes: 30
+        pool:
+          type: linux
+          name: trident-ubuntu-1es-pool-eastus2
+          hostArchitecture: amd64
+
+        variables:
+          ob_outputDirectory: /tmp/output
+          ob_artifactBaseName: "aclagent-storm-test"
+
+        steps:
+          - template: ../common_tasks/checkout_trident.yml
+          - template: ../common_tasks/avoid-pypi-usage.yml
+
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              buildType: current
+              artifactName: image-acl-agent-base
+              targetPath: "$(Build.ArtifactStagingDirectory)"
+            displayName: Download Base Image (qcow2)
+
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              buildType: current
+              artifactName: image-acl-agent-update
+              targetPath: "$(Build.ArtifactStagingDirectory)"
+            displayName: Download Update Image (cosi)
+
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              buildType: current
+              artifactName: ssh-keys
+              targetPath: "$(Build.ArtifactStagingDirectory)/ssh"
+            displayName: Download SSH Keys
+
+          - task: DownloadPipelineArtifact@2
+            displayName: "Download go-tools"
+            inputs:
+              buildType: current
+              artifactName: "go-tools"
+              patterns: |
+                storm-trident
+              targetPath: "$(TRIDENT_SOURCE_DIR)/bin"
+
+          - bash: |
+              set -eux
+              chmod +x $(TRIDENT_SOURCE_DIR)/bin/storm-trident
+              cp $(Build.ArtifactStagingDirectory)/ssh/id_rsa* ~/.ssh/
+              chmod -R 700 ~/.ssh/
+              mkdir -p $(ob_outputDirectory)
+            displayName: Set up SSH keys and output directory
+            workingDirectory: $(TRIDENT_SOURCE_DIR)
+
+          - bash: |
+              set -eux
+              ls -la $(Build.ArtifactStagingDirectory)/
+            displayName: List downloaded image artifacts
+
+          - bash: |
+              set -eux
+
+              FLAGS=""
+              if [ "${{ parameters.verboseLogging }}" == "True" ]; then
+                FLAGS="$FLAGS --verbose"
+              fi
+
+              sudo ./bin/storm-trident run aclagent $FLAGS \
+                --output-path $(ob_outputDirectory) \
+                --artifacts-dir $(Build.ArtifactStagingDirectory) \
+                --ssh-private-key-path ~/.ssh/id_rsa
+            displayName: "🧪 Run trident-acl-agent A/B update + rollback scenario"
+            workingDirectory: $(TRIDENT_SOURCE_DIR)

Makefile

@@ -1172,6 +1172,27 @@ artifacts/trident-vm-usr-verity-testimage.qcow2: \
 			--output-image-format qcow2 \
 			--config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-usr-verity.yaml
 
+artifacts/trident-vm-acl-agent-testimage.qcow2: \
+	$(QEMU_GUEST_IMAGE) \
+	$(TRIDENT_VM_DEPENDENCIES) \
+	$(VM_IMAGE_PATH_PREFIX)/baseimg-acl-agent.yaml \
+	$(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub \
+	artifacts/rpm-overrides
+	@echo "Building $@ from $<"
+	docker run --rm \
+		--privileged \
+		-v ".:/repo:z" \
+		-v "/dev:/dev" \
+		${MIC_CONTAINER_IMAGE} \
+			--log-level debug \
+			--rpm-source /repo/bin/RPMS \
+			--rpm-source /repo/artifacts/rpm-overrides \
+			--build-dir /build \
+			--image-file /repo/$< \
+			--output-image-file /repo/$@ \
+			--output-image-format qcow2 \
+			--config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-acl-agent.yaml
+
 artifacts/trident-vm-grub-verity-azure-testimage.vhd: \
 	$(CORE_SELINUX_IMAGE) \
 	$(TRIDENT_VM_DEPENDENCIES) \

docs/Development/Testing/Testing.md

@@ -54,6 +54,10 @@ manual rollback chains without using `netlaunch` or an installer ISO.
   rollback via `storm-trident run servicing`
 - [Rollback Tests](Rollback-Tests.md) — full rollback chain (A/B + runtime
   updates) via `storm-trident run rollback`
+- [Trident ACL Agent Tests](TridentAclAgent-Tests.md) — validates
+  `trident-acl-agent`'s label-driven update protocol against fake
+  Kubernetes API server and Nebraska/Omaha endpoints via
+  `storm-trident run aclagent`
 
 ## Code Coverage
 

docs/Development/Testing/TridentAclAgent-Tests.md

@@ -0,0 +1,203 @@
+---
+sidebar_position: 9
+---
+
+# Trident ACL Agent Tests
+
+`storm-trident run aclagent` is the single supported validation entrypoint for
+the label-driven `trident-acl-agent` protocol described in the ACL AKS
+node-label design. Unlike [Servicing Tests](Servicing-Tests.md), which drive
+Trident's own `stage`/`finalize` gRPC calls directly, this scenario validates
+`trident-acl-agent` itself: it deploys a VM, starts fake in-process test
+doubles for the Kubernetes API server and the Nebraska/Omaha update server,
+seeds bootstrap node labels, and lets the real `trident-acl-agent` binary
+running inside the VM drive a full A/B update against those fakes.
+
+There is intentionally no fake `tridentd` — the scenario talks to the real
+`tridentd` and real `trident-acl-agent` running inside the VM.
+
+## What It Validates
+
+- `trident-acl-agent` watching its own Kubernetes Node object for
+  RP-authored label changes (via `kube::runtime::watcher()`)
+- Reading the update image URL/hash from labels and triggering a real
+  Trident `stage` + `finalize` A/B update through the normal gRPC path
+- Patching back observed-state labels/annotations as the update progresses
+- Resuming correctly after a real reboot (see [Reboot Choice](#reboot-choice))
+
+## VM Image Contents
+
+The VM image used by this scenario must already contain:
+
+- `tridentd.socket` installed and enabled (starts `tridentd.service` on
+  demand)
+- `trident-acl-agent` package installed, but **`trident-acl-agent.service`
+  left disabled** — it must not start before a config file exists
+- the same SSH user/key setup expected by the [servicing](Servicing-Tests.md)
+  scenario
+
+Both the enabled/disabled state of `trident-acl-agent.service` and
+`/etc/trident/trident-acl-agent.conf` live under `/etc`, which is not part of
+the A/B-swapped `/usr`/root volume pair in this usr-verity image layout. That
+makes it safe for the scenario to write the config and enable the service
+once, after `deploy-vm`, rather than baking enablement into the image — the
+state persists across `run-ab-update`'s finalize the same way the config file
+does.
+
+## Prerequisites
+
+- **Linux host** with root access
+- **libvirt and QEMU** installed and configured
+- **Docker** (for building images with Image Customizer)
+- **Go 1.24+** (for building Go tools)
+- **Rust** (latest stable, for building Trident and `trident-acl-agent`)
+
+See [Dependencies](../Building/Dependencies.md) for full build dependency
+details.
+
+## Building Dependencies
+
+### 1. Build Trident, `trident-acl-agent`, and RPMs
+
+Always build through `make`, not a raw `cargo build`, when the RPM tarball
+needs to reflect a source change — `make` injects the dev version string
+(`TRIDENT_VERSION`) that the RPM spec's `%check` step verifies against. A
+plain `cargo build` skips that and produces an RPM build failure.
+
+```bash
+make target/release/trident target/release/trident-acl-agent
+make bin/trident-rpms.tar.gz
+```
+
+### 2. Build Go Tools
+
+```bash
+make bin/storm-trident
+```
+
+### 3. Generate SSH Keys
+
+```bash
+make artifacts/id_rsa
+```
+
+:::note
+The VM images below bake in the public key from `artifacts/id_rsa.pub` (via
+the `files/id_rsa.pub` Makefile rule), **not** `~/.ssh/id_rsa.pub`. Always
+pass `--ssh-private-key-path artifacts/id_rsa` when running the scenario
+locally — using your personal `~/.ssh/id_rsa` doesn't fail fast, it just
+hangs/retries during `check-deployment`'s SSH auth.
+:::
+
+### 4. Download the qemu_guest Base Image
+
+Same base image as the servicing tests — see [Servicing Tests, step
+4](Servicing-Tests.md#4-download-the-qemu_guest-base-image) for details.
+
+### 5. Build the Base and Update VM Images
+
+The scenario needs two images built from the current source:
+
+```bash
+# Base image: trident-acl-agent installed but disabled
+make artifacts/trident-vm-acl-agent-testimage.qcow2
+
+# Update image: what the agent updates the VM to
+make artifacts/trident-vm-acl-agent-update-testimage.cosi
+```
+
+:::caution Rebuild after any `trident-acl-agent` change
+Both image targets embed the RPM built in step 1. If you only rebuild the
+Rust binary and re-run the scenario without rebuilding these images, you are
+still testing the **old** binary baked into the existing qcow2/cosi files —
+the failure (or fix) you're trying to observe silently won't reproduce. Clear
+stale artifacts first if you're not sure they're current:
+
+```bash
+rm -f artifacts/trident-vm-acl-agent-testimage.qcow2 \
+      artifacts/trident-vm-acl-agent-update-testimage.cosi
+```
+:::
+
+## Running the ACL Agent Scenario
+
+The scenario requires root access for VM creation via `virt-install`:
+
+```bash
+sudo bin/storm-trident run aclagent \
+    --artifacts-dir ./artifacts \
+    --output-path /tmp/aclagent-output \
+    --ssh-private-key-path ./artifacts/id_rsa \
+    --verbose
+```
+
+### Test Cases
+
+The scenario runs these test cases in order:
+
+1. **deploy-vm** — Copies the base qcow2 image and creates a QEMU VM
+2. **check-deployment** — Verifies the VM booted and is accessible via SSH;
+   writes `/etc/trident/trident-acl-agent.conf` pointing at the
+   `localhost:<port>` endpoints storm reverse-SSH-forwards into the VM, then
+   runs `systemctl enable --now trident-acl-agent.service`
+3. **run-ab-update** — Starts the fake apiserver and fake Nebraska/Omaha
+   endpoints in-process, seeds bootstrap node labels, patches the desired
+   update-image label, and waits for `trident-acl-agent` to drive a real
+   Trident A/B update to completion (including a real reboot)
+4. **collect-logs** — Fetches `trident-acl-agent` and Trident logs from the
+   VM via SSH; also runs automatically (with a `journalctl` dump for
+   `trident-acl-agent.service`) if `run-ab-update` times out waiting for the
+   service to become active, to make crash-loops self-diagnosing
+5. **cleanup-vm** — Destroys the QEMU VM
+
+### Flags
+
+| Flag | Description | Default |
+|------|-------------|---------|
+| `--artifacts-dir` | Directory containing VM images | `/tmp` |
+| `--output-path` | Output directory for logs | `./output` |
+| `--platform` | `qemu` or `azure` | `qemu` |
+| `--ssh-private-key-path` | Path to SSH private key | `~/.ssh/id_rsa` |
+| `--api-server-port` | Port for the fake Kubernetes API server | `18080` |
+| `--nebraska-port` | Port for the fake Nebraska/Omaha server | `18081` |
+| `--verbose` | Enable verbose logging | `false` |
+| `--test-case-to-run` | Run a specific test case only | `all` |
+
+## Reboot Choice
+
+This scenario uses a real VM reboot rather than a shim: `trident-acl-agent`
+issues a genuine `systemctl reboot` on finalize, and the scenario polls SSH
+until it goes unreachable (confirming the reboot actually happened) and then
+reachable again (confirming the VM came back up), exercising the agent's
+real post-reboot resume logic end to end. This is slower than a shim-based
+approach, but it validates the real reboot path instead of a simulation of
+it.
+
+## Debugging Failures
+
+If `trident-acl-agent.service` gets stuck reporting `activating` and the test
+times out, that almost always means a **crash-restart loop**, not a slow
+start — the unit has no explicit `Type=`, so `Type=simple` (the implicit
+default) is used, and systemd marks such units active immediately on
+`fork`/`exec` with no readiness signal. A persistent `activating` state for
+the full wait window can only mean `Restart=on-failure` (`RestartSec=5`) is
+cycling the service.
+
+`run-ab-update`'s wait-for-active check captures `journalctl -u
+trident-acl-agent.service --no-pager -n 200` on timeout and includes it in
+the test failure, so the actual crash reason (e.g. a panic, a fatal error
+from the Kubernetes client, or a config problem) should be visible directly
+in the CI log or local output without a separate log-collection step.
+
+The fake Kubernetes API server (`tools/storm/aclagent/proxies/apiserver.go`)
+is a minimal, hand-rolled HTTP handler — it does not implement the full
+Kubernetes API surface. If `trident-acl-agent` is changed to make a new kind
+of API call (a different verb, a new field selector, list pagination,
+etc.), the fake apiserver's routing may need a corresponding update or the
+call will simply 404 and (since `trident-acl-agent` treats such client
+errors as fatal) crash-loop the service. For example, migrating node-watching
+from polling to `kube::runtime::watcher()` introduced an initial **LIST**
+call to the collection endpoint (`GET /api/v1/nodes?fieldSelector=...`) that
+the fake server didn't originally route, only the singular
+`/api/v1/nodes/<name>` path — surfacing as exactly this crash-loop symptom
+until the collection route was added.