From a71c287a33a4147c6abfb5fa087f7733abb5e558 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 5 Aug 2026 19:38:51 +0000 Subject: [PATCH 1/4] storm/aclagent: add trident-acl-agent E2E test harness, test images, and pipeline Adds the storm-trident E2E scenario that validates trident-acl-agent end-to-end against real tridentd/kubelet/Nebraska on a VM, exercising stage, finalize, rollback, and the post-reboot commit path. Go test harness (tools/storm/aclagent): - proxies/: fake apiserver (serves the Node annotation protocol the agent watches/patches, including a real K8s-compatible watch stream), fake Nebraska/image servers, a minimal kubelet API shim, and an RP client used to drive scenarios and poll status annotations. - tests/: update.go (stage/finalize/commit against a real reboot), rollback.go (rollback stage/finalize/commit against a real reboot, plus a regression test that a second rollback against an empty rollback chain is detected as a no-op via servicing_kind rather than reporting a false Success and rebooting again), vm.go/logs.go (VM lifecycle and log collection helpers). - trident.go, utils/config: scenario wiring and config plumbing specific to the aclagent suite. - README.md: usage instructions for running the suite locally. Registered in tools/cmd/storm-trident/main.go alongside storm-trident's other scenarios. Test images (tests/images): - baseimg-acl-agent.yaml / updateimg-acl-agent.yaml: VM image configurations for the base and post-update ACL test images. - testimages.py: wires the update image into the existing COSI-based image build path (the base qcow2 is built via the Makefile target below, not testimages.py, since the qcow2 tooling differs from COSI). Build tooling (Makefile): new artifacts/trident-vm-acl-agent-testimage.qcow2 target for the base image. Pipeline (.pipelines): new trident-acl-agent-test.yml stage wired into e2e-template.yml, running the aclagent storm scenario in CI. Docs: docs/Development/Testing/TridentAclAgent-Tests.md documents the suite; Testing.md links to it. Depends on the trident-acl-agent Rust implementation in the parent branch (user/bfjelds/acl-agent-rollback-grpc-rust). This is the final branch in the stack and, combined with its two parent branches, contains the full set of changes from user/bfjelds/acl-agent-rollback-grpc. Verified: go build ./... and gofmt clean under tools/; go vet ./... clean except one pre-existing, unrelated warning in storm/servicing/tests/update.go (confirmed present on main, untouched by this change); storm-trident binary builds; full local `storm-trident run aclagent` suite passes (6/6: deploy-vm, check-deployment, run-ab-update, run-rollback, collect-logs, cleanup-vm), rebuilt end-to-end from this branch's HEAD. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8c06585d-a82d-475f-a802-83fdfa012d86 --- .pipelines/templates/e2e-template.yml | 5 + .../trident-acl-agent-test.yml | 140 ++++++ Makefile | 24 +- docs/Development/Testing/Testing.md | 4 + .../Testing/TridentAclAgent-Tests.md | 205 ++++++++ tests/images/testimages.py | 8 + tests/images/trident-vm-testimage/README.md | 27 +- .../base/baseimg-acl-agent.yaml | 182 +++++++ .../base/updateimg-acl-agent.yaml | 191 ++++++++ tools/cmd/storm-trident/main.go | 4 + tools/go.mod | 19 +- tools/go.sum | 66 ++- tools/storm/aclagent/README.md | 99 ++++ tools/storm/aclagent/proxies/apiserver.go | 357 ++++++++++++++ tools/storm/aclagent/proxies/constants.go | 10 + tools/storm/aclagent/proxies/imageserver.go | 57 +++ tools/storm/aclagent/proxies/kubelet.go | 183 ++++++++ tools/storm/aclagent/proxies/nebraska.go | 155 ++++++ tools/storm/aclagent/proxies/rp.go | 169 +++++++ tools/storm/aclagent/proxies/scenario.go | 96 ++++ tools/storm/aclagent/tests/logs.go | 11 + tools/storm/aclagent/tests/rollback.go | 134 ++++++ tools/storm/aclagent/tests/update.go | 444 ++++++++++++++++++ tools/storm/aclagent/tests/vm.go | 43 ++ tools/storm/aclagent/trident.go | 92 ++++ tools/storm/aclagent/utils/config/config.go | 19 + tools/storm/utils/vm/qemu/qemu.go | 7 +- 27 files changed, 2729 insertions(+), 22 deletions(-) create mode 100644 .pipelines/templates/stages/testing_acl_agent/trident-acl-agent-test.yml create mode 100644 docs/Development/Testing/TridentAclAgent-Tests.md create mode 100644 tests/images/trident-vm-testimage/base/baseimg-acl-agent.yaml create mode 100644 tests/images/trident-vm-testimage/base/updateimg-acl-agent.yaml create mode 100644 tools/storm/aclagent/README.md create mode 100644 tools/storm/aclagent/proxies/apiserver.go create mode 100644 tools/storm/aclagent/proxies/constants.go create mode 100644 tools/storm/aclagent/proxies/imageserver.go create mode 100644 tools/storm/aclagent/proxies/kubelet.go create mode 100644 tools/storm/aclagent/proxies/nebraska.go create mode 100644 tools/storm/aclagent/proxies/rp.go create mode 100644 tools/storm/aclagent/proxies/scenario.go create mode 100644 tools/storm/aclagent/tests/logs.go create mode 100644 tools/storm/aclagent/tests/rollback.go create mode 100644 tools/storm/aclagent/tests/update.go create mode 100644 tools/storm/aclagent/tests/vm.go create mode 100644 tools/storm/aclagent/trident.go create mode 100644 tools/storm/aclagent/utils/config/config.go diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index a0654303a4..f06dc752c7 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -270,6 +270,11 @@ stages: dependsOnStage: ${{ parameters.baseImageArtifactStage }} testSecureBoot: ${{ parameters.testSecureBoot }} + # Validate trident-acl-agent (storm A/B update scenario against a real tridentd) + - template: stages/testing_acl_agent/trident-acl-agent-test.yml + parameters: + dependsOnStage: ${{ parameters.baseImageArtifactStage }} + # TESTING stages for PRERELEASE - ${{ if eq(parameters.stageType, 'pre') }}: # Functional Testing diff --git a/.pipelines/templates/stages/testing_acl_agent/trident-acl-agent-test.yml b/.pipelines/templates/stages/testing_acl_agent/trident-acl-agent-test.yml new file mode 100644 index 0000000000..72bf0689b4 --- /dev/null +++ b/.pipelines/templates/stages/testing_acl_agent/trident-acl-agent-test.yml @@ -0,0 +1,140 @@ +parameters: + - name: dependsOnStage + type: string + default: "" + + - name: micBuildType + displayName: MIC Build Type + type: string + values: + - dev + - preview + - release + default: release + + - name: micVersion + displayName: MIC Version + type: string + default: "*.*.*" + + - name: baseimgAzlVersion + displayName: Base Image AZL version + type: string + default: "3.0" + + - name: verboseLogging + displayName: "Enable verbose logging" + type: boolean + default: false + +stages: + - stage: BuildImagesAclAgent + displayName: Build Base and Update Images for trident-acl-agent + dependsOn: + - PrepareSSHKeys + - GetTridentBinaries_rpms_amd64 + - ${{ if ne(parameters.dependsOnStage, '') }}: + - ${{ parameters.dependsOnStage }} + + jobs: + - template: ../trident_images/build-image.yml + parameters: + label: "acl-agent-base" + makeTarget: "artifacts/trident-vm-acl-agent-testimage.qcow2" + baseimgType: qemu_guest + baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} + micBuildType: ${{ parameters.micBuildType }} + micVersion: ${{ parameters.micVersion }} + useStagedSshKeys: true + + - template: ../trident_images/build-image.yml + parameters: + label: "acl-agent-update" + makeTarget: "artifacts/trident-vm-acl-agent-update-testimage.cosi" + baseimgType: qemu_guest + baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} + micBuildType: ${{ parameters.micBuildType }} + micVersion: ${{ parameters.micVersion }} + useStagedSshKeys: true + + - stage: TridentAclAgentTest + displayName: Validate trident-acl-agent + dependsOn: + - BuildingTools + - BuildImagesAclAgent + + jobs: + - job: AclAgentStormTest + displayName: Run storm aclagent scenario + timeoutInMinutes: 30 + pool: + type: linux + name: trident-ubuntu-1es-pool-eastus2 + hostArchitecture: amd64 + + variables: + ob_outputDirectory: /tmp/output + ob_artifactBaseName: "aclagent-storm-test" + + steps: + - template: ../common_tasks/checkout_trident.yml + - template: ../common_tasks/avoid-pypi-usage.yml + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: image-acl-agent-base + targetPath: "$(Build.ArtifactStagingDirectory)" + displayName: Download Base Image (qcow2) + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: image-acl-agent-update + targetPath: "$(Build.ArtifactStagingDirectory)" + displayName: Download Update Image (cosi) + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ssh-keys + targetPath: "$(Build.ArtifactStagingDirectory)/ssh" + displayName: Download SSH Keys + + - task: DownloadPipelineArtifact@2 + displayName: "Download go-tools" + inputs: + buildType: current + artifactName: "go-tools" + patterns: | + storm-trident + targetPath: "$(TRIDENT_SOURCE_DIR)/bin" + + - bash: | + set -eux + chmod +x $(TRIDENT_SOURCE_DIR)/bin/storm-trident + cp $(Build.ArtifactStagingDirectory)/ssh/id_rsa* ~/.ssh/ + chmod -R 700 ~/.ssh/ + mkdir -p $(ob_outputDirectory) + displayName: Set up SSH keys and output directory + workingDirectory: $(TRIDENT_SOURCE_DIR) + + - bash: | + set -eux + ls -la $(Build.ArtifactStagingDirectory)/ + displayName: List downloaded image artifacts + + - bash: | + set -eux + + FLAGS="" + if [ "${{ parameters.verboseLogging }}" == "True" ]; then + FLAGS="$FLAGS --verbose" + fi + + sudo ./bin/storm-trident run aclagent $FLAGS \ + --output-path $(ob_outputDirectory) \ + --artifacts-dir $(Build.ArtifactStagingDirectory) \ + --ssh-private-key-path ~/.ssh/id_rsa + displayName: "๐Ÿงช Run trident-acl-agent A/B update + rollback scenario" + workingDirectory: $(TRIDENT_SOURCE_DIR) diff --git a/Makefile b/Makefile index bba5720ba1..0ba4ddf93a 100644 --- a/Makefile +++ b/Makefile @@ -479,7 +479,6 @@ go.sum: go.mod go mod tidy .PHONY: go-tools -go-tools: bin/netlaunch bin/netlisten bin/miniproxy bin/virtdeploy bin/isopatch bin/mkcosi bin/storm-trident bin/rcp-agent bin/netlaunch: tools/cmd/netlaunch/* tools/go.sum tools/pkg/* tools/pkg/netlaunch/* @mkdir -p bin @@ -520,6 +519,8 @@ bin/rcp-agent: tools/cmd/rcp-agent/* tools/go.sum tools/pkg/rcp/* tools/pkg/rcp/ cd tools && go generate pkg/rcp/tlscerts/certs.go cd tools && go build -o ../bin/rcp-agent ./cmd/rcp-agent/main.go + @mkdir -p bin + # Clean generated RCP TLS certificates .PHONY: clean-rcp-certs clean-rcp-certs: @@ -1172,6 +1173,27 @@ artifacts/trident-vm-usr-verity-testimage.qcow2: \ --output-image-format qcow2 \ --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-usr-verity.yaml +artifacts/trident-vm-acl-agent-testimage.qcow2: \ + $(QEMU_GUEST_IMAGE) \ + $(TRIDENT_VM_DEPENDENCIES) \ + $(VM_IMAGE_PATH_PREFIX)/baseimg-acl-agent.yaml \ + $(VM_IMAGE_PATH_PREFIX)/files/id_rsa.pub \ + artifacts/rpm-overrides + @echo "Building $@ from $<" + docker run --rm \ + --privileged \ + -v ".:/repo:z" \ + -v "/dev:/dev" \ + ${MIC_CONTAINER_IMAGE} \ + --log-level debug \ + --rpm-source /repo/bin/RPMS \ + --rpm-source /repo/artifacts/rpm-overrides \ + --build-dir /build \ + --image-file /repo/$< \ + --output-image-file /repo/$@ \ + --output-image-format qcow2 \ + --config-file /repo/$(VM_IMAGE_PATH_PREFIX)/baseimg-acl-agent.yaml + artifacts/trident-vm-grub-verity-azure-testimage.vhd: \ $(CORE_SELINUX_IMAGE) \ $(TRIDENT_VM_DEPENDENCIES) \ diff --git a/docs/Development/Testing/Testing.md b/docs/Development/Testing/Testing.md index 9812dd4b70..22d318d9d3 100644 --- a/docs/Development/Testing/Testing.md +++ b/docs/Development/Testing/Testing.md @@ -54,6 +54,10 @@ manual rollback chains without using `netlaunch` or an installer ISO. rollback via `storm-trident run servicing` - [Rollback Tests](Rollback-Tests.md) โ€” full rollback chain (A/B + runtime updates) via `storm-trident run rollback` +- [Trident ACL Agent Tests](TridentAclAgent-Tests.md) โ€” validates + `trident-acl-agent`'s label-driven update protocol against fake + Kubernetes API server and Nebraska/Omaha endpoints via + `storm-trident run aclagent` ## Code Coverage diff --git a/docs/Development/Testing/TridentAclAgent-Tests.md b/docs/Development/Testing/TridentAclAgent-Tests.md new file mode 100644 index 0000000000..537ed49abf --- /dev/null +++ b/docs/Development/Testing/TridentAclAgent-Tests.md @@ -0,0 +1,205 @@ +--- +sidebar_position: 9 +--- + +# Trident ACL Agent Tests + +`storm-trident run aclagent` is the single supported validation entrypoint for +the label-driven `trident-acl-agent` protocol described in the ACL AKS +node-label design. Unlike [Servicing Tests](Servicing-Tests.md), which drive +Trident's own `stage`/`finalize` gRPC calls directly, this scenario validates +`trident-acl-agent` itself: it deploys a VM, starts fake in-process test +doubles for the Kubernetes API server and the Nebraska/Omaha update server, +seeds bootstrap node labels, and lets the real `trident-acl-agent` binary +running inside the VM drive a full A/B update against those fakes. + +There is intentionally no fake `tridentd` โ€” the scenario talks to the real +`tridentd` and real `trident-acl-agent` running inside the VM. + +## What It Validates + +- `trident-acl-agent` watching its own Kubernetes Node object for + RP-authored label changes (via `kube::runtime::watcher()`) +- Reading the update image URL/hash from labels and triggering a real + Trident `stage` + `finalize` A/B update through the normal gRPC path +- Patching back observed-state labels/annotations as the update progresses +- Resuming correctly after a simulated reboot (shim-intercepted, not a real + VM reboot โ€” see [Reboot Choice](#reboot-choice)) + +## VM Image Contents + +The VM image used by this scenario must already contain: + +- `tridentd.socket` installed and enabled (starts `tridentd.service` on + demand) +- `trident-acl-agent` package installed, but **`trident-acl-agent.service` + left disabled** โ€” it must not start before a config file exists +- the same SSH user/key setup expected by the [servicing](Servicing-Tests.md) + scenario + +Both the enabled/disabled state of `trident-acl-agent.service` and +`/etc/trident/trident-acl-agent.conf` live under `/etc`, which is not part of +the A/B-swapped `/usr`/root volume pair in this usr-verity image layout. That +makes it safe for the scenario to write the config and enable the service +once, after `deploy-vm`, rather than baking enablement into the image โ€” the +state persists across `run-ab-update`'s finalize the same way the config file +does. + +## Prerequisites + +- **Linux host** with root access +- **libvirt and QEMU** installed and configured +- **Docker** (for building images with Image Customizer) +- **Go 1.24+** (for building Go tools) +- **Rust** (latest stable, for building Trident and `trident-acl-agent`) + +See [Dependencies](../Building/Dependencies.md) for full build dependency +details. + +## Building Dependencies + +### 1. Build Trident, `trident-acl-agent`, and RPMs + +Always build through `make`, not a raw `cargo build`, when the RPM tarball +needs to reflect a source change โ€” `make` injects the dev version string +(`TRIDENT_VERSION`) that the RPM spec's `%check` step verifies against. A +plain `cargo build` skips that and produces an RPM build failure. + +```bash +make target/release/trident target/release/trident-acl-agent +make bin/trident-rpms.tar.gz +``` + +### 2. Build Go Tools + +```bash +make bin/storm-trident +``` + +### 3. Generate SSH Keys + +```bash +make artifacts/id_rsa +``` + +:::note +The VM images below bake in the public key from `artifacts/id_rsa.pub` (via +the `files/id_rsa.pub` Makefile rule), **not** `~/.ssh/id_rsa.pub`. Always +pass `--ssh-private-key-path artifacts/id_rsa` when running the scenario +locally โ€” using your personal `~/.ssh/id_rsa` doesn't fail fast, it just +hangs/retries during `check-deployment`'s SSH auth. +::: + +### 4. Download the qemu_guest Base Image + +Same base image as the servicing tests โ€” see [Servicing Tests, step +4](Servicing-Tests.md#4-download-the-qemu_guest-base-image) for details. + +### 5. Build the Base and Update VM Images + +The scenario needs two images built from the current source: + +```bash +# Base image: trident-acl-agent installed but disabled +make artifacts/trident-vm-acl-agent-testimage.qcow2 + +# Update image: what the agent updates the VM to +make artifacts/trident-vm-acl-agent-update-testimage.cosi +``` + +:::caution Rebuild after any `trident-acl-agent` change +Both image targets embed the RPM built in step 1. If you only rebuild the +Rust binary and re-run the scenario without rebuilding these images, you are +still testing the **old** binary baked into the existing qcow2/cosi files โ€” +the failure (or fix) you're trying to observe silently won't reproduce. Clear +stale artifacts first if you're not sure they're current: + +```bash +rm -f artifacts/trident-vm-acl-agent-testimage.qcow2 \ + artifacts/trident-vm-acl-agent-update-testimage.cosi +``` +::: + +## Running the ACL Agent Scenario + +The scenario requires root access for VM creation via `virt-install`: + +```bash +sudo bin/storm-trident run aclagent \ + --artifacts-dir ./artifacts \ + --output-path /tmp/aclagent-output \ + --ssh-private-key-path ./artifacts/id_rsa \ + --verbose +``` + +### Test Cases + +The scenario runs these test cases in order: + +1. **deploy-vm** โ€” Copies the base qcow2 image and creates a QEMU VM +2. **check-deployment** โ€” Verifies the VM booted and is accessible via SSH; + writes `/etc/trident/trident-acl-agent.conf` pointing at the + `localhost:` endpoints storm reverse-SSH-forwards into the VM, then + runs `systemctl enable --now trident-acl-agent.service` +3. **run-ab-update** โ€” Starts the fake apiserver and fake Nebraska/Omaha + endpoints in-process, seeds bootstrap node labels, patches the desired + update-image label, and waits for `trident-acl-agent` to drive a real + Trident A/B update to completion (including the shim-based simulated + reboot) +4. **collect-logs** โ€” Fetches `trident-acl-agent` and Trident logs from the + VM via SSH; also runs automatically (with a `journalctl` dump for + `trident-acl-agent.service`) if `run-ab-update` times out waiting for the + service to become active, to make crash-loops self-diagnosing +5. **cleanup-vm** โ€” Destroys the QEMU VM + +### Flags + +| Flag | Description | Default | +|------|-------------|---------| +| `--artifacts-dir` | Directory containing VM images | `/tmp` | +| `--output-path` | Output directory for logs | `./output` | +| `--platform` | `qemu` or `azure` | `qemu` | +| `--ssh-private-key-path` | Path to SSH private key | `~/.ssh/id_rsa` | +| `--api-server-port` | Port for the fake Kubernetes API server | `18080` | +| `--nebraska-port` | Port for the fake Nebraska/Omaha server | `18081` | +| `--verbose` | Enable verbose logging | `false` | +| `--test-case-to-run` | Run a specific test case only | `all` | + +## Reboot Choice + +This scenario uses shim-based reboot interception rather than a full VM +reboot: a `reboot`/`systemctl reboot` shim on `PATH` inside the VM signals the +scenario's controller and exits the agent process instead of actually +rebooting. The scenario then restarts `trident-acl-agent` fresh, exercising +its post-reboot resume logic without tearing down the SSH session or the +in-process fake services. This is less realistic than a full reboot, but it +keeps the test deterministic and fast. + +## Debugging Failures + +If `trident-acl-agent.service` gets stuck reporting `activating` and the test +times out, that almost always means a **crash-restart loop**, not a slow +start โ€” the unit has no explicit `Type=`, so `Type=simple` (the implicit +default) is used, and systemd marks such units active immediately on +`fork`/`exec` with no readiness signal. A persistent `activating` state for +the full wait window can only mean `Restart=on-failure` (`RestartSec=5`) is +cycling the service. + +`run-ab-update`'s wait-for-active check captures `journalctl -u +trident-acl-agent.service --no-pager -n 200` on timeout and includes it in +the test failure, so the actual crash reason (e.g. a panic, a fatal error +from the Kubernetes client, or a config problem) should be visible directly +in the CI log or local output without a separate log-collection step. + +The fake Kubernetes API server (`tools/storm/aclagent/proxies/apiserver.go`) +is a minimal, hand-rolled HTTP handler โ€” it does not implement the full +Kubernetes API surface. If `trident-acl-agent` is changed to make a new kind +of API call (a different verb, a new field selector, list pagination, +etc.), the fake apiserver's routing may need a corresponding update or the +call will simply 404 and (since `trident-acl-agent` treats such client +errors as fatal) crash-loop the service. For example, migrating node-watching +from polling to `kube::runtime::watcher()` introduced an initial **LIST** +call to the collection endpoint (`GET /api/v1/nodes?fieldSelector=...`) that +the fake server didn't originally route, only the singular +`/api/v1/nodes/` path โ€” surfacing as exactly this crash-loop symptom +until the collection route was added. diff --git a/tests/images/testimages.py b/tests/images/testimages.py index 9ab341cba9..a5137a2114 100755 --- a/tests/images/testimages.py +++ b/tests/images/testimages.py @@ -155,6 +155,14 @@ requires_ukify=True, ssh_key="files/id_rsa.pub", ), + ImageConfig( + "trident-vm-acl-agent-update-testimage", + base_image=BaseImage.QEMU_GUEST, + config="trident-vm-testimage", + config_file="base/updateimg-acl-agent.yaml", + requires_ukify=True, + ssh_key="files/id_rsa.pub", + ), ImageConfig( "trident-vm-grub-verity-azure-testimage", base_image=BaseImage.CORE_SELINUX, diff --git a/tests/images/trident-vm-testimage/README.md b/tests/images/trident-vm-testimage/README.md index e527ae04ee..4990bcccb2 100644 --- a/tests/images/trident-vm-testimage/README.md +++ b/tests/images/trident-vm-testimage/README.md @@ -10,8 +10,15 @@ Two sets of images are available: - regular - with verity +- with UKI usr-verity for ACL-agent-driven A/B update testing -For both, a set of corresponding update images is available. +For both, a set of corresponding update images is available. The ACL-agent +variant reuses the servicing-style VM image layout but additionally installs +`trident-acl-agent` and enables `trident-acl-agent.service` so storm ACL-agent +scenarios can drive a real in-guest agent talking to `tridentd`. The image does +not preseed /etc/trident/trident-acl-agent.conf with runner-specific tunnel +ports; the test scenario should SSH in after boot and write the real localhost +proxy endpoints for Nebraska and the Kubernetes API server. ## Additional Prerequisites @@ -23,15 +30,17 @@ For both, a set of corresponding update images is available. To build the base image, run: -| Image type | Make command | Output path | -| ---------------------------- | --------------------------------------------------- | ---------------------------------------------- | -| Regular | `make artifacts/trident-vm-grub-testimage.qcow2` | `artifacts/trident-vm-grub-testimage.qcow2` | -| With verity `qcow2` | `make artifacts/trident-vm-grub-verity-testimage.qcow2` | `artifacts/trident-vm-grub-verity-testimage.qcow2` | -| With verity fixed size `vhd` | `make artifacts/trident-vm-grub-verity-testimage.vhd` | `artifacts/trident-vm-grub-verity-testimage.vhd` | +| Image type | Make command | Output path | +| ---------- | ------------ | ----------- | +| Regular | `make artifacts/trident-vm-grub-testimage.qcow2` | `artifacts/trident-vm-grub-testimage.qcow2` | +| With verity `qcow2` | `make artifacts/trident-vm-grub-verity-testimage.qcow2` | `artifacts/trident-vm-grub-verity-testimage.qcow2` | +| With verity fixed size `vhd` | `make artifacts/trident-vm-grub-verity-testimage.vhd` | `artifacts/trident-vm-grub-verity-testimage.vhd` | +| ACL-agent UKI usr-verity `qcow2` | `make artifacts/trident-vm-acl-agent-testimage.qcow2` | `artifacts/trident-vm-acl-agent-testimage.qcow2` | To build the update images, run: -| Image type | Make command | Output path | -| ----------- | --------------------------------------- | ----------------------------------- | -| Regular | `make trident-vm-grub-testimage` | `artifacts/trident-vm-grub-testimage/*` | +| Image type | Make command | Output path | +| ---------- | ------------ | ----------- | +| Regular | `make trident-vm-grub-testimage` | `artifacts/trident-vm-grub-testimage/*` | | With verity | `make trident-vm-grub-verity-testimage` | `artifacts/trident-vm-grub-testimage/*` | +| ACL-agent UKI usr-verity | `make artifacts/trident-vm-acl-agent-update-testimage.cosi` | `artifacts/trident-vm-acl-agent-update-testimage.cosi` | diff --git a/tests/images/trident-vm-testimage/base/baseimg-acl-agent.yaml b/tests/images/trident-vm-testimage/base/baseimg-acl-agent.yaml new file mode 100644 index 0000000000..2bda5d96b0 --- /dev/null +++ b/tests/images/trident-vm-testimage/base/baseimg-acl-agent.yaml @@ -0,0 +1,182 @@ +# UKI usr-verity VM test image for validating trident-acl-agent's +# label-driven A/B update protocol against a real tridentd. +storage: + bootType: efi + + disks: + - partitionTableType: gpt + partitions: + - id: esp + type: esp + label: esp + size: 512M + + - id: boot-a + size: 256M + + - id: boot-b + size: 256M + + - id: root-a + size: 4G + + - id: root-b + size: 4G + + - id: usr-a + size: 1G + + - id: usr-b + size: 1G + + - id: usr-hash-a + size: 128M + + - id: usr-hash-b + size: 128M + + - id: trident + label: trident + size: 512M + + - id: trident-acl-agent + label: trident-acl-agent + size: 128M + + - id: home + label: home + size: 1G + + - id: srv + label: srv + size: 128M + + verity: + - id: usrverity + name: usr + dataDeviceId: usr-a + hashDeviceId: usr-hash-a + dataDeviceMountIdType: uuid + hashDeviceMountIdType: uuid + + filesystems: + - deviceId: esp + type: fat32 + mountPoint: + idType: part-label + path: /boot/efi + options: umask=0077 + + - deviceId: boot-a + type: ext4 + mountPoint: + idType: uuid + path: /boot + + - deviceId: usrverity + type: ext4 + mountPoint: + path: /usr + options: defaults,ro + + - deviceId: root-a + type: ext4 + mountPoint: + idType: uuid + path: / + + - deviceId: trident + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/trident + + - deviceId: trident-acl-agent + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/trident-acl-agent + + - deviceId: home + type: ext4 + mountPoint: + idType: part-label + path: /home + + - deviceId: srv + type: ext4 + mountPoint: + idType: part-label + path: /srv + +os: + bootloader: + resetType: hard-reset + hostname: trident-acl-agent-testimg + + selinux: + mode: disabled + + uki: + mode: create + kernelCommandLine: + # Replicates BM base image settings, that would otherwise be lost + extraCommandLine: + - console=tty0 + - console=tty1 + - console=ttyS0 + - rd.debug + - loglevel=6 + - log_buf_len=1M + - systemd.journald.forward_to_console=1 + - rd.hostonly=0 + + packages: + install: + - device-mapper + - dnf + - efibootmgr + - iproute + - iptables + - jq + - kexec-tools + - lvm2 + - openssh-server + - systemd-boot + - systemd-udev + - trident-acl-agent + - veritysetup + - vim + - netplan + remove: + - grub2-efi-binary + + additionalFiles: + - source: files/99-dhcp-eth0.network + destination: /etc/systemd/network/99-dhcp-eth0.network + - source: files/sudoers-wheel + destination: /etc/sudoers.d/wheel + + services: + enable: + - kdump + - tridentd.socket + + users: + - name: testuser + sshPublicKeyPaths: + - files/id_rsa.pub + secondaryGroups: + - wheel + +scripts: + postCustomization: + - path: scripts/post-install.sh + - path: scripts/update-host-status.sh + - path: scripts/prepare-update-config-verity.sh + arguments: + - uki + - path: scripts/duid-type-to-link-layer.sh + +previewFeatures: + - uki diff --git a/tests/images/trident-vm-testimage/base/updateimg-acl-agent.yaml b/tests/images/trident-vm-testimage/base/updateimg-acl-agent.yaml new file mode 100644 index 0000000000..cc32362d6f --- /dev/null +++ b/tests/images/trident-vm-testimage/base/updateimg-acl-agent.yaml @@ -0,0 +1,191 @@ +# UKI usr-verity VM test image for validating trident-acl-agent's +# label-driven A/B update protocol against a real tridentd. +# +# This is the "update" counterpart to baseimg-acl-agent.yaml: it is built as +# a .cosi and served to the initial qcow2-booted VM as the A/B update target, +# not booted directly. Unlike the qcow2 base image, this image enables +# trident-acl-agent.service by default, since after a real A/B update boots +# into this image's root, there is no SSH session available anymore to +# `systemctl enable --now` it as the storm test harness does for the initial +# qcow2 boot. +storage: + bootType: efi + + disks: + - partitionTableType: gpt + partitions: + - id: esp + type: esp + label: esp + size: 512M + + - id: boot-a + size: 256M + + - id: boot-b + size: 256M + + - id: root-a + size: 4G + + - id: root-b + size: 4G + + - id: usr-a + size: 1G + + - id: usr-b + size: 1G + + - id: usr-hash-a + size: 128M + + - id: usr-hash-b + size: 128M + + - id: trident + label: trident + size: 512M + + - id: trident-acl-agent + label: trident-acl-agent + size: 128M + + - id: home + label: home + size: 1G + + - id: srv + label: srv + size: 128M + + verity: + - id: usrverity + name: usr + dataDeviceId: usr-a + hashDeviceId: usr-hash-a + dataDeviceMountIdType: uuid + hashDeviceMountIdType: uuid + + filesystems: + - deviceId: esp + type: fat32 + mountPoint: + idType: part-label + path: /boot/efi + options: umask=0077 + + - deviceId: boot-a + type: ext4 + mountPoint: + idType: uuid + path: /boot + + - deviceId: usrverity + type: ext4 + mountPoint: + path: /usr + options: defaults,ro + + - deviceId: root-a + type: ext4 + mountPoint: + idType: uuid + path: / + + - deviceId: trident + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/trident + + - deviceId: trident-acl-agent + type: ext4 + mountPoint: + idType: part-label + path: /var/lib/trident-acl-agent + + - deviceId: home + type: ext4 + mountPoint: + idType: part-label + path: /home + + - deviceId: srv + type: ext4 + mountPoint: + idType: part-label + path: /srv + +os: + bootloader: + resetType: hard-reset + hostname: trident-acl-agent-testimg + + selinux: + mode: disabled + + uki: + mode: create + kernelCommandLine: + # Replicates BM base image settings, that would otherwise be lost + extraCommandLine: + - console=tty0 + - console=tty1 + - console=ttyS0 + - rd.debug + - loglevel=6 + - log_buf_len=1M + - systemd.journald.forward_to_console=1 + - rd.hostonly=0 + + packages: + install: + - device-mapper + - dnf + - efibootmgr + - iproute + - iptables + - jq + - kexec-tools + - lvm2 + - openssh-server + - systemd-boot + - systemd-udev + - trident-acl-agent + - veritysetup + - vim + - netplan + remove: + - grub2-efi-binary + + additionalFiles: + - source: files/99-dhcp-eth0.network + destination: /etc/systemd/network/99-dhcp-eth0.network + - source: files/sudoers-wheel + destination: /etc/sudoers.d/wheel + + services: + enable: + - kdump + - tridentd.socket + - trident-acl-agent.service + + users: + - name: testuser + sshPublicKeyPaths: + - files/id_rsa.pub + secondaryGroups: + - wheel + +scripts: + postCustomization: + - path: scripts/post-install.sh + - path: scripts/update-host-status.sh + - path: scripts/prepare-update-config-verity.sh + arguments: + - uki + - path: scripts/duid-type-to-link-layer.sh + +previewFeatures: + - uki diff --git a/tools/cmd/storm-trident/main.go b/tools/cmd/storm-trident/main.go index ef550475fd..eb0fd1e308 100644 --- a/tools/cmd/storm-trident/main.go +++ b/tools/cmd/storm-trident/main.go @@ -1,6 +1,7 @@ package main import ( + "tridenttools/storm/aclagent" "tridenttools/storm/e2e" "tridenttools/storm/helpers" "tridenttools/storm/rollback" @@ -34,6 +35,9 @@ func main() { // Add Trident servicing scenario storm.AddScenario(&servicing.TridentServicingScenario{}) + // Add Trident ACL agent scenario + storm.AddScenario(&aclagent.TridentAclAgentScenario{}) + // Add Trident rollback scenario storm.AddScenario(&rollback.TridentRollbackScenario{}) diff --git a/tools/go.mod b/tools/go.mod index cc70e90bfe..d9ff2ec2c8 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -17,6 +17,8 @@ require ( github.com/spf13/viper v1.19.0 google.golang.org/grpc v1.82.1 gopkg.in/yaml.v2 v2.4.0 + k8s.io/api v0.34.1 + k8s.io/apimachinery v0.34.1 libvirt.org/go/libvirtxml v1.11007.0 libvirt.org/libvirt-go-xml v7.4.0+incompatible modernc.org/sqlite v1.20.3 @@ -24,20 +26,28 @@ require ( ) require ( + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/jstemmer/go-junit-report/v2 v2.1.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/stretchr/testify v1.11.1 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/mod v0.35.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/tools v0.44.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/protobuf v1.36.11 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect lukechampine.com/uint128 v1.2.0 // indirect modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect @@ -47,6 +57,9 @@ require ( modernc.org/opt v0.1.3 // indirect modernc.org/strutil v1.1.3 // indirect modernc.org/token v1.0.1 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect ) require ( @@ -78,7 +91,7 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.7.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/stmcginnis/gofish v0.19.0 github.com/subosito/gotenv v1.6.0 // indirect github.com/vishvananda/netlink v1.3.0 diff --git a/tools/go.sum b/tools/go.sum index a6677a3754..4187da1758 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -33,18 +33,23 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5zVrZ4= github.com/glebarez/go-sqlite v1.20.3/go.mod h1:u3N6D/wftiAzIOJtZl6BmedqxmmkDfH3q+ihjqxC9u0= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -64,21 +69,22 @@ github.com/jacobweinstock/iamt v0.0.0-20230502042727-d7cdbe67d9ef h1:G4k02HGmBUf github.com/jacobweinstock/iamt v0.0.0-20230502042727-d7cdbe67d9ef/go.mod h1:FgmiLTU6cJewV4Xgrq6m5o8CUlTQOJtqzaFLGA0mG+E= github.com/jacobweinstock/registrar v0.4.7 h1:s4dOExccgD+Pc7rJC+f3Mc3D+NXHcXUaOibtcEsPxOc= github.com/jacobweinstock/registrar v0.4.7/go.mod h1:PWmkdGFG5/ZdCqgMo7pvB3pXABOLHc5l8oQ0sgmBNDU= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipYafciZUWHEmsc= github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/knqyf263/go-rpmdb v0.1.1 h1:oh68mTCvp1XzxdU7EfafcWzzfstUZAEa3MW0IJye584= github.com/knqyf263/go-rpmdb v0.1.1/go.mod h1:9LQcoMCMQ9vrF7HcDtXfvqGO4+ddxFQ8+YF/0CVGDww= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -94,6 +100,12 @@ github.com/microsoft/storm v0.4.0-alpha1 h1:U4Bn6rZQW33xrAq+s4pVT7D+RG7crWYTcOPb github.com/microsoft/storm v0.4.0-alpha1/go.mod h1:QMLHpLhA/rI2bmMFor4Vxd6N99k1eadlrXv6lPLbXks= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -110,8 +122,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 h1:VstopitMQi3hZP0fzvnsLmzXZdQGc4bEcgu24cp+d4M= github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= @@ -133,14 +145,16 @@ github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stmcginnis/gofish v0.19.0 h1:fmxdRZ5WHfs+4ExArMYoeRfoh+SAxLELKtmoVplBkU4= github.com/stmcginnis/gofish v0.19.0/go.mod h1:lq2jHj2t8Krg0Gx02ABk8MbK7Dz9jvWpO/TGnVksn00= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -152,6 +166,10 @@ github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQ github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= @@ -169,7 +187,11 @@ go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= @@ -179,6 +201,8 @@ golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -186,7 +210,10 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -197,6 +224,8 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= @@ -206,6 +235,8 @@ golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -247,6 +278,8 @@ golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= @@ -254,6 +287,9 @@ golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxb golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -269,6 +305,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -276,6 +314,14 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= libvirt.org/go/libvirtxml v1.11007.0 h1:SNc8wjOprLl0nsR0H1T0qs2pRD3j/0cVrUTZiP931zI= libvirt.org/go/libvirtxml v1.11007.0/go.mod h1:7Oq2BLDstLr/XtoQD8Fr3mfDNrzlI3utYKySXF2xkng= libvirt.org/libvirt-go-xml v7.4.0+incompatible h1:NaCRjbtz//xuTZOp1nDHbe0eu5BQlhIy5PPuc09EWtU= @@ -310,3 +356,11 @@ modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE= modernc.org/z v1.7.0/go.mod h1:hVdgNMh8ggTuRG1rGU8x+xGRFfiQUIAw0ZqlPy8+HyQ= oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc= oras.land/oras-go/v2 v2.6.0/go.mod h1:magiQDfG6H1O9APp+rOsvCPcW1GD2MM7vgnKY0Y+u1o= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/tools/storm/aclagent/README.md b/tools/storm/aclagent/README.md new file mode 100644 index 0000000000..bcbaa71b86 --- /dev/null +++ b/tools/storm/aclagent/README.md @@ -0,0 +1,99 @@ +# Trident ACL agent storm scenario + +`storm-trident aclagent` is the single supported validation entrypoint for the +label-driven trident ACL agent protocol. + +## What it does + +- deploys a QEMU or Azure VM using the existing storm VM helpers +- starts the fake single-node Kubernetes apiserver in-process inside the storm binary +- starts the fake Nebraska/Omaha endpoint in-process inside the storm binary +- seeds bootstrap node labels and simulated Ready flips with an in-process kubelet helper +- talks to the real `tridentd` and real `trident-acl-agent` running inside the VM +- intercepts `reboot` / `systemctl reboot` with a shim so the finalize path can be exercised without tearing down the SSH session + +There is intentionally no fake `tridentd`. + +## Test cases + +- `deploy-vm` +- `check-deployment` +- `run-ab-update` +- `run-rollback` +- `collect-logs` +- `cleanup-vm` + +## Expected image contents + +The VM image used by this scenario must already contain: + +- `tridentd.socket` installed and enabled (starts `tridentd.service` on demand) +- `trident-acl-agent` package installed, but **`trident-acl-agent.service` + left disabled** -- it must not start before a config exists +- the same SSH user/key setup expected by the existing storm servicing scenario + +Both the enabled/disabled state of `trident-acl-agent.service` +(`/etc/systemd/system/multi-user.target.wants/...`) and +`/etc/trident/trident-acl-agent.conf` live under `/etc`, which is not part of +the A/B-swapped `/usr`/root volume pair in this usr-verity layout. That makes +it safe for the scenario to write the config and enable the service once, +after `deploy-vm`, rather than baking enablement into the image: the state +persists across `run-ab-update`'s finalize the same way the config file does. + +`prepareVmForAclAgent` writes `/etc/trident/trident-acl-agent.conf` pointing +at the `localhost:` endpoints storm reverse-SSH-forwards into the VM, +then runs `systemctl enable --now trident-acl-agent.service`. Before that +runs, the service simply isn't started -- no crash-looping, no log noise. + +## Local usage + +The VM image (`make artifacts/trident-vm-acl-agent-testimage.qcow2`) bakes in +the public key from `artifacts/id_rsa.pub`, not `~/.ssh/id_rsa.pub` -- pass +`--ssh-private-key-path` pointing at `artifacts/id_rsa` (the matching private +key) or `check-deployment` will hang/fail trying to authenticate with the +wrong key. + +```bash +make bin/storm-trident +./bin/storm-trident run aclagent deploy-vm \ + --artifacts-dir --ssh-private-key-path /id_rsa +./bin/storm-trident run aclagent check-deployment \ + --artifacts-dir --ssh-private-key-path /id_rsa +./bin/storm-trident run aclagent run-ab-update \ + --artifacts-dir --ssh-private-key-path /id_rsa +./bin/storm-trident run aclagent run-rollback \ + --artifacts-dir --ssh-private-key-path /id_rsa +./bin/storm-trident run aclagent collect-logs \ + --artifacts-dir --ssh-private-key-path /id_rsa +./bin/storm-trident run aclagent cleanup-vm \ + --artifacts-dir --ssh-private-key-path /id_rsa +``` + +Common overrides mirror other storm VM scenarios, for example: + +```bash +./bin/storm-trident run aclagent run-ab-update \ + --platform qemu \ + --artifacts-dir ./artifacts \ + --output-path ./output/aclagent \ + --ssh-private-key-path ./artifacts/id_rsa \ + --api-server-port 18080 \ + --nebraska-port 18081 +``` + +## Reboot choice + +This scenario keeps the shim-based reboot interception from the old tester. +That is less realistic than a full VM reboot, but it keeps the test deterministic +and lets the storm runner hold the reverse SSH tunnels and in-process fake services +steady while the agent drives the finalize path. + +## `run-rollback` + +`run-rollback` exercises the `rollback` annotation end-to-end against +tridentd's stable `RollbackService` gRPC API (`RollbackStage`/ +`RollbackFinalize`), followed by a real reboot and post-reboot commit - +mirroring `run-ab-update`'s stage/finalize/commit flow. It must run *after* +`run-ab-update` in the same VM lifetime, since rollback re-activates the +volume that was active before `run-ab-update`'s finalize and there is +nothing to roll back to on a freshly-deployed VM. diff --git a/tools/storm/aclagent/proxies/apiserver.go b/tools/storm/aclagent/proxies/apiserver.go new file mode 100644 index 0000000000..73edf1ba96 --- /dev/null +++ b/tools/storm/aclagent/proxies/apiserver.go @@ -0,0 +1,357 @@ +package proxies + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "strings" + "sync" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type NodeStore struct { + mu sync.RWMutex + node *corev1.Node + watchers map[int]chan *corev1.Node + nextID int + resourceVersion int64 +} + +func NewSeedNode(name string, labels map[string]string) *corev1.Node { + seed := &corev1.Node{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Node"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + } + for key, value := range labels { + seed.Labels[key] = value + } + return seed +} + +func LoadSeedNode(data []byte) (*corev1.Node, error) { + var node corev1.Node + if err := json.Unmarshal(data, &node); err != nil { + return nil, fmt.Errorf("failed to parse seed node json: %w", err) + } + if node.Name == "" { + return nil, fmt.Errorf("seed node json must set metadata.name") + } + if node.APIVersion == "" { + node.APIVersion = "v1" + } + if node.Kind == "" { + node.Kind = "Node" + } + if node.Labels == nil { + node.Labels = map[string]string{} + } + if node.Annotations == nil { + node.Annotations = map[string]string{} + } + return &node, nil +} + +func NewNodeStore(seed *corev1.Node) *NodeStore { + node := seed.DeepCopy() + store := &NodeStore{node: node, watchers: map[int]chan *corev1.Node{}, resourceVersion: 1} + node.ResourceVersion = "1" + return store +} + +// bumpLocked increments the store's resourceVersion counter and stamps it +// onto the current node object. Every real Kubernetes object always carries +// metadata.resourceVersion, and kube-rs's watcher() rejects watch events +// (and the LIST used to bootstrap a watch) that omit it, so this must be +// set on every mutation. Callers must hold s.mu for writing. +func (s *NodeStore) bumpLocked() { + s.resourceVersion++ + s.node.ResourceVersion = fmt.Sprintf("%d", s.resourceVersion) +} + +func (s *NodeStore) Snapshot() *corev1.Node { + s.mu.RLock() + defer s.mu.RUnlock() + return s.node.DeepCopy() +} + +func (s *NodeStore) MergePatch(raw []byte) (*corev1.Node, error) { + var patch metadataPatch + if err := json.Unmarshal(raw, &patch); err != nil { + return nil, fmt.Errorf("failed to parse merge patch: %w", err) + } + + s.mu.Lock() + defer s.mu.Unlock() + applyOptionalStringMap(s.node.Labels, patch.Metadata.Labels) + applyOptionalStringMap(s.node.Annotations, patch.Metadata.Annotations) + if patch.Status.Conditions != nil { + s.node.Status.Conditions = append([]corev1.NodeCondition(nil), (*patch.Status.Conditions)...) + } + s.bumpLocked() + s.broadcastLocked() + return s.node.DeepCopy(), nil +} + +func (s *NodeStore) PatchLabels(labels map[string]string) *corev1.Node { + s.mu.Lock() + defer s.mu.Unlock() + for key, value := range labels { + s.node.Labels[key] = value + } + s.bumpLocked() + s.broadcastLocked() + return s.node.DeepCopy() +} + +func (s *NodeStore) PatchAnnotations(annotations map[string]string) *corev1.Node { + s.mu.Lock() + defer s.mu.Unlock() + for key, value := range annotations { + s.node.Annotations[key] = value + } + s.bumpLocked() + s.broadcastLocked() + return s.node.DeepCopy() +} + +func (s *NodeStore) SetReadyCondition(ready bool) *corev1.Node { + s.mu.Lock() + defer s.mu.Unlock() + status := corev1.ConditionFalse + message := "Simulated reboot in progress" + reason := "TridentACLAgentTesterReboot" + if ready { + status = corev1.ConditionTrue + message = "Node ready" + reason = "TridentACLAgentTesterReady" + } + s.node.Status.Conditions = []corev1.NodeCondition{{ + Type: corev1.NodeReady, + Status: status, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + }} + s.bumpLocked() + s.broadcastLocked() + return s.node.DeepCopy() +} + +func (s *NodeStore) Subscribe() (int, <-chan *corev1.Node, *corev1.Node) { + s.mu.Lock() + defer s.mu.Unlock() + id := s.nextID + s.nextID++ + ch := make(chan *corev1.Node, 8) + s.watchers[id] = ch + return id, ch, s.node.DeepCopy() +} + +func (s *NodeStore) Unsubscribe(id int) { + s.mu.Lock() + defer s.mu.Unlock() + if ch, ok := s.watchers[id]; ok { + delete(s.watchers, id) + close(ch) + } +} + +func (s *NodeStore) broadcastLocked() { + snapshot := s.node.DeepCopy() + for _, ch := range s.watchers { + select { + case ch <- snapshot.DeepCopy(): + default: + } + } +} + +type APIServer struct { + nodeName string + store *NodeStore +} + +func NewAPIServer(nodeName string, store *NodeStore) *APIServer { + return &APIServer{nodeName: nodeName, store: store} +} + +func (s *APIServer) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/nodes": + // Collection endpoint. kube-rs's watcher always performs an + // initial LIST here (optionally filtered by fieldSelector) before + // switching to a watch on the same collection; both must be + // served or the watcher treats the 404 as fatal and the process + // exits, taking down the whole service. + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if r.URL.Query().Get("watch") == "true" { + s.handleWatch(w, r) + return + } + s.handleList(w, r) + return + case "/api/v1/nodes/" + s.nodeName: + if r.Method == http.MethodGet && r.URL.Query().Get("watch") == "true" { + s.handleWatch(w, r) + return + } + switch r.Method { + case http.MethodGet: + s.handleGet(w, r) + case http.MethodPatch: + s.handlePatch(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } + return + default: + http.NotFound(w, r) + } + }) +} + +func (s *APIServer) ListenAndServe(ctx context.Context, listenAddr string) (net.Listener, error) { + listener, err := net.Listen("tcp", listenAddr) + if err != nil { + return nil, fmt.Errorf("failed to listen on %s: %w", listenAddr, err) + } + server := &http.Server{Handler: s.Handler()} + go func() { + <-ctx.Done() + _ = server.Shutdown(context.Background()) + }() + go func() { _ = server.Serve(listener) }() + return listener, nil +} + +func (s *APIServer) handleGet(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, s.store.Snapshot()) +} + +// handleList serves the collection endpoint's plain (non-watch) LIST +// request. kube-rs's watcher() issues this before it ever opens a watch +// stream, so it must return a well-formed NodeList (including +// metadata.resourceVersion) even though this fake only ever tracks one node. +func (s *APIServer) handleList(w http.ResponseWriter, r *http.Request) { + node := s.store.Snapshot() + items := []corev1.Node{} + if selector := r.URL.Query().Get("fieldSelector"); selector != "" { + if selector == "metadata.name="+s.nodeName { + items = append(items, *node) + } + } else { + items = append(items, *node) + } + list := corev1.NodeList{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "NodeList"}, + ListMeta: metav1.ListMeta{ResourceVersion: node.ResourceVersion}, + Items: items, + } + writeJSON(w, http.StatusOK, &list) +} + +func (s *APIServer) handlePatch(w http.ResponseWriter, r *http.Request) { + if contentType := r.Header.Get("Content-Type"); contentType != "" && !strings.Contains(contentType, "merge-patch+json") { + http.Error(w, "expected application/merge-patch+json", http.StatusUnsupportedMediaType) + return + } + defer r.Body.Close() + body := json.NewDecoder(r.Body) + body.DisallowUnknownFields() + var raw map[string]any + if err := body.Decode(&raw); err != nil { + http.Error(w, fmt.Sprintf("invalid patch body: %v", err), http.StatusBadRequest) + return + } + bytes, err := json.Marshal(raw) + if err != nil { + http.Error(w, fmt.Sprintf("failed to re-marshal patch: %v", err), http.StatusInternalServerError) + return + } + updated, err := s.store.MergePatch(bytes) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + writeJSON(w, http.StatusOK, updated) +} + +func (s *APIServer) handleWatch(w http.ResponseWriter, r *http.Request) { + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "streaming unsupported", http.StatusInternalServerError) + return + } + watchID, ch, current := s.store.Subscribe() + defer s.store.Unsubscribe(watchID) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := writeWatchEvent(w, "ADDED", current); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + flusher.Flush() + for { + select { + case <-r.Context().Done(): + return + case node, ok := <-ch: + if !ok { + return + } + if err := writeWatchEvent(w, "MODIFIED", node); err != nil { + return + } + flusher.Flush() + } + } +} + +func writeWatchEvent(w http.ResponseWriter, eventType string, node *corev1.Node) error { + raw, err := json.Marshal(node) + if err != nil { + return err + } + event := metav1.WatchEvent{Type: eventType, Object: runtime.RawExtension{Raw: raw}} + return json.NewEncoder(w).Encode(&event) +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} + +type metadataPatch struct { + Metadata struct { + Labels map[string]*string `json:"labels"` + Annotations map[string]*string `json:"annotations"` + } `json:"metadata"` + Status struct { + Conditions *[]corev1.NodeCondition `json:"conditions"` + } `json:"status"` +} + +func applyOptionalStringMap(target map[string]string, patch map[string]*string) { + for key, value := range patch { + if value == nil { + delete(target, key) + continue + } + target[key] = *value + } +} diff --git a/tools/storm/aclagent/proxies/constants.go b/tools/storm/aclagent/proxies/constants.go new file mode 100644 index 0000000000..2077ba55d1 --- /dev/null +++ b/tools/storm/aclagent/proxies/constants.go @@ -0,0 +1,10 @@ +package proxies + +const ( + UpdateRequestAnnotation = "acl.azure.com/update-request" + UpdateStatusAnnotation = "acl.azure.com/update-status" + NodeImageVersionLabel = "kubernetes.azure.com/node-image-version" + + DefaultNodeName = "trident-node" + DefaultMarkerFile = "./trident-acl-agent-reboot-signal" +) diff --git a/tools/storm/aclagent/proxies/imageserver.go b/tools/storm/aclagent/proxies/imageserver.go new file mode 100644 index 0000000000..94da81076c --- /dev/null +++ b/tools/storm/aclagent/proxies/imageserver.go @@ -0,0 +1,57 @@ +package proxies + +import ( + "context" + "fmt" + "net" + "net/http" + "path/filepath" +) + +// ImageServer serves a single OS update image (e.g. a .cosi file) over plain +// HTTP so trident-acl-agent's fake Nebraska endpoint can point tridentd at a +// real, downloadable artifact during A/B update staging. tridentd downloads +// the image itself (not the acl-agent), so this just needs to serve the raw +// bytes at a stable path. +type ImageServer struct { + // ImagePath is the local filesystem path to the image file to serve. + ImagePath string +} + +// Handler returns an http.Handler that serves ImagePath at the request path's +// base name (e.g. "/acl.cosi"), regardless of the requested path, so it works +// whether the caller mounts it at "/" or "/images/". +func (s *ImageServer) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + http.ServeFile(w, r, s.ImagePath) + }) +} + +// ListenAndServe starts the image server on listenAddr and serves the image +// at every path under the given package name (so codebase + package name +// joins correctly regardless of trailing slash handling). +func (s *ImageServer) ListenAndServe(ctx context.Context, listenAddr string) (net.Listener, error) { + listener, err := net.Listen("tcp", listenAddr) + if err != nil { + return nil, fmt.Errorf("failed to listen on %s: %w", listenAddr, err) + } + mux := http.NewServeMux() + mux.Handle("/", s.Handler()) + server := &http.Server{Handler: mux} + go func() { + <-ctx.Done() + _ = server.Shutdown(context.Background()) + }() + go func() { _ = server.Serve(listener) }() + return listener, nil +} + +// PackageBaseName returns the file name portion of ImagePath, used as both +// the Nebraska package name and the served URL path segment. +func (s *ImageServer) PackageBaseName() string { + return filepath.Base(s.ImagePath) +} diff --git a/tools/storm/aclagent/proxies/kubelet.go b/tools/storm/aclagent/proxies/kubelet.go new file mode 100644 index 0000000000..36522987c0 --- /dev/null +++ b/tools/storm/aclagent/proxies/kubelet.go @@ -0,0 +1,183 @@ +package proxies + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const RebootStateAnnotation = "trident-acl-agent/reboot-state" + +type KubeletProxy struct { + HTTPClient *http.Client + APIServerURL string + NodeName string + NodeStore *NodeStore + BootstrapLabels map[string]string + MarkerFile string + RebootDuration time.Duration +} + +func (k *KubeletProxy) Run(ctx context.Context) error { + if k.NodeStore != nil { + if len(k.BootstrapLabels) > 0 { + k.NodeStore.PatchLabels(k.BootstrapLabels) + } + k.NodeStore.SetReadyCondition(true) + } else { + if len(k.BootstrapLabels) > 0 { + if err := patchStringMap(ctx, k.client(), k.nodeURL(), "labels", k.BootstrapLabels); err != nil { + return err + } + } + if err := patchReadyCondition(ctx, k.client(), k.nodeURL(), true); err != nil { + return err + } + } + + if k.MarkerFile == "" { + k.MarkerFile = DefaultMarkerFile + } + if k.RebootDuration <= 0 { + k.RebootDuration = 30 * time.Second + } + + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if _, err := os.Stat(k.MarkerFile); err == nil { + if k.NodeStore != nil { + k.NodeStore.SetReadyCondition(false) + k.NodeStore.PatchAnnotations(map[string]string{RebootStateAnnotation: "not-ready"}) + } else { + if err := patchReadyCondition(ctx, k.client(), k.nodeURL(), false); err != nil { + return err + } + if err := patchStringMap(ctx, k.client(), k.nodeURL(), "annotations", map[string]string{RebootStateAnnotation: "not-ready"}); err != nil { + return err + } + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(k.RebootDuration): + } + if k.NodeStore != nil { + k.NodeStore.SetReadyCondition(true) + k.NodeStore.PatchAnnotations(map[string]string{RebootStateAnnotation: "ready"}) + } else { + if err := patchReadyCondition(ctx, k.client(), k.nodeURL(), true); err != nil { + return err + } + if err := patchStringMap(ctx, k.client(), k.nodeURL(), "annotations", map[string]string{RebootStateAnnotation: "ready"}); err != nil { + return err + } + } + if err := os.Remove(k.MarkerFile); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove reboot marker %s: %w", k.MarkerFile, err) + } + } + } + } +} + +func WriteRebootMarker(markerFile string) error { + if markerFile == "" { + markerFile = DefaultMarkerFile + } + if err := os.MkdirAll(filepath.Dir(markerFile), 0o755); err != nil { + return fmt.Errorf("failed to create reboot marker directory: %w", err) + } + return os.WriteFile(markerFile, []byte("reboot-requested\n"), 0o644) +} + +func (k *KubeletProxy) nodeURL() string { + return strings.TrimRight(k.APIServerURL, "/") + "/api/v1/nodes/" + k.NodeName +} + +func (k *KubeletProxy) client() *http.Client { + if k.HTTPClient != nil { + return k.HTTPClient + } + return http.DefaultClient +} + +func patchStringMap(ctx context.Context, client *http.Client, nodeURL string, field string, values map[string]string) error { + body, err := json.Marshal(map[string]any{ + "metadata": map[string]any{field: values}, + }) + if err != nil { + return err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPatch, nodeURL, bytes.NewReader(body)) + if err != nil { + return err + } + request.Header.Set("Content-Type", "application/merge-patch+json") + response, err := client.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode >= 300 { + return fmt.Errorf("fake apiserver patch failed with %s", response.Status) + } + return nil +} + +func patchReadyCondition(ctx context.Context, client *http.Client, nodeURL string, ready bool) error { + status := corev1.ConditionFalse + message := "Simulated reboot in progress" + reason := "TridentACLAgentTesterReboot" + if ready { + status = corev1.ConditionTrue + message = "Node ready" + reason = "TridentACLAgentTesterReady" + } + + condition := corev1.NodeCondition{ + Type: corev1.NodeReady, + Status: status, + LastHeartbeatTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + } + + body, err := json.Marshal(map[string]any{ + "status": map[string]any{ + "conditions": []corev1.NodeCondition{condition}, + }, + }) + if err != nil { + return err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPatch, nodeURL, bytes.NewReader(body)) + if err != nil { + return err + } + request.Header.Set("Content-Type", "application/merge-patch+json") + response, err := client.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode >= 300 { + return fmt.Errorf("fake apiserver status patch failed with %s", response.Status) + } + return nil +} diff --git a/tools/storm/aclagent/proxies/nebraska.go b/tools/storm/aclagent/proxies/nebraska.go new file mode 100644 index 0000000000..6062d97c01 --- /dev/null +++ b/tools/storm/aclagent/proxies/nebraska.go @@ -0,0 +1,155 @@ +package proxies + +import ( + "context" + "encoding/xml" + "fmt" + "net" + "net/http" + "os" + + "gopkg.in/yaml.v3" +) + +type NebraskaScenario struct { + Available bool `yaml:"available"` + Version string `yaml:"version,omitempty"` + URL string `yaml:"url,omitempty"` + SHA384 string `yaml:"sha384,omitempty"` + PackageName string `yaml:"package-name,omitempty"` +} + +func LoadNebraskaScenario(path string) (*NebraskaScenario, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read Nebraska scenario %s: %w", path, err) + } + var scenario NebraskaScenario + if err := yaml.Unmarshal(data, &scenario); err != nil { + return nil, fmt.Errorf("failed to parse Nebraska scenario yaml: %w", err) + } + return &scenario, nil +} + +type NebraskaProxy struct { + Scenario *NebraskaScenario +} + +func (p *NebraskaProxy) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + defer r.Body.Close() + var request omahaRequest + if err := xml.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, fmt.Sprintf("failed to parse Omaha request: %v", err), http.StatusBadRequest) + return + } + appID := "test" + if len(request.Apps) > 0 && request.Apps[0].AppID != "" { + appID = request.Apps[0].AppID + } + response, err := p.Scenario.BuildResponse(appID) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write(response) + }) +} + +func (p *NebraskaProxy) ListenAndServe(ctx context.Context, listenAddr string) (net.Listener, error) { + listener, err := net.Listen("tcp", listenAddr) + if err != nil { + return nil, fmt.Errorf("failed to listen on %s: %w", listenAddr, err) + } + server := &http.Server{Handler: p.Handler()} + go func() { + <-ctx.Done() + _ = server.Shutdown(context.Background()) + }() + go func() { _ = server.Serve(listener) }() + return listener, nil +} + +func (s *NebraskaScenario) BuildResponse(appID string) ([]byte, error) { + response := omahaResponse{XMLName: xml.Name{Local: "response"}, Protocol: "3.0", Server: "tester", Daystart: daystart{ElapsedSeconds: 0}, Apps: []omahaApp{{AppID: appID, Status: "ok"}}} + if !s.Available { + response.Apps[0].UpdateCheck = &updateCheck{Status: "noupdate"} + } else { + version := s.Version + if version == "" { + version = "1.0.0" + } + packageName := s.PackageName + if packageName == "" { + packageName = "acl.cosi" + } + baseURL := s.URL + if baseURL == "" { + baseURL = "https://example.invalid/images/" + } + hash := s.SHA384 + if hash == "" { + hash = "ignored" + } + response.Apps[0].UpdateCheck = &updateCheck{ + Status: "ok", + URLs: &urls{Entries: []urlEntry{{Codebase: baseURL}}}, + Manifest: &manifest{Version: version, Packages: &packages{Entries: []packageEntry{{Hash: hash, Name: packageName, Size: 1, Required: true}}}}, + } + } + payload, err := xml.MarshalIndent(response, "", " ") + if err != nil { + return nil, err + } + return append([]byte(xml.Header), payload...), nil +} + +type omahaRequest struct { + Apps []struct { + AppID string `xml:"appid,attr"` + } `xml:"app"` +} +type omahaResponse struct { + XMLName xml.Name `xml:"response"` + Protocol string `xml:"protocol,attr"` + Server string `xml:"server,attr"` + Daystart daystart `xml:"daystart"` + Apps []omahaApp `xml:"app"` +} +type daystart struct { + ElapsedSeconds int `xml:"elapsed_seconds,attr"` +} +type omahaApp struct { + AppID string `xml:"appid,attr"` + Status string `xml:"status,attr"` + UpdateCheck *updateCheck `xml:"updatecheck,omitempty"` +} +type updateCheck struct { + Status string `xml:"status,attr"` + URLs *urls `xml:"urls,omitempty"` + Manifest *manifest `xml:"manifest,omitempty"` +} +type urls struct { + Entries []urlEntry `xml:"url"` +} +type urlEntry struct { + Codebase string `xml:"codebase,attr"` +} +type manifest struct { + Version string `xml:"version,attr"` + Packages *packages `xml:"packages,omitempty"` +} +type packages struct { + Entries []packageEntry `xml:"package"` +} +type packageEntry struct { + Hash string `xml:"hash,attr,omitempty"` + Name string `xml:"name,attr"` + Size int `xml:"size,attr"` + Required bool `xml:"required,attr"` +} diff --git a/tools/storm/aclagent/proxies/rp.go b/tools/storm/aclagent/proxies/rp.go new file mode 100644 index 0000000000..e165a6ee88 --- /dev/null +++ b/tools/storm/aclagent/proxies/rp.go @@ -0,0 +1,169 @@ +package proxies + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" +) + +type RPClient struct { + HTTPClient *http.Client + APIServerURL string + NodeName string +} + +type updateRequest struct { + SchemaVersion string `json:"schemaVersion"` + NodeUpdateID string `json:"nodeUpdateId"` + OperationID string `json:"operationId"` + Operation string `json:"operation"` + TargetVersion string `json:"targetVersion,omitempty"` +} + +type updateStatus struct { + SchemaVersion string `json:"schemaVersion"` + NodeUpdateID string `json:"nodeUpdateId"` + OperationID string `json:"operationId"` + Operation string `json:"operation"` + Code string `json:"code"` +} + +func (c *RPClient) RunScenario(ctx context.Context, scenario *Scenario) (*ScenarioReport, error) { + report := &ScenarioReport{Steps: make([]StepReport, 0, len(scenario.Steps)), Passed: true} + for index, step := range scenario.Steps { + started := time.Now() + stepReport, err := c.runStep(ctx, index, step) + if err != nil { + return nil, err + } + stepReport.ElapsedMS = time.Since(started).Milliseconds() + report.Steps = append(report.Steps, *stepReport) + report.Passed = report.Passed && stepReport.Passed + } + return report, nil +} + +func (c *RPClient) runStep(ctx context.Context, index int, step ScenarioStep) (*StepReport, error) { + switch { + case step.Patch != nil: + if err := c.patchNodeRequest(ctx, step.Patch); err != nil { + return nil, err + } + return &StepReport{Index: index, Kind: "patch", Passed: true, Message: "patched fake Node request annotation"}, nil + case step.Expect != nil: + return c.expectStatus(ctx, index, step.Expect) + default: + return nil, fmt.Errorf("step %d had no recognized action", index) + } +} + +func (c *RPClient) expectStatus(ctx context.Context, index int, step *ExpectStep) (*StepReport, error) { + deadline := time.Now().Add(step.Timeout) + pollInterval := 500 * time.Millisecond + var lastObserved map[string]string + matched := false + for time.Now().Before(deadline) { + node, err := c.getNode(ctx) + if err != nil { + return nil, err + } + status, _ := decodeStatus(node) + if status != nil { + lastObserved = map[string]string{"operation-id": status.OperationID, "operation": status.Operation, "code": status.Code} + if status.Code == step.Code && (step.OperationID == "" || status.OperationID == step.OperationID) && (step.Operation == "" || status.Operation == step.Operation) { + matched = true + break + } + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(pollInterval): + } + } + passed := matched + message := "observed expected status" + if step.ExpectTimeout { + passed = !matched + message = "timed out as expected" + } + if !passed && !step.ExpectTimeout { + message = "status expectation failed" + } + return &StepReport{Index: index, Kind: "expect", Passed: passed, Message: message, Expected: map[string]any{"operation-id": step.OperationID, "operation": step.Operation, "code": step.Code, "timeout": step.Timeout.String()}, Actual: lastObserved}, nil +} + +func (c *RPClient) patchNodeRequest(ctx context.Context, step *PatchStep) error { + request := updateRequest{SchemaVersion: "1.0", NodeUpdateID: step.NodeUpdateID, OperationID: step.OperationID, Operation: step.Operation, TargetVersion: step.TargetOSImageVersion} + raw, err := json.Marshal(request) + if err != nil { + return err + } + body, err := json.Marshal(map[string]any{"metadata": map[string]any{"annotations": map[string]string{UpdateRequestAnnotation: string(raw)}}}) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, c.nodeURL(), bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/merge-patch+json") + resp, err := c.client().Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + return fmt.Errorf("fake apiserver patch failed with %s", resp.Status) + } + return nil +} + +func decodeStatus(node *corev1.Node) (*updateStatus, error) { + raw := node.Annotations[UpdateStatusAnnotation] + if raw == "" { + return nil, nil + } + var status updateStatus + if err := json.Unmarshal([]byte(raw), &status); err != nil { + return nil, err + } + return &status, nil +} + +func (c *RPClient) getNode(ctx context.Context) (*corev1.Node, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.nodeURL(), nil) + if err != nil { + return nil, err + } + response, err := c.client().Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode >= 300 { + return nil, fmt.Errorf("fake apiserver get failed with %s", response.Status) + } + var node corev1.Node + if err := json.NewDecoder(response.Body).Decode(&node); err != nil { + return nil, fmt.Errorf("failed to decode fake Node response: %w", err) + } + return &node, nil +} + +func (c *RPClient) nodeURL() string { + return strings.TrimRight(c.APIServerURL, "/") + "/api/v1/nodes/" + c.NodeName +} + +func (c *RPClient) client() *http.Client { + if c.HTTPClient != nil { + return c.HTTPClient + } + return http.DefaultClient +} diff --git a/tools/storm/aclagent/proxies/scenario.go b/tools/storm/aclagent/proxies/scenario.go new file mode 100644 index 0000000000..d43e974c1a --- /dev/null +++ b/tools/storm/aclagent/proxies/scenario.go @@ -0,0 +1,96 @@ +package proxies + +import ( + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +type Scenario struct { + Steps []ScenarioStep `yaml:"steps"` +} + +type ScenarioStep struct { + Patch *PatchStep `yaml:"patch,omitempty"` + Expect *ExpectStep `yaml:"expect,omitempty"` + AssertFailureReason string `yaml:"assert-failure-reason,omitempty"` +} + +type PatchStep struct { + NodeUpdateID string `yaml:"node-update-id,omitempty" json:"nodeUpdateId,omitempty"` + OperationID string `yaml:"operation-id,omitempty" json:"operationId,omitempty"` + Operation string `yaml:"operation,omitempty" json:"operation,omitempty"` + TargetOSImageVersion string `yaml:"target-os-image-version,omitempty" json:"targetVersion,omitempty"` +} + +type ExpectStep struct { + OperationID string `yaml:"operation-id,omitempty" json:"operationId,omitempty"` + Operation string `yaml:"operation,omitempty" json:"operation,omitempty"` + Code string `yaml:"code" json:"code"` + Timeout time.Duration `yaml:"-" json:"timeoutSeconds"` + TimeoutRaw string `yaml:"timeout,omitempty" json:"-"` + ExpectTimeout bool `yaml:"expect-timeout,omitempty" json:"expectTimeout,omitempty"` +} + +type ScenarioReport struct { + Passed bool `json:"passed"` + Steps []StepReport `json:"steps"` +} + +type StepReport struct { + Index int `json:"index"` + Kind string `json:"kind"` + Passed bool `json:"passed"` + ElapsedMS int64 `json:"elapsedMs"` + Message string `json:"message"` + Expected any `json:"expected,omitempty"` + Actual any `json:"actual,omitempty"` +} + +func LoadScenario(path string) (*Scenario, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read scenario %s: %w", path, err) + } + var scenario Scenario + if err := yaml.Unmarshal(data, &scenario); err != nil { + return nil, fmt.Errorf("failed to parse scenario yaml: %w", err) + } + if err := scenario.Validate(); err != nil { + return nil, err + } + return &scenario, nil +} + +func (s *Scenario) Validate() error { + for index := range s.Steps { + step := &s.Steps[index] + kinds := 0 + if step.Patch != nil { + kinds++ + } + if step.Expect != nil { + kinds++ + } + if step.AssertFailureReason != "" { + kinds++ + } + if kinds != 1 { + return fmt.Errorf("scenario step %d must set exactly one of patch/expect/assert-failure-reason", index) + } + if step.Expect != nil { + timeout := 60 * time.Second + if step.Expect.TimeoutRaw != "" { + var err error + timeout, err = time.ParseDuration(step.Expect.TimeoutRaw) + if err != nil { + return fmt.Errorf("scenario step %d has invalid timeout %q: %w", index, step.Expect.TimeoutRaw, err) + } + } + step.Expect.Timeout = timeout + } + } + return nil +} diff --git a/tools/storm/aclagent/tests/logs.go b/tools/storm/aclagent/tests/logs.go new file mode 100644 index 0000000000..b61ef7ca0b --- /dev/null +++ b/tools/storm/aclagent/tests/logs.go @@ -0,0 +1,11 @@ +package tests + +import ( + stormaclconfig "tridenttools/storm/aclagent/utils/config" + stormvm "tridenttools/storm/utils/vm" + stormvmconfig "tridenttools/storm/utils/vm/config" +) + +func FetchLogs(testConfig stormaclconfig.TestConfig, vmConfig stormvmconfig.AllVMConfig) error { + return stormvm.FetchLogs(vmConfig, testConfig.OutputPath) +} diff --git a/tools/storm/aclagent/tests/rollback.go b/tools/storm/aclagent/tests/rollback.go new file mode 100644 index 0000000000..07bc8f86c6 --- /dev/null +++ b/tools/storm/aclagent/tests/rollback.go @@ -0,0 +1,134 @@ +package tests + +import ( + "context" + "fmt" + "os" + "time" + + stormproxies "tridenttools/storm/aclagent/proxies" + stormaclconfig "tridenttools/storm/aclagent/utils/config" + stormvm "tridenttools/storm/utils/vm" + stormvmconfig "tridenttools/storm/utils/vm/config" +) + +// RunRollback exercises trident-acl-agent's rollback annotation end-to-end +// against the real gRPC-backed RollbackService (rollback_stage + +// rollback_finalize) implemented by tridentd, followed by the real reboot +// and post-reboot commit. It assumes the VM is already staged/finalized to +// testConfig.TargetVersion (i.e. it runs after run-ab-update in the same +// scenario), so ManualRollbackAbStaged/Finalized has a prior version to roll +// back to. +func RunRollback(testConfig stormaclconfig.TestConfig, vmConfig stormvmconfig.AllVMConfig) error { + vmIP, err := stormvm.GetVmIP(vmConfig) + if err != nil { + return fmt.Errorf("failed to get VM IP: %w", err) + } + if err := os.MkdirAll(testConfig.OutputPath, 0o755); err != nil { + return err + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Rollback doesn't stage a new image from Nebraska - it re-activates the + // previously-finalized volume trident already has on disk - so only the + // fake apiserver is needed here, not the Nebraska/image-server mocks + // run-ab-update starts. The agent config still references a Nebraska + // endpoint (required by the config schema) but it is never queried + // during a rollback request. + nodeStore := stormproxies.NewNodeStore(stormproxies.NewSeedNode(testConfig.NodeName, map[string]string{})) + apiServer := stormproxies.NewAPIServer(testConfig.NodeName, nodeStore) + if _, err := apiServer.ListenAndServe(ctx, fmt.Sprintf("0.0.0.0:%d", testConfig.APIServerPort)); err != nil { + return fmt.Errorf("failed to start fake apiserver: %w", err) + } + + nodeStore.PatchLabels(map[string]string{stormproxies.NodeImageVersionLabel: testConfig.TargetVersion}) + nodeStore.SetReadyCondition(true) + + if err := prepareVmForAclAgent(vmConfig.VMConfig, vmIP, testConfig); err != nil { + return err + } + + rp := &stormproxies.RPClient{APIServerURL: fmt.Sprintf("http://%s:%d", testConfig.HostEndpointIP, testConfig.APIServerPort), NodeName: testConfig.NodeName} + scenario := &stormproxies.Scenario{Steps: []stormproxies.ScenarioStep{ + {Patch: &stormproxies.PatchStep{NodeUpdateID: "22222222-2222-2222-2222-222222222222", OperationID: "rollback-op", Operation: "rollback"}}, + {Expect: &stormproxies.ExpectStep{OperationID: "rollback-op", Operation: "rollback", Code: "Success", Timeout: 180 * time.Second}}, + }} + report, err := rp.RunScenario(ctx, scenario) + logScenarioTimeline("rollback stage/finalize", report) + if err != nil { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("ACL agent rollback scenario failed (stage/finalize): %w", err) + } + if !report.Passed { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("ACL agent rollback scenario failed (stage/finalize): %+v", report) + } + + // rollback_finalize triggers a real "systemctl reboot" from + // trident-acl-agent, same as update's finalize - wait it out the same + // way run-ab-update does. + nodeStore.SetReadyCondition(false) + if err := waitForVmRebootAndSshBack(vmConfig, vmIP, testConfig); err != nil { + return fmt.Errorf("failed waiting for VM to come back after rollback finalize reboot: %w", err) + } + nodeStore.SetReadyCondition(true) + + // Same rationale as run-ab-update: the rollback reboot lands on the + // previous root, which needs the agent config/kubeconfig re-delivered + // before it can talk to the fake apiserver again. + if err := prepareVmForAclAgent(vmConfig.VMConfig, vmIP, testConfig); err != nil { + return fmt.Errorf("failed to reconfigure ACL agent on post-rollback-reboot root: %w", err) + } + + finalScenario := &stormproxies.Scenario{Steps: []stormproxies.ScenarioStep{ + {Expect: &stormproxies.ExpectStep{OperationID: "rollback-op.commit", Operation: "commit", Code: "Success", Timeout: 180 * time.Second}}, + }} + finalReport, err := rp.RunScenario(ctx, finalScenario) + logScenarioTimeline("post-rollback-reboot commit", finalReport) + if err != nil { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("ACL agent rollback scenario failed (post-reboot commit): %w", err) + } + if !finalReport.Passed { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("ACL agent rollback scenario failed (post-reboot commit): %+v", finalReport) + } + + snapshot := nodeStore.Snapshot() + if got := snapshot.Annotations[stormproxies.UpdateStatusAnnotation]; got == "" { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("final rollback status annotation missing") + } + + // Regression coverage for the "rollback with nothing to roll back" + // bug: the only AB rollback available was just consumed above, so a + // second rollback request now must be detected as a no-op (via + // RollbackStage's servicing_kind, which tridentd now reports the same + // way update/install do) rather than reporting a false Success and + // rebooting the node again for no reason. This exercises that fix + // end-to-end against the real tridentd, not just the mock. + secondScenario := &stormproxies.Scenario{Steps: []stormproxies.ScenarioStep{ + {Patch: &stormproxies.PatchStep{NodeUpdateID: "33333333-3333-3333-3333-333333333333", OperationID: "rollback-op-2", Operation: "rollback"}}, + {Expect: &stormproxies.ExpectStep{OperationID: "rollback-op-2", Operation: "rollback", Code: "OperationFailed", Timeout: 60 * time.Second}}, + }} + secondReport, err := rp.RunScenario(ctx, secondScenario) + logScenarioTimeline("second rollback with empty chain", secondReport) + if err != nil { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("ACL agent second-rollback (empty chain) scenario failed: %w", err) + } + if !secondReport.Passed { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("ACL agent second-rollback (empty chain) scenario failed: %+v", secondReport) + } + // A no-op rollback must not trigger another reboot: the VM should + // still be reachable immediately, with no reboot wait needed. + if _, err := stormvm.GetVmIP(vmConfig); err != nil { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("VM appears to have rebooted (or become unreachable) after a no-op rollback, which should not trigger a reboot: %w", err) + } + + return collectAclArtifacts(vmConfig.VMConfig, vmIP, testConfig.OutputPath) +} diff --git a/tools/storm/aclagent/tests/update.go b/tools/storm/aclagent/tests/update.go new file mode 100644 index 0000000000..137ef3e73b --- /dev/null +++ b/tools/storm/aclagent/tests/update.go @@ -0,0 +1,444 @@ +package tests + +import ( + "archive/tar" + "context" + "crypto/sha512" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + stormproxies "tridenttools/storm/aclagent/proxies" + stormaclconfig "tridenttools/storm/aclagent/utils/config" + stormfile "tridenttools/storm/utils/file" + stormssh "tridenttools/storm/utils/ssh" + stormvm "tridenttools/storm/utils/vm" + stormvmconfig "tridenttools/storm/utils/vm/config" + + "github.com/sirupsen/logrus" +) + +// sha384File computes the lowercase hex-encoded SHA-384 digest that tridentd +// expects for a given image path. +// +// For a .cosi file, tridentd does NOT hash the whole archive: a COSI is a +// plain tar with an embedded "metadata.json" entry, and tridentd's Host +// Configuration "sha384" field must match the hash of just that entry's +// bytes (see crates/trident/src/osimage/cosi/mod.rs's read_cosi_metadata). +// For any other file, this hashes the whole file's contents directly. +// logScenarioTimeline prints a human-readable, step-by-step trace of a +// scenario's progress through the ACL agent's stage/finalize/commit state +// machine. It runs regardless of pass/fail so a test run's output always +// documents exactly how far the state machine got and why, instead of +// forcing readers to reconstruct the timeline from raw journal logs. +func logScenarioTimeline(label string, report *stormproxies.ScenarioReport) { + if report == nil { + return + } + logrus.Infof("=== %s state machine timeline ===", label) + for _, step := range report.Steps { + status := "PASS" + if !step.Passed { + status = "FAIL" + } + logrus.Infof(" [%d] %-6s kind=%-8s (%dms) %s", step.Index, status, step.Kind, step.ElapsedMS, step.Message) + if !step.Passed { + logrus.Infof(" expected: %+v", step.Expected) + logrus.Infof(" actual: %+v", step.Actual) + } + } + overall := "PASSED" + if !report.Passed { + overall = "FAILED" + } + logrus.Infof("=== %s state machine timeline: %s ===", label, overall) +} + +func sha384File(path string) (string, error) { + if strings.HasSuffix(path, ".cosi") { + return sha384CosiMetadata(path) + } + + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha512.New384() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// sha384CosiMetadata extracts the "metadata.json" entry from a COSI (a plain +// tar archive) and returns the lowercase hex-encoded SHA-384 digest of its +// raw bytes, matching what tridentd validates the Host Configuration's +// "sha384" field against. +func sha384CosiMetadata(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + tr := tar.NewReader(f) + for { + header, err := tr.Next() + if err == io.EOF { + return "", fmt.Errorf("metadata.json entry not found in COSI file %s", path) + } + if err != nil { + return "", fmt.Errorf("failed to read COSI tar entries in %s: %w", path, err) + } + if header.Name != "metadata.json" { + continue + } + h := sha512.New384() + if _, err := io.Copy(h, tr); err != nil { + return "", fmt.Errorf("failed to hash metadata.json in COSI file %s: %w", path, err) + } + return hex.EncodeToString(h.Sum(nil)), nil + } +} + +func RunABUpdate(testConfig stormaclconfig.TestConfig, vmConfig stormvmconfig.AllVMConfig) error { + vmIP, err := stormvm.GetVmIP(vmConfig) + if err != nil { + return fmt.Errorf("failed to get VM IP: %w", err) + } + if err := os.MkdirAll(testConfig.OutputPath, 0o755); err != nil { + return err + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + nodeStore := stormproxies.NewNodeStore(stormproxies.NewSeedNode(testConfig.NodeName, map[string]string{})) + apiServer := stormproxies.NewAPIServer(testConfig.NodeName, nodeStore) + // Bind on all interfaces (not 127.0.0.1) so the VM can reach the fake + // apiserver directly over the libvirt NAT network at testConfig.HostEndpointIP, + // instead of relying on reverse SSH tunnels. Tunnels don't survive a real + // VM reboot; a real host IP does. + if _, err := apiServer.ListenAndServe(ctx, fmt.Sprintf("0.0.0.0:%d", testConfig.APIServerPort)); err != nil { + return fmt.Errorf("failed to start fake apiserver: %w", err) + } + + nebraskaCodebase := testConfig.NebraskaCodebase + nebraskaPackageName := testConfig.NebraskaPackageName + nebraskaSHA384 := testConfig.NebraskaSHA384 + + // tridentd downloads and hashes the image itself as part of staging a + // runtime update; it is not enough for the acl-agent to merely reach the + // fake apiserver/Nebraska endpoints. When a real image is configured, serve + // it over plain HTTP from this same test runner and advertise its real + // SHA-384 hash, so tridentd's download+verify path is exercised faithfully + // instead of failing on an unreachable https://example.invalid URL or a + // hash that doesn't match any downloadable bytes. + imagePath := testConfig.ImagePath + if imagePath == "" { + found, err := stormfile.FindFile(testConfig.ArtifactsDir, ".*\\.cosi$") + if err != nil { + return fmt.Errorf("failed to find a .cosi update image under %s: %w", testConfig.ArtifactsDir, err) + } + imagePath = found + } + + { + hash, err := sha384File(imagePath) + if err != nil { + return fmt.Errorf("failed to hash image %s: %w", imagePath, err) + } + imageServer := &stormproxies.ImageServer{ImagePath: imagePath} + if _, err := imageServer.ListenAndServe(ctx, fmt.Sprintf("0.0.0.0:%d", testConfig.ImageServerPort)); err != nil { + return fmt.Errorf("failed to start fake image server: %w", err) + } + nebraskaCodebase = fmt.Sprintf("http://%s:%d/", testConfig.HostEndpointIP, testConfig.ImageServerPort) + nebraskaPackageName = imageServer.PackageBaseName() + nebraskaSHA384 = hash + } + + nebraska := &stormproxies.NebraskaProxy{Scenario: &stormproxies.NebraskaScenario{ + Available: true, + Version: testConfig.TargetVersion, + URL: nebraskaCodebase, + SHA384: nebraskaSHA384, + PackageName: nebraskaPackageName, + }} + if _, err := nebraska.ListenAndServe(ctx, fmt.Sprintf("0.0.0.0:%d", testConfig.NebraskaPort)); err != nil { + return fmt.Errorf("failed to start fake Nebraska endpoint: %w", err) + } + + nodeStore.PatchLabels(map[string]string{stormproxies.NodeImageVersionLabel: testConfig.ExpectedInitialVolume}) + nodeStore.SetReadyCondition(true) + + if err := prepareVmForAclAgent(vmConfig.VMConfig, vmIP, testConfig); err != nil { + return err + } + + rp := &stormproxies.RPClient{APIServerURL: fmt.Sprintf("http://%s:%d", testConfig.HostEndpointIP, testConfig.APIServerPort), NodeName: testConfig.NodeName} + scenario := &stormproxies.Scenario{Steps: []stormproxies.ScenarioStep{ + {Patch: &stormproxies.PatchStep{NodeUpdateID: "11111111-1111-1111-1111-111111111111", OperationID: "stage-op", Operation: "stage", TargetOSImageVersion: testConfig.TargetVersion}}, + {Expect: &stormproxies.ExpectStep{OperationID: "stage-op", Operation: "stage", Code: "Success", Timeout: 120 * time.Second}}, + {Patch: &stormproxies.PatchStep{NodeUpdateID: "11111111-1111-1111-1111-111111111111", OperationID: "finalize-op", Operation: "finalize", TargetOSImageVersion: testConfig.TargetVersion}}, + }} + report, err := rp.RunScenario(ctx, scenario) + logScenarioTimeline("stage/finalize", report) + if err != nil { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("ACL agent scenario failed (stage/finalize): %w", err) + } + if !report.Passed { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("ACL agent scenario failed (stage/finalize): %+v", report) + } + + // Finalize triggers a real "systemctl reboot" from trident-acl-agent + // itself. Reflect the VM actually going away/coming back in the fake + // Node's Ready condition, then wait for SSH to come back before + // checking the agent's post-reboot commit. + nodeStore.SetReadyCondition(false) + if err := waitForVmRebootAndSshBack(vmConfig, vmIP, testConfig); err != nil { + return fmt.Errorf("failed waiting for VM to come back after finalize reboot: %w", err) + } + nodeStore.SetReadyCondition(true) + + // The real A/B reboot lands on a different root filesystem than the + // one prepareVmForAclAgent originally configured: /etc/trident and + // /var/lib/kubelet are per-root ext4 partitions, not shared storage, + // so the config/kubeconfig written before staging do not carry over + // to the newly-activated root. trident-acl-agent.service is enabled + // by default there (baked into the update image), but it needs its + // config re-delivered before it can talk to the fake Nebraska/API + // server endpoints. Re-run the same delivery+restart steps now that + // we're SSH'd into the post-reboot root. + if err := prepareVmForAclAgent(vmConfig.VMConfig, vmIP, testConfig); err != nil { + return fmt.Errorf("failed to reconfigure ACL agent on post-reboot root: %w", err) + } + + finalScenario := &stormproxies.Scenario{Steps: []stormproxies.ScenarioStep{ + {Expect: &stormproxies.ExpectStep{OperationID: "finalize-op.commit", Operation: "commit", Code: "Success", Timeout: 180 * time.Second}}, + }} + finalReport, err := rp.RunScenario(ctx, finalScenario) + logScenarioTimeline("post-reboot commit", finalReport) + if err != nil { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("ACL agent scenario failed (post-reboot commit): %w", err) + } + if !finalReport.Passed { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("ACL agent scenario failed (post-reboot commit): %+v", finalReport) + } + + snapshot := nodeStore.Snapshot() + if got := snapshot.Annotations[stormproxies.UpdateStatusAnnotation]; got == "" { + collectAclArtifactsBestEffort(vmConfig.VMConfig, vmIP, testConfig.OutputPath) + return fmt.Errorf("final status annotation missing") + } + return collectAclArtifacts(vmConfig.VMConfig, vmIP, testConfig.OutputPath) +} + +// collectAclArtifactsBestEffort collects the same diagnostic artifacts as +// collectAclArtifacts, but on a failure path where the harness is about to +// return an error anyway. run-ab-update's failure is otherwise a dead end +// for diagnostics: the storm-trident test runner marks collect-logs (and +// cleanup-vm) as NOTR ("dependency failure") whenever run-ab-update fails, +// so nothing ever calls collectAclArtifacts and the post-reboot journal +// (trident-acl-agent.log / tridentd.log) that would explain the failure is +// never captured or published as a pipeline artifact. Errors here are +// logged but swallowed so they never mask the original failure. +func collectAclArtifactsBestEffort(cfg stormvmconfig.VMConfig, vmIP string, outputPath string) { + if err := collectAclArtifacts(cfg, vmIP, outputPath); err != nil { + logrus.Warnf("best-effort artifact collection after test failure also failed: %v", err) + } +} + +func prepareVmForAclAgent(cfg stormvmconfig.VMConfig, vmIP string, testConfig stormaclconfig.TestConfig) error { + config := fmt.Sprintf(`[nebraska] +endpoint = "http://%s:%d" +app_id = "trident-acl-agent-storm-test" +poll_interval = "5m" + +[kubernetes] +api_server = "http://%s:%d" +kubeconfig = "/var/lib/kubelet/kubeconfig" +node_name = "%s" + +[trident] +socket = "unix:///run/trident/trident.sock" + +[orchestration] +goal_source = "labels" +`, testConfig.HostEndpointIP, testConfig.NebraskaPort, testConfig.HostEndpointIP, testConfig.APIServerPort, testConfig.NodeName) + + // Write the config to a local temp file and scp it up rather than + // piping it through an SSH heredoc: heredocs are fragile to compose + // with trailing shell operators (a bare "&&" right after the closing + // delimiter is a syntax error), whereas scp-then-move is a plain file + // transfer with no quoting/escaping pitfalls. + localConfigFile, err := os.CreateTemp("", "trident-acl-agent-*.conf") + if err != nil { + return fmt.Errorf("failed to create local temp file for ACL agent config: %w", err) + } + defer os.Remove(localConfigFile.Name()) + if _, err := localConfigFile.WriteString(config); err != nil { + localConfigFile.Close() + return fmt.Errorf("failed to write local temp ACL agent config: %w", err) + } + if err := localConfigFile.Close(); err != nil { + return fmt.Errorf("failed to close local temp ACL agent config: %w", err) + } + + if _, err := stormssh.SshCommandCombinedOutput(cfg, vmIP, "sudo mkdir -p /etc/trident"); err != nil { + return fmt.Errorf("failed to create /etc/trident on VM: %w", err) + } + if err := stormssh.ScpUploadFileWithSudo(cfg, vmIP, localConfigFile.Name(), "/etc/trident/trident-acl-agent.conf"); err != nil { + return fmt.Errorf("failed to upload ACL agent config to VM: %w", err) + } + + // The fake apiserver has no real kubelet-managed kubeconfig backing it, + // and it takes plain HTTP with no auth/TLS, so provide a minimal + // insecure kubeconfig pointing at it instead of relying on a real + // kubelet bootstrap file that doesn't exist on this test image. + kubeconfig := fmt.Sprintf(`apiVersion: v1 +kind: Config +clusters: +- name: fake + cluster: + server: http://%s:%d + insecure-skip-tls-verify: true +contexts: +- name: fake + context: + cluster: fake + user: fake +current-context: fake +users: +- name: fake + user: {} +`, testConfig.HostEndpointIP, testConfig.APIServerPort) + + localKubeconfigFile, err := os.CreateTemp("", "trident-acl-agent-kubeconfig-*.yaml") + if err != nil { + return fmt.Errorf("failed to create local temp file for fake kubeconfig: %w", err) + } + defer os.Remove(localKubeconfigFile.Name()) + if _, err := localKubeconfigFile.WriteString(kubeconfig); err != nil { + localKubeconfigFile.Close() + return fmt.Errorf("failed to write local temp fake kubeconfig: %w", err) + } + if err := localKubeconfigFile.Close(); err != nil { + return fmt.Errorf("failed to close local temp fake kubeconfig: %w", err) + } + if _, err := stormssh.SshCommandCombinedOutput(cfg, vmIP, "sudo mkdir -p /var/lib/kubelet"); err != nil { + return fmt.Errorf("failed to create /var/lib/kubelet on VM: %w", err) + } + if err := stormssh.ScpUploadFileWithSudo(cfg, vmIP, localKubeconfigFile.Name(), "/var/lib/kubelet/kubeconfig"); err != nil { + return fmt.Errorf("failed to upload fake kubeconfig to VM: %w", err) + } + + // Enable (without "--now"), then always issue a single "restart" of + // trident-acl-agent.service. Each RunX test case starts its own fresh + // fake-apiserver/Nebraska instances, so a plain "enable --now" isn't + // enough: it's a no-op restart-wise if the service is already active + // (e.g. run-rollback runs right after run-ab-update, no reboot in + // between), leaving the agent's watch connected to the prior test + // case's now-torn-down apiserver, silently missing the new one and + // timing out. A single unconditional "restart" fixes that by both + // starting the unit if needed and cleanly restarting it if already + // running. Do NOT combine "enable --now" with a separate "restart" + // call: on the post-reboot path the unit auto-starts at boot and can + // already be mid-commit (calling tridentd) by the time these SSH + // commands run, so a second restart right after "--now" started it can + // kill and restart the agent mid-call, and the new instance's retry + // then fails with tridentd's "Servicing is active" error. + command := strings.Join([]string{ + "sudo systemctl restart tridentd.service", + "sudo systemctl enable trident-acl-agent.service", + "sudo systemctl restart trident-acl-agent.service", + fmt.Sprintf("sudo grep -q '%s:%d' /etc/trident/trident-acl-agent.conf", testConfig.HostEndpointIP, testConfig.APIServerPort), + fmt.Sprintf("sudo grep -q '%s:%d' /etc/trident/trident-acl-agent.conf", testConfig.HostEndpointIP, testConfig.NebraskaPort), + }, " && ") + if _, err := stormssh.SshCommandCombinedOutput(cfg, vmIP, command); err != nil { + return fmt.Errorf("failed to prepare VM ACL agent config: %w", err) + } + + // Both services can briefly report "activating" right after + // "enable --now" before settling into "active" -- poll rather than + // checking is-active exactly once. + for _, svc := range []string{"tridentd.service", "trident-acl-agent.service"} { + if err := waitForServiceActive(cfg, vmIP, svc, 30*time.Second); err != nil { + return fmt.Errorf("failed waiting for %s to become active: %w", svc, err) + } + } + return nil +} + +func waitForServiceActive(cfg stormvmconfig.VMConfig, vmIP, service string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + out, err := stormssh.SshCommandCombinedOutput(cfg, vmIP, fmt.Sprintf("sudo systemctl is-active %s", service)) + if err == nil && strings.TrimSpace(out) == "active" { + return nil + } + lastErr = err + time.Sleep(2 * time.Second) + } + + // Pull the service's journal so a timeout is self-diagnosing even when + // the scenario fails before the dedicated collect-logs test case runs. + journal, journalErr := stormssh.SshCommandCombinedOutput(cfg, vmIP, fmt.Sprintf("sudo journalctl -u %s --no-pager -n 200", service)) + if journalErr != nil { + journal = fmt.Sprintf("", journalErr) + } + return fmt.Errorf("service %s did not become active within %s (last error: %v)\njournal for %s:\n%s", service, timeout, lastErr, service, journal) +} + +// waitForVmRebootAndSshBack polls SSH until it is unreachable (confirming +// the agent's real "systemctl reboot" actually took the VM down) and then +// reachable again (confirming it came back up), mirroring the real-reboot +// wait pattern already used by the storm servicing scenario. +func waitForVmRebootAndSshBack(vmConfig stormvmconfig.AllVMConfig, vmIP string, testConfig stormaclconfig.TestConfig) error { + downTimeout := time.Now().Add(60 * time.Second) + for time.Now().Before(downTimeout) { + if _, err := stormssh.SshCommandCombinedOutput(vmConfig.VMConfig, vmIP, "true"); err != nil { + break + } + time.Sleep(2 * time.Second) + } + + upTimeout := time.Now().Add(5 * time.Minute) + for time.Now().Before(upTimeout) { + if _, err := stormssh.SshCommandCombinedOutput(vmConfig.VMConfig, vmIP, "true"); err == nil { + return nil + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("VM did not come back up over SSH within timeout after finalize reboot") +} + +func collectAclArtifacts(cfg stormvmconfig.VMConfig, vmIP string, outputPath string) error { + if outputPath == "" { + return nil + } + cmds := []string{ + "sudo journalctl --no-pager -u trident-acl-agent.service > /tmp/trident-acl-agent.log && sudo chmod 644 /tmp/trident-acl-agent.log", + "sudo journalctl --no-pager -u tridentd.service > /tmp/tridentd.log && sudo chmod 644 /tmp/tridentd.log", + "sudo cat /etc/trident/trident-acl-agent.conf > /tmp/trident-acl-agent.conf && sudo chmod 644 /tmp/trident-acl-agent.conf", + } + for _, cmd := range cmds { + _, _ = stormssh.SshCommandCombinedOutput(cfg, vmIP, cmd) + } + for _, remote := range []string{"/tmp/trident-acl-agent.log", "/tmp/tridentd.log", "/tmp/trident-acl-agent.conf"} { + if err := stormssh.ScpDownloadFile(cfg, vmIP, remote, filepath.Join(outputPath, filepath.Base(remote))); err != nil { + logrus.Warnf("failed to download %s: %v", remote, err) + } + } + return nil +} diff --git a/tools/storm/aclagent/tests/vm.go b/tools/storm/aclagent/tests/vm.go new file mode 100644 index 0000000000..a80ef66eb4 --- /dev/null +++ b/tools/storm/aclagent/tests/vm.go @@ -0,0 +1,43 @@ +package tests + +import ( + "fmt" + + stormaclconfig "tridenttools/storm/aclagent/utils/config" + stormvm "tridenttools/storm/utils/vm" + stormvmconfig "tridenttools/storm/utils/vm/config" + + "github.com/sirupsen/logrus" +) + +func CheckDeployment(testConfig stormaclconfig.TestConfig, vmConfig stormvmconfig.AllVMConfig) error { + return stormvm.CheckDeployment(vmConfig, testConfig.ExpectedInitialVolume) +} + +func DeployVM(testConfig stormaclconfig.TestConfig, vmConfig stormvmconfig.AllVMConfig) error { + if vmConfig.VMConfig.Platform == stormvmconfig.PlatformQEMU { + logrus.Tracef("Deploying VM on QEMU platform with name '%s'", vmConfig.VMConfig.Name) + if err := vmConfig.QemuConfig.DeployQemuVM(vmConfig.VMConfig.Name, testConfig.ArtifactsDir, testConfig.OutputPath, testConfig.Verbose); err != nil { + return fmt.Errorf("failed to deploy qemu vm: %w", err) + } + } else if vmConfig.VMConfig.Platform == stormvmconfig.PlatformAzure { + logrus.Tracef("Deploying VM on Azure platform with name '%s'", vmConfig.VMConfig.Name) + if err := vmConfig.AzureConfig.DeployAzureVM(vmConfig.VMConfig.Name, vmConfig.VMConfig.User); err != nil { + return fmt.Errorf("failed to deploy azure vm: %w", err) + } + } + return nil +} + +func CleanupVM(testConfig stormaclconfig.TestConfig, vmConfig stormvmconfig.AllVMConfig) error { + if vmConfig.VMConfig.Platform == stormvmconfig.PlatformAzure { + if err := vmConfig.AzureConfig.CleanupAzureVM(); err != nil { + return fmt.Errorf("failed to cleanup Azure VM: %w", err) + } + } else if vmConfig.VMConfig.Platform == stormvmconfig.PlatformQEMU { + if err := vmConfig.QemuConfig.CleanupQemuVM(vmConfig.VMConfig.Name); err != nil { + return fmt.Errorf("failed to cleanup QEMU VM: %w", err) + } + } + return nil +} diff --git a/tools/storm/aclagent/trident.go b/tools/storm/aclagent/trident.go new file mode 100644 index 0000000000..2f5ab036f5 --- /dev/null +++ b/tools/storm/aclagent/trident.go @@ -0,0 +1,92 @@ +package aclagent + +import ( + "fmt" + "os" + "path/filepath" + + stormtests "tridenttools/storm/aclagent/tests" + stormaclconfig "tridenttools/storm/aclagent/utils/config" + stormvmazure "tridenttools/storm/utils/vm/azure" + stormvmconfig "tridenttools/storm/utils/vm/config" + stormvmqemu "tridenttools/storm/utils/vm/qemu" + + "github.com/microsoft/storm" + "github.com/sirupsen/logrus" +) + +type TridentAclAgentScenario struct { + args TridentAclAgentScenarioArgs +} + +type TridentAclAgentScenarioArgs struct { + stormaclconfig.TestConfig `embed:""` + stormvmconfig.VMConfig `embed:""` + stormvmqemu.QemuConfig `embed:""` + stormvmazure.AzureConfig `embed:""` + TestCaseToRun string `help:"Name of the test case to run. If not specified, all test cases will be run." default:"all"` +} + +func (s *TridentAclAgentScenario) Name() string { return "aclagent" } +func (s *TridentAclAgentScenario) Args() any { return &s.args } +func (s *TridentAclAgentScenario) Tags() []string { return []string{} } +func (s *TridentAclAgentScenario) StagePaths() []string { return []string{} } +func (s *TridentAclAgentScenario) RequiredFiles() []string { return nil } +func (s TridentAclAgentScenario) Setup(ctx storm.SetupCleanupContext) error { return nil } + +func (s *TridentAclAgentScenario) Cleanup(ctx storm.SetupCleanupContext) error { + if s.args.TestConfig.ForceCleanup { + _ = stormtests.CleanupVM(s.args.TestConfig, stormvmconfig.AllVMConfig{VMConfig: s.args.VMConfig, QemuConfig: s.args.QemuConfig, AzureConfig: s.args.AzureConfig}) + } + return nil +} + +func (s *TridentAclAgentScenario) RegisterTestCases(r storm.TestRegistrar) error { + r.RegisterTestCase("deploy-vm", s.deployVm) + r.RegisterTestCase("check-deployment", s.checkDeployment) + r.RegisterTestCase("run-ab-update", s.runABUpdate) + r.RegisterTestCase("run-rollback", s.runRollback) + r.RegisterTestCase("collect-logs", s.collectLogs) + r.RegisterTestCase("cleanup-vm", s.cleanupVm) + return nil +} + +func (s *TridentAclAgentScenario) runTestCase(tc storm.TestCase, testFunc func(stormaclconfig.TestConfig, stormvmconfig.AllVMConfig) error) error { + if tc.Name() != s.args.TestCaseToRun && s.args.TestCaseToRun != "all" { + tc.Skip(fmt.Sprintf("Test case '%s' does not align to TestCaseToRun '%s'", tc.Name(), s.args.TestCaseToRun)) + return nil + } + logrus.Infof("Running test case '%s'", tc.Name()) + testCaseSpecificConfig := s.args.TestConfig + if testCaseSpecificConfig.OutputPath != "" { + testCaseSpecificConfig.OutputPath = filepath.Join(testCaseSpecificConfig.OutputPath, tc.Name()) + if err := os.MkdirAll(testCaseSpecificConfig.OutputPath, 0o755); err != nil { + tc.FailFromError(err) + } + } + if err := testFunc(testCaseSpecificConfig, stormvmconfig.AllVMConfig{VMConfig: s.args.VMConfig, QemuConfig: s.args.QemuConfig, AzureConfig: s.args.AzureConfig}); err != nil { + logrus.Infof("test case '%s' failed", tc.Name()) + tc.FailFromError(err) + } + logrus.Infof("test case '%s' passed", tc.Name()) + return nil +} + +func (s *TridentAclAgentScenario) deployVm(tc storm.TestCase) error { + return s.runTestCase(tc, stormtests.DeployVM) +} +func (s *TridentAclAgentScenario) checkDeployment(tc storm.TestCase) error { + return s.runTestCase(tc, stormtests.CheckDeployment) +} +func (s *TridentAclAgentScenario) runABUpdate(tc storm.TestCase) error { + return s.runTestCase(tc, stormtests.RunABUpdate) +} +func (s *TridentAclAgentScenario) runRollback(tc storm.TestCase) error { + return s.runTestCase(tc, stormtests.RunRollback) +} +func (s *TridentAclAgentScenario) collectLogs(tc storm.TestCase) error { + return s.runTestCase(tc, stormtests.FetchLogs) +} +func (s *TridentAclAgentScenario) cleanupVm(tc storm.TestCase) error { + return s.runTestCase(tc, stormtests.CleanupVM) +} diff --git a/tools/storm/aclagent/utils/config/config.go b/tools/storm/aclagent/utils/config/config.go new file mode 100644 index 0000000000..297c238f5e --- /dev/null +++ b/tools/storm/aclagent/utils/config/config.go @@ -0,0 +1,19 @@ +package config + +type TestConfig struct { + ArtifactsDir string `help:"Directory containing artifacts for the VM" default:"."` + OutputPath string `help:"Path to the output directory for logs and artifacts" default:"./output"` + Verbose bool `help:"Enable verbose logging" default:"false"` + ForceCleanup bool `help:"Force cleanup of VM when test finishes" default:"false"` + APIServerPort int `help:"Runner port exposed into VM for fake apiserver" default:"18080"` + NebraskaPort int `help:"Runner port exposed into VM for fake Nebraska endpoint" default:"18081"` + TargetVersion string `help:"Target OS image version to request" default:"202507.28.0"` + NebraskaPackageName string `help:"Package name returned by the fake Nebraska endpoint (overridden by the image file name when ImagePath is set)" default:"acl.cosi"` + NebraskaCodebase string `help:"Base URL returned by the fake Nebraska endpoint (overridden to point at the fake image server when ImagePath is set)" default:"https://example.invalid/images/"` + NebraskaSHA384 string `help:"SHA384 returned by the fake Nebraska endpoint (overridden by the real hash of ImagePath when set)" default:"111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111"` + ImagePath string `help:"Path to a real OS update image (e.g. a .cosi file) to serve to tridentd during staging; when set, this takes precedence over NebraskaCodebase/NebraskaPackageName/NebraskaSHA384. When empty, the first *.cosi file found under ArtifactsDir is used."` + ImageServerPort int `help:"Runner port exposed into VM for the fake image server" default:"18082"` + NodeName string `help:"Node name served by the fake apiserver" default:"trident-node"` + HostEndpointIP string `help:"Host IP the VM can reach the fake apiserver/Nebraska endpoints at" default:"192.168.122.1"` + ExpectedInitialVolume string `help:"Expected active volume immediately after deployment" default:"volume-a"` +} diff --git a/tools/storm/utils/vm/qemu/qemu.go b/tools/storm/utils/vm/qemu/qemu.go index fcec985b9e..3f757d1ddb 100644 --- a/tools/storm/utils/vm/qemu/qemu.go +++ b/tools/storm/utils/vm/qemu/qemu.go @@ -19,8 +19,9 @@ import ( ) type QemuConfig struct { - SecureBoot bool `help:"Enable secure boot for the VM" default:"false"` - SerialLog string `help:"Path to the serial log file" default:"/tmp/trident-vm-verity-test.log"` + SecureBoot bool `help:"Enable secure boot for the VM" default:"false"` + SerialLog string `help:"Path to the serial log file" default:"/tmp/trident-vm-verity-test.log"` + ImagePattern string `help:"Regex pattern used to find the base VM image (.qcow2) in the artifacts directory" default:"^trident-vm-.*-testimage.qcow2$"` } func (cfg QemuConfig) DeployQemuVM(vmName string, artifactsDir string, outputPath string, verbose bool) error { @@ -32,7 +33,7 @@ func (cfg QemuConfig) DeployQemuVM(vmName string, artifactsDir string, outputPat } // Find image file - imageFile, err := stormfile.FindFile(artifactsDir, "^trident-vm-.*-testimage.qcow2$") + imageFile, err := stormfile.FindFile(artifactsDir, cfg.ImagePattern) if err != nil { return fmt.Errorf("failed to find image file: %w", err) } From faad3a924696090a0b20b2a21251fb37da7caf9b Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 5 Aug 2026 20:02:17 +0000 Subject: [PATCH 2/4] Makefile: restore go-tools dependency list; drop orphaned @mkdir -p bin A prior commit (37b479e1, retiring the old bin/trident-acl-agent-tester target) removed that targets one dependency from go-tools via an imprecise line-range deletion, which wiped the entire go-tools dependency list instead of just the appended trident-acl-agent-tester token - go-tools has been building nothing (make go-tools was a no-op) since. The same deletion also left an orphaned "@mkdir -p bin" line dangling where the removed targets recipe used to be. Restored the dependency list to its pre-regression contents (minus the retired trident-acl-agent-tester entry, which no longer exists) and removed the orphaned mkdir line. Verified: "make -n go-tools" now correctly expands to build all 7 tools (netlaunch, netlisten, miniproxy, virtdeploy, isopatch, mkcosi, storm-trident, rcp-agent); "make bin/storm-trident" builds successfully. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8c06585d-a82d-475f-a802-83fdfa012d86 --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0ba4ddf93a..ab751c9c52 100644 --- a/Makefile +++ b/Makefile @@ -479,6 +479,7 @@ go.sum: go.mod go mod tidy .PHONY: go-tools +go-tools: bin/netlaunch bin/netlisten bin/miniproxy bin/virtdeploy bin/isopatch bin/mkcosi bin/storm-trident bin/rcp-agent bin/netlaunch: tools/cmd/netlaunch/* tools/go.sum tools/pkg/* tools/pkg/netlaunch/* @mkdir -p bin @@ -519,8 +520,6 @@ bin/rcp-agent: tools/cmd/rcp-agent/* tools/go.sum tools/pkg/rcp/* tools/pkg/rcp/ cd tools && go generate pkg/rcp/tlscerts/certs.go cd tools && go build -o ../bin/rcp-agent ./cmd/rcp-agent/main.go - @mkdir -p bin - # Clean generated RCP TLS certificates .PHONY: clean-rcp-certs clean-rcp-certs: From 8156911b4eb52d03239b127f0450077209c0a2a8 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 5 Aug 2026 22:12:45 +0000 Subject: [PATCH 3/4] storm/aclagent: update goal_source to renamed "annotations" value GoalSource::Labels was renamed to GoalSource::Annotations (TOML value "labels" -> "annotations") in the parent branch. The VM config template this harness writes to /etc/trident/trident-acl-agent.conf still had the old value, which would now fail to deserialize since "labels" is no longer a valid GoalSource variant. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8c06585d-a82d-475f-a802-83fdfa012d86 --- tools/storm/aclagent/tests/update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/storm/aclagent/tests/update.go b/tools/storm/aclagent/tests/update.go index 137ef3e73b..f7c7eea34d 100644 --- a/tools/storm/aclagent/tests/update.go +++ b/tools/storm/aclagent/tests/update.go @@ -273,7 +273,7 @@ node_name = "%s" socket = "unix:///run/trident/trident.sock" [orchestration] -goal_source = "labels" +goal_source = "annotations" `, testConfig.HostEndpointIP, testConfig.NebraskaPort, testConfig.HostEndpointIP, testConfig.APIServerPort, testConfig.NodeName) // Write the config to a local temp file and scp it up rather than From 5a5ecce2999360214c80e47461bccbec27a684cf Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 5 Aug 2026 23:21:28 +0000 Subject: [PATCH 4/4] fix Copilot-flagged storm harness issues on PR #731 - use grep -qF for literal IP:port config checks (avoid regex-dot false match) - waitForVmRebootAndSshBack now fails if SSH never goes down, instead of silently passing when no reboot occurred - remove unimplemented assert-failure-reason scenario field - fix ExpectTimeout step message to not claim a timeout when a match actually occurred (the real failure case) - correct stale docs describing shim-based reboot interception; scenario does a real VM reboot + SSH reachability wait - correct trident-vm-testimage README: only the update image enables trident-acl-agent.service by default, base image leaves it disabled Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8c06585d-a82d-475f-a802-83fdfa012d86 --- .../Testing/TridentAclAgent-Tests.md | 20 +++++++++---------- tests/images/trident-vm-testimage/README.md | 14 ++++++++----- tools/storm/aclagent/README.md | 4 ++-- tools/storm/aclagent/proxies/rp.go | 9 ++++++--- tools/storm/aclagent/proxies/scenario.go | 10 +++------- tools/storm/aclagent/tests/update.go | 9 +++++++-- 6 files changed, 36 insertions(+), 30 deletions(-) diff --git a/docs/Development/Testing/TridentAclAgent-Tests.md b/docs/Development/Testing/TridentAclAgent-Tests.md index 537ed49abf..a3004c6a59 100644 --- a/docs/Development/Testing/TridentAclAgent-Tests.md +++ b/docs/Development/Testing/TridentAclAgent-Tests.md @@ -23,8 +23,7 @@ There is intentionally no fake `tridentd` โ€” the scenario talks to the real - Reading the update image URL/hash from labels and triggering a real Trident `stage` + `finalize` A/B update through the normal gRPC path - Patching back observed-state labels/annotations as the update progresses -- Resuming correctly after a simulated reboot (shim-intercepted, not a real - VM reboot โ€” see [Reboot Choice](#reboot-choice)) +- Resuming correctly after a real reboot (see [Reboot Choice](#reboot-choice)) ## VM Image Contents @@ -144,8 +143,7 @@ The scenario runs these test cases in order: 3. **run-ab-update** โ€” Starts the fake apiserver and fake Nebraska/Omaha endpoints in-process, seeds bootstrap node labels, patches the desired update-image label, and waits for `trident-acl-agent` to drive a real - Trident A/B update to completion (including the shim-based simulated - reboot) + Trident A/B update to completion (including a real reboot) 4. **collect-logs** โ€” Fetches `trident-acl-agent` and Trident logs from the VM via SSH; also runs automatically (with a `journalctl` dump for `trident-acl-agent.service`) if `run-ab-update` times out waiting for the @@ -167,13 +165,13 @@ The scenario runs these test cases in order: ## Reboot Choice -This scenario uses shim-based reboot interception rather than a full VM -reboot: a `reboot`/`systemctl reboot` shim on `PATH` inside the VM signals the -scenario's controller and exits the agent process instead of actually -rebooting. The scenario then restarts `trident-acl-agent` fresh, exercising -its post-reboot resume logic without tearing down the SSH session or the -in-process fake services. This is less realistic than a full reboot, but it -keeps the test deterministic and fast. +This scenario uses a real VM reboot rather than a shim: `trident-acl-agent` +issues a genuine `systemctl reboot` on finalize, and the scenario polls SSH +until it goes unreachable (confirming the reboot actually happened) and then +reachable again (confirming the VM came back up), exercising the agent's +real post-reboot resume logic end to end. This is slower than a shim-based +approach, but it validates the real reboot path instead of a simulation of +it. ## Debugging Failures diff --git a/tests/images/trident-vm-testimage/README.md b/tests/images/trident-vm-testimage/README.md index 4990bcccb2..51d1d3c3b3 100644 --- a/tests/images/trident-vm-testimage/README.md +++ b/tests/images/trident-vm-testimage/README.md @@ -14,11 +14,15 @@ Two sets of images are available: For both, a set of corresponding update images is available. The ACL-agent variant reuses the servicing-style VM image layout but additionally installs -`trident-acl-agent` and enables `trident-acl-agent.service` so storm ACL-agent -scenarios can drive a real in-guest agent talking to `tridentd`. The image does -not preseed /etc/trident/trident-acl-agent.conf with runner-specific tunnel -ports; the test scenario should SSH in after boot and write the real localhost -proxy endpoints for Nebraska and the Kubernetes API server. +`trident-acl-agent` so storm ACL-agent scenarios can drive a real in-guest +agent talking to `tridentd`. The base (qcow2) image installs the package but +leaves `trident-acl-agent.service` disabled -- the storm scenario enables and +starts it itself after seeding a real config. Only the update image enables +`trident-acl-agent.service` by default, since after a real A/B update boots +into it there is no test harness left to `systemctl enable --now` it. Neither +image preseeds /etc/trident/trident-acl-agent.conf with runner-specific +tunnel ports; the test scenario should SSH in after boot and write the real +localhost proxy endpoints for Nebraska and the Kubernetes API server. ## Additional Prerequisites diff --git a/tools/storm/aclagent/README.md b/tools/storm/aclagent/README.md index bcbaa71b86..5da8ecd144 100644 --- a/tools/storm/aclagent/README.md +++ b/tools/storm/aclagent/README.md @@ -8,9 +8,9 @@ label-driven trident ACL agent protocol. - deploys a QEMU or Azure VM using the existing storm VM helpers - starts the fake single-node Kubernetes apiserver in-process inside the storm binary - starts the fake Nebraska/Omaha endpoint in-process inside the storm binary -- seeds bootstrap node labels and simulated Ready flips with an in-process kubelet helper +- seeds bootstrap node annotations and simulated Ready flips with an in-process kubelet helper - talks to the real `tridentd` and real `trident-acl-agent` running inside the VM -- intercepts `reboot` / `systemctl reboot` with a shim so the finalize path can be exercised without tearing down the SSH session +- lets `trident-acl-agent` issue a real `systemctl reboot` on finalize, then polls SSH until it drops and comes back up to confirm the reboot actually happened There is intentionally no fake `tridentd`. diff --git a/tools/storm/aclagent/proxies/rp.go b/tools/storm/aclagent/proxies/rp.go index e165a6ee88..71c9de017c 100644 --- a/tools/storm/aclagent/proxies/rp.go +++ b/tools/storm/aclagent/proxies/rp.go @@ -91,9 +91,12 @@ func (c *RPClient) expectStatus(ctx context.Context, index int, step *ExpectStep message := "observed expected status" if step.ExpectTimeout { passed = !matched - message = "timed out as expected" - } - if !passed && !step.ExpectTimeout { + if passed { + message = "timed out as expected" + } else { + message = "expected no matching status before timeout, but status matched" + } + } else if !passed { message = "status expectation failed" } return &StepReport{Index: index, Kind: "expect", Passed: passed, Message: message, Expected: map[string]any{"operation-id": step.OperationID, "operation": step.Operation, "code": step.Code, "timeout": step.Timeout.String()}, Actual: lastObserved}, nil diff --git a/tools/storm/aclagent/proxies/scenario.go b/tools/storm/aclagent/proxies/scenario.go index d43e974c1a..e175407e59 100644 --- a/tools/storm/aclagent/proxies/scenario.go +++ b/tools/storm/aclagent/proxies/scenario.go @@ -13,9 +13,8 @@ type Scenario struct { } type ScenarioStep struct { - Patch *PatchStep `yaml:"patch,omitempty"` - Expect *ExpectStep `yaml:"expect,omitempty"` - AssertFailureReason string `yaml:"assert-failure-reason,omitempty"` + Patch *PatchStep `yaml:"patch,omitempty"` + Expect *ExpectStep `yaml:"expect,omitempty"` } type PatchStep struct { @@ -74,11 +73,8 @@ func (s *Scenario) Validate() error { if step.Expect != nil { kinds++ } - if step.AssertFailureReason != "" { - kinds++ - } if kinds != 1 { - return fmt.Errorf("scenario step %d must set exactly one of patch/expect/assert-failure-reason", index) + return fmt.Errorf("scenario step %d must set exactly one of patch/expect", index) } if step.Expect != nil { timeout := 60 * time.Second diff --git a/tools/storm/aclagent/tests/update.go b/tools/storm/aclagent/tests/update.go index f7c7eea34d..a06c5b6492 100644 --- a/tools/storm/aclagent/tests/update.go +++ b/tools/storm/aclagent/tests/update.go @@ -361,8 +361,8 @@ users: "sudo systemctl restart tridentd.service", "sudo systemctl enable trident-acl-agent.service", "sudo systemctl restart trident-acl-agent.service", - fmt.Sprintf("sudo grep -q '%s:%d' /etc/trident/trident-acl-agent.conf", testConfig.HostEndpointIP, testConfig.APIServerPort), - fmt.Sprintf("sudo grep -q '%s:%d' /etc/trident/trident-acl-agent.conf", testConfig.HostEndpointIP, testConfig.NebraskaPort), + fmt.Sprintf("sudo grep -qF '%s:%d' /etc/trident/trident-acl-agent.conf", testConfig.HostEndpointIP, testConfig.APIServerPort), + fmt.Sprintf("sudo grep -qF '%s:%d' /etc/trident/trident-acl-agent.conf", testConfig.HostEndpointIP, testConfig.NebraskaPort), }, " && ") if _, err := stormssh.SshCommandCombinedOutput(cfg, vmIP, command); err != nil { return fmt.Errorf("failed to prepare VM ACL agent config: %w", err) @@ -405,13 +405,18 @@ func waitForServiceActive(cfg stormvmconfig.VMConfig, vmIP, service string, time // reachable again (confirming it came back up), mirroring the real-reboot // wait pattern already used by the storm servicing scenario. func waitForVmRebootAndSshBack(vmConfig stormvmconfig.AllVMConfig, vmIP string, testConfig stormaclconfig.TestConfig) error { + wentDown := false downTimeout := time.Now().Add(60 * time.Second) for time.Now().Before(downTimeout) { if _, err := stormssh.SshCommandCombinedOutput(vmConfig.VMConfig, vmIP, "true"); err != nil { + wentDown = true break } time.Sleep(2 * time.Second) } + if !wentDown { + return fmt.Errorf("VM never became unreachable over SSH within %s; reboot did not appear to happen", 60*time.Second) + } upTimeout := time.Now().Add(5 * time.Minute) for time.Now().Before(upTimeout) {