diff --git a/.gitignore b/.gitignore index e7d3febb7d..e472dc8a27 100644 --- a/.gitignore +++ b/.gitignore @@ -366,4 +366,11 @@ vendor/ # Virtdeploy files /tools/vm-netlaunch.yaml -/tools/virt-deploy-metadata.json \ No newline at end of file +/tools/virt-deploy-metadata.json +# Local storm-trident e2e run outputs +/junit*.xml +/logs*/ +/out*/ +/logstream*.log +/*-metrics*.jsonl +/metrics-*.jsonl diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index a0654303a4..100312a497 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -119,6 +119,7 @@ stages: - template: stages/testing_e2e/storm_e2e.yml parameters: stageType: ${{ parameters.stageType }} + acrServiceConnectionName: ${{ parameters.acrServiceConnectionName }} # Makefile validation, only for CI - ${{ if eq(parameters.stageType, 'ci') }}: diff --git a/.pipelines/templates/stages/common_tasks/build-storm-trident.yml b/.pipelines/templates/stages/common_tasks/build-storm-trident.yml new file mode 100644 index 0000000000..b331c98f52 --- /dev/null +++ b/.pipelines/templates/stages/common_tasks/build-storm-trident.yml @@ -0,0 +1,38 @@ +# Builds bin/storm-trident from the checked-out source. Used by the shortcut +# (testing-trident.yml) storm E2E path so the test matrix and validation logic +# come from the branch under test rather than a downloaded, possibly-stale +# go-tools artifact. Mirrors the storm-trident build in building-tools.yml. +# +# Assumes the trident sources are already checked out (checkout_trident.yml) and +# runs on a build-capable amd64 pool (e.g. trident-ubuntu-1es-pool-eastus2). +steps: + - template: avoid-pypi-usage.yml + - template: update-protoc.yml + parameters: + protocArch: x86_64 + - bash: | + set -eux + go install google.golang.org/protobuf/cmd/protoc-gen-go@latest + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest + echo "Adding '$(go env GOPATH)/bin' to PATH for subsequent tasks" + echo "##vso[task.prependpath]$(go env GOPATH)/bin" + displayName: "Install protoc-gen-go" + - bash: | + set -eux + # `make bin/storm-trident` only regenerates the embedded configurations; + # the protobuf (pkg/tridentgrpc) and TLS cert (pkg/rcp/tlscerts) sources + # are generated by other targets (e.g. bin/netlaunch) in a full build. + # Generate them here so the standalone storm-trident build succeeds. + # + # Remove any pre-existing binary first: the test jobs download a go-tools + # artifact (a possibly-stale storm-trident) before this step, and make's + # timestamp check would otherwise treat that binary as up-to-date and skip + # the rebuild, silently using the stale binary. + rm -f bin/storm-trident + cd tools + go generate pkg/rcp/tlscerts/certs.go + go generate pkg/tridentgrpc/grpc.go + cd .. + make bin/storm-trident + displayName: "Build storm-trident from source" + workingDirectory: $(TRIDENT_SOURCE_DIR) diff --git a/.pipelines/templates/stages/testing_e2e/build-storm-trident-stage.yml b/.pipelines/templates/stages/testing_e2e/build-storm-trident-stage.yml new file mode 100644 index 0000000000..1c88f9e4e7 --- /dev/null +++ b/.pipelines/templates/stages/testing_e2e/build-storm-trident-stage.yml @@ -0,0 +1,38 @@ +# Builds storm-trident once from the checked-out source and publishes it as a +# pipeline artifact, so the E2E stages (matrix definition + test execution) can +# consume the branch-under-test binary instead of a downloaded, possibly-stale +# go-tools artifact. Used by the testing-trident.yml shortcut (testingRun). +parameters: + - name: stageName + type: string + default: BuildStormTridentE2E + + # Name of the published artifact containing the storm-trident binary. + - name: artifactName + type: string + default: storm-trident-e2e + +stages: + - stage: ${{ parameters.stageName }} + displayName: Build storm-trident for E2E + dependsOn: [] + jobs: + - job: BuildStormTrident + displayName: Build storm-trident from source + timeoutInMinutes: 15 + pool: + type: linux + name: trident-ubuntu-1es-pool-eastus2 + hostArchitecture: amd64 + variables: + ob_outputDirectory: /tmp/${{ parameters.artifactName }} + ob_artifactBaseName: ${{ parameters.artifactName }} + steps: + - template: ../common_tasks/checkout_trident.yml + - template: ../common_tasks/build-storm-trident.yml + - bash: | + set -eux + mkdir -p $(ob_outputDirectory) + cp bin/storm-trident $(ob_outputDirectory)/storm-trident + workingDirectory: $(TRIDENT_SOURCE_DIR) + displayName: "Stage storm-trident artifact" diff --git a/.pipelines/templates/stages/testing_e2e/storm_e2e.yml b/.pipelines/templates/stages/testing_e2e/storm_e2e.yml index adba26b584..9936022010 100644 --- a/.pipelines/templates/stages/testing_e2e/storm_e2e.yml +++ b/.pipelines/templates/stages/testing_e2e/storm_e2e.yml @@ -3,15 +3,47 @@ parameters: displayName: "Pipeline configuration type" type: string + - name: acrServiceConnectionName + displayName: "Service connection used to push extension/COSI images to ACR" + type: string + default: trident-dev-acr-write-umi-ECF + + # When true, the E2E stages consume pre-built artifacts from the + # DownloadTestingElements stage (shortcut / testing-trident.yml) instead of + # depending on the in-pipeline build stages. + - name: testingRun + type: boolean + default: false + + # Which VM runtimes to run. Both default to true so the full pipeline is + # unchanged; the shortcut pipeline toggles them individually. + - name: runVMHost + type: boolean + default: true + + - name: runVMContainer + type: boolean + default: true + stages: + # In a testingRun (shortcut), build storm-trident once from source and publish + # it so the matrix-definition and test-execution stages consume the + # branch-under-test binary instead of a downloaded, possibly-stale go-tools + # artifact. + - ${{ if eq(parameters.testingRun, true) }}: + - template: build-storm-trident-stage.yml + - stage: DefineTests_E2E displayName: Define all E2E test matrices dependsOn: - - BuildingTools + - ${{ if eq(parameters.testingRun, true) }}: + - BuildStormTridentE2E + - ${{ else }}: + - BuildingTools jobs: - job: DefineTests displayName: Get List of tests to run - timeoutInMinutes: 10 + timeoutInMinutes: 15 pool: type: linux @@ -21,14 +53,25 @@ stages: steps: - template: ../common_tasks/checkout_trident.yml - - task: DownloadPipelineArtifact@2 - displayName: "Download go-tools" - inputs: - buildType: current - artifactName: "go-tools" - patterns: | - storm-trident - targetPath: "$(TRIDENT_SOURCE_DIR)/bin" + # In a testingRun (shortcut) consume the storm-trident built from the + # branch under test (BuildStormTridentE2E); otherwise download it from + # the in-pipeline build's go-tools artifact. + - ${{ if eq(parameters.testingRun, true) }}: + - task: DownloadPipelineArtifact@2 + displayName: "Download storm-trident (built from source)" + inputs: + buildType: current + artifactName: "storm-trident-e2e" + targetPath: "$(TRIDENT_SOURCE_DIR)/bin" + - ${{ else }}: + - task: DownloadPipelineArtifact@2 + displayName: "Download go-tools" + inputs: + buildType: current + artifactName: "go-tools" + patterns: | + storm-trident + targetPath: "$(TRIDENT_SOURCE_DIR)/bin" - bash: | chmod +x ./bin/storm-trident ./bin/storm-trident script e2e-matrix "${{ parameters.stageType }}" @@ -36,17 +79,23 @@ stages: workingDirectory: $(TRIDENT_SOURCE_DIR) displayName: Matrix of Trident configurations for E2E Tests - # TODO: enable once storm tests are ready - - template: test_execution_template.yml - parameters: - hardwareType: "VM" - runtimeType: "HOST" + - ${{ if parameters.runVMHost }}: + - template: test_execution_template.yml + parameters: + hardwareType: "VM" + runtimeType: "HOST" + testingRun: ${{ parameters.testingRun }} + acrServiceConnectionName: ${{ parameters.acrServiceConnectionName }} - # - template: test_execution_template.yml - # parameters: - # hardwareType: "VM" - # runtimeType: "CONTAINER" + - ${{ if parameters.runVMContainer }}: + - template: test_execution_template.yml + parameters: + hardwareType: "VM" + runtimeType: "CONTAINER" + testingRun: ${{ parameters.testingRun }} + acrServiceConnectionName: ${{ parameters.acrServiceConnectionName }} + # TODO: enable once bare-metal host setup is supported in storm # - template: test_execution_template.yml # parameters: # hardwareType: "BM" diff --git a/.pipelines/templates/stages/testing_e2e/test_execution_template.yml b/.pipelines/templates/stages/testing_e2e/test_execution_template.yml index 6f37b937a9..1cb39a028e 100644 --- a/.pipelines/templates/stages/testing_e2e/test_execution_template.yml +++ b/.pipelines/templates/stages/testing_e2e/test_execution_template.yml @@ -19,6 +19,11 @@ parameters: type: string default: null + - name: acrServiceConnectionName + displayName: "Service connection used to push extension/COSI images to ACR" + type: string + default: trident-dev-acr-write-umi-ECF + stages: - stage: E2ETesting_${{ parameters.hardwareType }}_${{ parameters.runtimeType }} displayName: E2E ${{ parameters.hardwareType }} ${{ parameters.runtimeType }} Testing @@ -26,6 +31,7 @@ stages: - DefineTests_E2E - ${{ if eq(parameters.testingRun, true) }}: - DownloadTestingElements + - BuildStormTridentE2E - ${{ else }}: - BuildingTools - ${{ if eq(parameters.runtimeType, 'CONTAINER') }}: @@ -59,6 +65,8 @@ stages: condition: ne(variables['matrixJson'], '{}') variables: + # Provides: ACR_NAME (used to push/pull extension images). + - group: trident_e2e_params - name: ob_outputDirectory value: /tmp/output - name: ob_artifactBaseName @@ -91,7 +99,18 @@ stages: - template: ../common_tasks/avoid-pypi-usage.yml - bash: | - if [ ${{ variables['tridentConfigurationName'] }} == 'split' ]; then + set -eux + # The matrix only exposes the full scenario name (e.g. + # "extensions_vm-host"); derive the configuration name and runtime + # from it for the per-config handling below. + config=$(echo "$(SCENARIO)" | sed -E 's/_(vm|bm)-(host|container)$//') + runtime=$(echo "$(SCENARIO)" | sed -E 's/.*_(vm|bm)-(host|container)$/\2/') + echo "##vso[task.setvariable variable=tridentConfigurationName]$config" + echo "##vso[task.setvariable variable=tridentRuntimeEnv]$runtime" + displayName: "Resolve configuration name from scenario" + + - bash: | + if [ "$(tridentConfigurationName)" == 'split' ]; then splitInstallerIsoName="trident-split-installer" echo "setting variable.installerISOName to $splitInstallerIsoName" echo "##vso[task.setvariable variable=installerISOName]$splitInstallerIsoName" @@ -107,15 +126,185 @@ stages: downloadTridentContainer: ${{ variables.downloadTridentContainer }} tridentTestImageUsrVerity: ${{ variables.usrVerityTestImageName }} tridentSourceDirectory: $(TRIDENT_SOURCE_DIR) + + # In a testingRun (shortcut) consume the storm-trident built from the + # branch under test (BuildStormTridentE2E), overriding the possibly + # stale binary from the go-tools/image download above. + - ${{ if eq(parameters.testingRun, true) }}: + - task: DownloadPipelineArtifact@2 + displayName: "Download storm-trident (built from source)" + inputs: + buildType: current + artifactName: "storm-trident-e2e" + targetPath: "$(TRIDENT_SOURCE_DIR)/bin" + - bash: | + set -eux + chmod +x ./bin/storm-trident + displayName: "Make storm-trident executable" + workingDirectory: $(TRIDENT_SOURCE_DIR) + + # For the extensions config, build sample sysext images and push them + # to ACR so Trident can pull them from an OCI registry during install. + # The config name is only known at runtime (matrix SCENARIO), so these + # steps branch on it at runtime and no-op for other configs. + - bash: | + set -eux + if [ "$(tridentConfigurationName)" != 'extensions' ]; then + echo "Configuration is '$(tridentConfigurationName)'; skipping extension image build." + exit 0 + fi + + ./bin/storm-trident script build-extension-images --build-sysexts --num-clones 2 + + # Install ORAS for the ACR push. + VERSION="1.2.2" + curl -LO "https://github.com/oras-project/oras/releases/download/v${VERSION}/oras_${VERSION}_linux_amd64.tar.gz" + mkdir -p oras-install/ + tar -zxf oras_${VERSION}_*.tar.gz -C oras-install/ + sudo mv oras-install/oras /usr/local/bin/ + rm -rf oras_${VERSION}_*.tar.gz oras-install/ + displayName: "Build extension images (extensions only)" + workingDirectory: $(TRIDENT_SOURCE_DIR) + retryCountOnTaskFailure: 3 + + - task: AzureCLI@2 + displayName: "Push extension images to ACR (extensions only)" + inputs: + azureSubscription: ${{ parameters.acrServiceConnectionName }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -eux + if [ "$(tridentConfigurationName)" != 'extensions' ]; then + echo "Configuration is '$(tridentConfigurationName)'; skipping ACR push." + exit 0 + fi + + cd $(TRIDENT_SOURCE_DIR) + # Sets the SYSEXT_REPO and TAG_BASE pipeline variables. + # Namespaced away from the legacy suite's "sysext-" + # repo. Both suites build their own sysext (mksquashfs embeds + # timestamps, so the bytes differ) and derive the same tag from + # the shared build id, so sharing a repo means the last pusher + # wins and the other job fails its sha384 check. It would also + # let this job's ACR cleanup delete images legacy is still + # using. The cleanup step below follows via SYSEXT_REPO. + ./bin/storm-trident script acr-push \ + --config extensions \ + --deployment-environment virtualMachine \ + --acr-name $(ACR_NAME) \ + --repo-name sysext-storm-$(tridentRuntimeEnv) \ + --build-id $(Build.BuildId) \ + --file-paths $(TRIDENT_SOURCE_DIR)/test-sysext-1.raw \ + --file-paths $(TRIDENT_SOURCE_DIR)/test-sysext-2.raw \ + --tag-var-name TAG_BASE \ + --repo-var-name SYSEXT_REPO + retryCountOnTaskFailure: 3 + - bash: | set -eux mkdir -p $(ob_outputDirectory) + # UKI/usr-verity images boot their kernel directly through + # firmware Secure Boot, so the image signing certificate must be + # enrolled into the VM's EFI variables via --signing-cert. The + # cert ships alongside the usrverity test image. Enrolling it is + # harmless for grub-based images, so pass it whenever present. + SIGNING_CERT_ARG="" + CA_CERT_PATH="$(System.ArtifactsDirectory)/usrverity-testimage/ca_cert.pem" + if [ -f "$CA_CERT_PATH" ]; then + SIGNING_CERT_ARG="--signing-cert $CA_CERT_PATH" + fi + + # For the extensions config, point the Host Configuration at the + # sysext image pushed to ACR above. SYSEXT_REPO and TAG_BASE were + # set by the acr-push step; ACR_NAME comes from trident_e2e_params. + EXTENSION_ARGS="" + if [ "$(tridentConfigurationName)" == 'extensions' ]; then + sysext_tag="$(TAG_BASE).1" + sysext_oci_url="oci://$(ACR_NAME).azurecr.io/$(SYSEXT_REPO):${sysext_tag}" + sysext_sha384=$(sha384sum "$(TRIDENT_SOURCE_DIR)/test-sysext-1.raw" | awk '{print $1}') + EXTENSION_ARGS="--sysext-oci-url ${sysext_oci_url} --sysext-sha384 ${sysext_sha384}" + fi + ./bin/storm-trident run "$(SCENARIO)" \ -o $(ob_outputDirectory) \ + -j "$(ob_outputDirectory)/storm-e2e.junit.xml" \ -- \ --pipeline-run \ - --iso "./artifacts/iso/$(installerISOName).iso" + --iso "./artifacts/iso/$(installerISOName).iso" \ + $SIGNING_CERT_ARG \ + $EXTENSION_ARGS displayName: "🧪 Run E2E Test" workingDirectory: $(TRIDENT_SOURCE_DIR) + + # Sanitize and disambiguate the JUnit XML before publishing: + # 1. Strip XML-1.0-illegal C0 control bytes (NUL and friends) that + # storm copies verbatim from the serial console into the + # system-out CDATA. PublishTestResults@2 rejects the file + # otherwise ("hexadecimal value 0x00 is an invalid character"). + # Keep tab/LF/CR (the only C0 bytes XML 1.0 allows). + # 2. Qualify each classname with the scenario (which + # includes the runtime, e.g. base_vm-host). storm emits an empty + # classname and identical case names across runtimes, so ADO's + # Tests tab (keyed on classname + name) would otherwise merge the + # host and container variants of the same case. + # A byte/line filter is used rather than an XML parser precisely + # because the raw document is not well-formed until step 1 runs. Runs + # even on test failure so the failing XML is still published. + - bash: | + set -eu + JUNIT="$(ob_outputDirectory)/storm-e2e.junit.xml" + if [ ! -f "$JUNIT" ]; then + echo "No JUnit file at $JUNIT; skipping sanitization." + exit 0 + fi + SCENARIO="$(SCENARIO)" + tr -d '\000-\010\013\014\016-\037' < "$JUNIT" > "${JUNIT}.clean" + mv "${JUNIT}.clean" "$JUNIT" + sed -i "/ "${JUNIT}.clean" + mv "${JUNIT}.clean" "$JUNIT" + sed -i "/= pre). v1 and v2 must be provided as distinct real +// images; v3+ are hard-links (see prepareTestImages). +const ( + maxCiRingImageVersion = 3 + maxSplitRingImageVersion = 4 +) + +// prepareTestImages ensures the versioned test images the scenario's A/B updates +// will request exist in the test image directory. It folds the versioning half +// of the legacy `prepare-images` helper into the scenario: v1 (.cosi) and +// v2 (_v2.cosi) are real, distinct images that must already be present, +// and this creates the higher versions as hard-links following the same scheme +// prepare-images used — odd versions alias v1, even versions alias v2 (so a +// version's filesystem UUID differs from the currently-active volume). It is a +// no-op for configs without A/B updates and for OCI-hosted images (which the +// pipeline stages in ACR). +func (s *TridentE2EScenario) prepareTestImages(tc storm.TestCase) error { + if !s.originalConfig.HasABUpdate() { + return nil + } + + url, ok := s.config.S("image", "url").Data().(string) + if !ok { + return fmt.Errorf("failed to read image.url from Host Config") + } + if strings.HasPrefix(url, "oci://") { + tc.Skip("Image is OCI-hosted; versioned images are staged in ACR by the pipeline") + } + + base := path.Base(url) + ext := strings.TrimPrefix(filepath.Ext(base), ".") + if ext == "" { + return fmt.Errorf("failed to determine extension of image %q", base) + } + imageType := strings.TrimSuffix(base, "."+ext) + + maxVersion := maxCiRingImageVersion + if !s.splitTestsSkippedForCurrentRing() { + maxVersion = maxSplitRingImageVersion + } + + for version := 3; version <= maxVersion; version++ { + if err := s.ensureVersionedImage(imageType, ext, version); err != nil { + return err + } + } + return nil +} + +// ensureVersionedImage hard-links _v. to the appropriate +// base image if it does not already exist: odd versions alias v1 +// (.), even versions alias v2 (_v2.). +func (s *TridentE2EScenario) ensureVersionedImage(imageType, ext string, version int) error { + dir := s.args.TestImageDir + targetName := fmt.Sprintf("%s_v%d.%s", imageType, version, ext) + targetPath := filepath.Join(dir, targetName) + + if _, err := os.Stat(targetPath); err == nil { + logrus.Debugf("Versioned image %q already exists; leaving as-is", targetName) + return nil + } + + var sourceName string + if version%2 == 0 { + sourceName = fmt.Sprintf("%s_v2.%s", imageType, ext) + } else { + sourceName = fmt.Sprintf("%s.%s", imageType, ext) + } + sourcePath := filepath.Join(dir, sourceName) + + if _, err := os.Stat(sourcePath); err != nil { + return fmt.Errorf("cannot create versioned image %q: source image %q not found: %w", + targetName, sourceName, err) + } + + logrus.Infof("Linking test image %q -> %q (v%d)", targetName, sourceName, version) + if err := os.Link(sourcePath, targetPath); err != nil { + return fmt.Errorf("failed to link %q to %q: %w", targetName, sourceName, err) + } + return nil +} diff --git a/tools/storm/e2e/scenario/prepare_images_test.go b/tools/storm/e2e/scenario/prepare_images_test.go new file mode 100644 index 0000000000..eea11f5fc5 --- /dev/null +++ b/tools/storm/e2e/scenario/prepare_images_test.go @@ -0,0 +1,95 @@ +package scenario + +import ( + "os" + "path/filepath" + "testing" +) + +// newImagePrepScenario builds a scenario whose TestImageDir is a temp dir +// pre-seeded with distinct v1 (regular.cosi) and v2 (regular_v2.cosi) images. +func newImagePrepScenario(t *testing.T, dir string) *TridentE2EScenario { + t.Helper() + for name, content := range map[string]string{ + "regular.cosi": "v1-content", + "regular_v2.cosi": "v2-content", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0644); err != nil { + t.Fatalf("seed image %s: %v", name, err) + } + } + s := &TridentE2EScenario{} + s.args.TestImageDir = dir + return s +} + +func TestEnsureVersionedImage_HardlinkScheme(t *testing.T) { + dir := t.TempDir() + s := newImagePrepScenario(t, dir) + + if err := s.ensureVersionedImage("regular", "cosi", 3); err != nil { + t.Fatalf("ensure v3: %v", err) + } + if err := s.ensureVersionedImage("regular", "cosi", 4); err != nil { + t.Fatalf("ensure v4: %v", err) + } + + // v3 (odd) must alias v1 (regular.cosi); v4 (even) must alias v2. + assertSameContent(t, dir, "regular_v3.cosi", "v1-content") + assertSameContent(t, dir, "regular_v4.cosi", "v2-content") + + // They must be hard-links (same inode) to their source, not copies. + assertSameInode(t, filepath.Join(dir, "regular_v3.cosi"), filepath.Join(dir, "regular.cosi")) + assertSameInode(t, filepath.Join(dir, "regular_v4.cosi"), filepath.Join(dir, "regular_v2.cosi")) +} + +func TestEnsureVersionedImage_ExistingLeftAsIs(t *testing.T) { + dir := t.TempDir() + s := newImagePrepScenario(t, dir) + + // Pre-existing v3 with distinct content must not be overwritten. + existing := filepath.Join(dir, "regular_v3.cosi") + if err := os.WriteFile(existing, []byte("real-v3"), 0644); err != nil { + t.Fatal(err) + } + if err := s.ensureVersionedImage("regular", "cosi", 3); err != nil { + t.Fatalf("ensure v3: %v", err) + } + assertSameContent(t, dir, "regular_v3.cosi", "real-v3") +} + +func TestEnsureVersionedImage_MissingSourceFails(t *testing.T) { + dir := t.TempDir() + s := &TridentE2EScenario{} + s.args.TestImageDir = dir + // No base images seeded. + if err := s.ensureVersionedImage("regular", "cosi", 3); err == nil { + t.Error("expected an error when the source base image is missing") + } +} + +func assertSameContent(t *testing.T, dir, name, want string) { + t.Helper() + got, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + if string(got) != want { + t.Errorf("%s content = %q, want %q", name, got, want) + } +} + +func assertSameInode(t *testing.T, a, b string) { + t.Helper() + ai, err := os.Stat(a) + if err != nil { + t.Fatal(err) + } + bi, err := os.Stat(b) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(ai, bi) { + t.Errorf("%s and %s are not the same file (expected a hard link)", a, b) + } +} diff --git a/tools/storm/e2e/scenario/rebuild_raid.go b/tools/storm/e2e/scenario/rebuild_raid.go new file mode 100644 index 0000000000..3ed7d89c24 --- /dev/null +++ b/tools/storm/e2e/scenario/rebuild_raid.go @@ -0,0 +1,102 @@ +package scenario + +import ( + "context" + "fmt" + "time" + + "github.com/microsoft/storm" + "github.com/sirupsen/logrus" + + "tridenttools/storm/utils/trident" +) + +// raidMemberDiskIndex is the 0-based index of the VM data disk failed to +// simulate a degraded RAID array. Disk 0 is the OS disk; the RAID member disks +// start at index 1. +const raidMemberDiskIndex uint = 1 + +// addRebuildRaidTests registers the rebuild-raid test cases. They simulate a +// failed RAID member disk, boot the (now degraded) host, and run `trident +// rebuild-raid` to rebuild the array onto a fresh disk. This ports the legacy +// VM-only `storm-trident helper rebuild-raid` step and self-selects with the +// HasRebuildableRaid() && IsVM() gate at the call site. +func (s *TridentE2EScenario) addRebuildRaidTests(r storm.TestRegistrar) { + r.RegisterTestCase("rebuild-raid-fail-disk", s.rebuildRaidFailDisk) + r.RegisterTestCase("rebuild-raid", s.rebuildRaid) + r.RegisterTestCase("validate-rebuild-raid", s.validateHostState) +} + +// rebuildRaidFailDisk powers off the VM, replaces one RAID member disk with a +// blank one, boots the host back up (now with a degraded array), and waits for +// it to reach the login prompt. +func (s *TridentE2EScenario) rebuildRaidFailDisk(tc storm.TestCase) error { + vmInfo := s.testHost.VmInfo() + if vmInfo == nil { + return fmt.Errorf("rebuild-raid requires a VM test host") + } + + // Serve phonehome + capture the serial log across the reboot that follows + // the disk replacement. + monitorCtx, cancel := context.WithCancel(tc.Context()) + defer cancel() + monWaitChan, monErr := s.spawnVMSerialMonitor(monitorCtx, tc.ArtifactBroker().StreamArtifactData(tc.Name()+"/serial.log")) + if monErr != nil { + return fmt.Errorf("failed to start VM serial monitor: %w", monErr) + } + defer func() { + select { + case <-time.After(time.Minute): + logrus.Infof("Waited 1 minute for serial monitor to reach login prompt, cancelling monitor.") + cancel() + case <-monWaitChan: + } + }() + + // Drop the current SSH client: the VM is about to be powered off. + if s.sshClient != nil { + s.sshClient.Close() + s.sshClient = nil + } + + logrus.Infof("Failing RAID member disk %d and rebooting the host...", raidMemberDiskIndex) + if err := vmInfo.FailAndReplaceDataDisk(raidMemberDiskIndex); err != nil { + return fmt.Errorf("failed to replace RAID member disk: %w", err) + } + + // Reconnect once the degraded host is back up, and confirm Trident is + // healthy (the previous servicing commit is unchanged by a disk failure). + connCtx, cancel := context.WithTimeout(tc.Context(), time.Minute*5) + defer cancel() + if err := s.populateSshClient(connCtx); err != nil { + return fmt.Errorf("failed to reconnect after RAID member disk failure: %w", err) + } + logrus.Info("Reacquired SSH connection to degraded host.") + + if err := trident.CheckTridentService(s.sshClient, s.runtime, time.Minute*2, true); err != nil { + tc.FailFromError(err) + } + + return nil +} + +// rebuildRaid runs `trident rebuild-raid` on the degraded host to rebuild the +// RAID array onto the replacement disk. +func (s *TridentE2EScenario) rebuildRaid(tc storm.TestCase) error { + if err := s.populateSshClient(tc.Context()); err != nil { + return fmt.Errorf("failed to connect before rebuild-raid: %w", err) + } + + logrus.Info("Running Trident rebuild-raid...") + out, err := trident.InvokeTrident(s.runtime, s.sshClient, nil, "rebuild-raid -v trace") + if err != nil { + return fmt.Errorf("failed to invoke Trident rebuild-raid: %w", err) + } + if err := out.Check(); err != nil { + tc.FailFromError(fmt.Errorf("Trident rebuild-raid failed: %s", out.Report())) + return nil + } + logrus.Info("Trident rebuild-raid succeeded.") + + return nil +} diff --git a/tools/storm/e2e/scenario/register_test.go b/tools/storm/e2e/scenario/register_test.go new file mode 100644 index 0000000000..cf497af507 --- /dev/null +++ b/tools/storm/e2e/scenario/register_test.go @@ -0,0 +1,216 @@ +package scenario + +import ( + "slices" + "testing" + + "github.com/microsoft/storm/pkg/storm/core" + + "tridenttools/pkg/hostconfig" + "tridenttools/storm/e2e/testrings" + "tridenttools/storm/utils/trident" +) + +// fakeRegistrar records the order of registered test-case names. +type fakeRegistrar struct { + names []string +} + +func (f *fakeRegistrar) RegisterTestCase(name string, _ core.TestCaseFunction) { + f.names = append(f.names, name) +} + +func newScenarioForTest(t *testing.T, configYaml string) *TridentE2EScenario { + t.Helper() + hc, err := hostconfig.NewHostConfigFromYaml([]byte(configYaml)) + if err != nil { + t.Fatalf("failed to parse config: %v", err) + } + s, err := NewTridentE2EScenario( + "test", []string{"e2e"}, hc, TridentE2EHostConfigParams{}, + HardwareTypeVM, trident.RuntimeTypeHost, testrings.TestRingSet{testrings.TestRingPrE2e}, + ) + if err != nil { + t.Fatalf("failed to create scenario: %v", err) + } + return s +} + +const abConfig = ` +storage: + abUpdate: + volumePairs: + - id: root + volumeAId: root-a + volumeBId: root-b +` + +const noAbConfig = ` +storage: + disks: + - id: os + partitions: + - id: root + size: 8G +` + +const raidConfig = ` +storage: + raid: + software: + - id: root + name: root + level: raid1 + devices: [root-a, root-b] +` + +const usrVerityRaidConfig = ` +storage: + raid: + software: + - id: usr + name: usr + level: raid1 + devices: [usr-a, usr-b] + verity: + - id: usr + name: usr +` + +func TestRegisterTestCases_ABUpdate_RegistersValidation(t *testing.T) { + s := newScenarioForTest(t, abConfig) + var r fakeRegistrar + if err := s.RegisterTestCases(&r); err != nil { + t.Fatalf("RegisterTestCases error: %v", err) + } + + // validate-install must come right after check-trident-ssh. + assertOrder(t, r.names, "check-trident-ssh", "validate-install") + // Image prep runs after prepare-hc and before setup-test-host. + mustContain(t, r.names, "prepare-test-images") + assertOrder(t, r.names, "prepare-hc", "prepare-test-images") + assertOrder(t, r.names, "prepare-test-images", "setup-test-host") + // Host diagnostics validation runs right after install validation. + mustContain(t, r.names, "validate-host-diagnostics") + assertOrder(t, r.names, "validate-install", "validate-host-diagnostics") + // Post-A/B-update validations must be registered. + mustContain(t, r.names, "validate-ab-update-1") + mustContain(t, r.names, "validate-ab-update-split") + // validate-ab-update-1 must come after the ab-update-1 update case. + assertOrder(t, r.names, "ab-update-1-ab-update", "validate-ab-update-1") + + // Auto-rollback cases must be registered in order, after the first A/B + // update's validation and before the split A/B update. + for _, n := range []string{ + "auto-rollback-sync-hc", "auto-rollback-update-hc", "auto-rollback-inject-hc", + "auto-rollback-upload-hc", "auto-rollback-update", "validate-auto-rollback", + } { + mustContain(t, r.names, n) + } + assertOrder(t, r.names, "validate-ab-update-1", "auto-rollback-sync-hc") + assertOrder(t, r.names, "auto-rollback-update-hc", "auto-rollback-inject-hc") + assertOrder(t, r.names, "auto-rollback-inject-hc", "auto-rollback-update") + assertOrder(t, r.names, "auto-rollback-update", "validate-auto-rollback") + assertOrder(t, r.names, "validate-auto-rollback", "ab-update-split-sync-hc") + + // Second A/B update (return into OS A) must be registered in order, after + // the auto-rollback and before the split A/B update. + for _, n := range []string{ + "ab-update-2-sync-hc", "ab-update-2-clear-hc", "ab-update-2-update-hc", + "ab-update-2-upload-new-hc", "ab-update-2-ab-update", "validate-ab-update-2", + } { + mustContain(t, r.names, n) + } + assertOrder(t, r.names, "validate-auto-rollback", "ab-update-2-sync-hc") + assertOrder(t, r.names, "ab-update-2-ab-update", "validate-ab-update-2") + assertOrder(t, r.names, "validate-ab-update-2", "ab-update-split-sync-hc") + + // Manual rollback (VM A/B configs) must be registered after the split + // validation, in order. + mustContain(t, r.names, "manual-rollback") + mustContain(t, r.names, "validate-manual-rollback") + assertOrder(t, r.names, "validate-ab-update-split", "manual-rollback") + assertOrder(t, r.names, "manual-rollback", "validate-manual-rollback") + + assertUnique(t, r.names) +} + +func TestRegisterTestCases_NoABUpdate_OnlyInstallValidation(t *testing.T) { + s := newScenarioForTest(t, noAbConfig) + var r fakeRegistrar + if err := s.RegisterTestCases(&r); err != nil { + t.Fatalf("RegisterTestCases error: %v", err) + } + + mustContain(t, r.names, "validate-install") + if slices.Contains(r.names, "validate-ab-update-1") { + t.Error("validate-ab-update-1 should not be registered without abUpdate") + } + if slices.Contains(r.names, "rebuild-raid") { + t.Error("rebuild-raid should not be registered without RAID") + } + assertUnique(t, r.names) +} + +func TestRegisterTestCases_Raid_RegistersRebuildRaid(t *testing.T) { + s := newScenarioForTest(t, raidConfig) + var r fakeRegistrar + if err := s.RegisterTestCases(&r); err != nil { + t.Fatalf("RegisterTestCases error: %v", err) + } + + for _, n := range []string{"rebuild-raid-fail-disk", "rebuild-raid", "validate-rebuild-raid"} { + mustContain(t, r.names, n) + } + assertOrder(t, r.names, "rebuild-raid-fail-disk", "rebuild-raid") + assertOrder(t, r.names, "rebuild-raid", "validate-rebuild-raid") + // A RAID config without abUpdate must not register A/B cases. + if slices.Contains(r.names, "validate-ab-update-1") { + t.Error("non-A/B RAID config should not register A/B update cases") + } + assertUnique(t, r.names) +} + +func TestRegisterTestCases_UsrVerityRaid_NoRebuildRaid(t *testing.T) { + s := newScenarioForTest(t, usrVerityRaidConfig) + var r fakeRegistrar + if err := s.RegisterTestCases(&r); err != nil { + t.Fatalf("RegisterTestCases error: %v", err) + } + if slices.Contains(r.names, "rebuild-raid") { + t.Error("usr-verity RAID config must not register rebuild-raid (verity rebuild unsupported)") + } +} + +func assertOrder(t *testing.T, names []string, before, after string) { + t.Helper() + bi := slices.Index(names, before) + ai := slices.Index(names, after) + if bi < 0 { + t.Fatalf("%q not registered (have %v)", before, names) + } + if ai < 0 { + t.Fatalf("%q not registered (have %v)", after, names) + } + if bi >= ai { + t.Errorf("%q (idx %d) should come before %q (idx %d)", before, bi, after, ai) + } +} + +func mustContain(t *testing.T, names []string, want string) { + t.Helper() + if !slices.Contains(names, want) { + t.Errorf("expected %q to be registered, have %v", want, names) + } +} + +func assertUnique(t *testing.T, names []string) { + t.Helper() + seen := map[string]struct{}{} + for _, n := range names { + if _, dup := seen[n]; dup { + t.Errorf("duplicate test case name %q", n) + } + seen[n] = struct{}{} + } +} diff --git a/tools/storm/e2e/scenario/setup.go b/tools/storm/e2e/scenario/setup.go index f8ef70a977..51e1e46fd1 100644 --- a/tools/storm/e2e/scenario/setup.go +++ b/tools/storm/e2e/scenario/setup.go @@ -17,6 +17,10 @@ import ( "libvirt.org/go/libvirtxml" ) +// defaultDataDiskSizeGB is the size (GB) virtdeploy creates the VM data disks +// at (see setupTestHostVm Disks); used when replacing a failed RAID member disk. +const defaultDataDiskSizeGB uint = 32 + type testHostInfo interface { // Retrieve the IP address of the test host. IPAddress() net.IP @@ -50,6 +54,12 @@ type testVmHostInfo interface { // Returns the libvirt DOMAIN object for the VM. LvDomain() libvirt.Domain + + // FailAndReplaceDataDisk simulates a failed RAID member disk: it forcibly + // powers off the VM, deletes the given data disk volume, recreates it blank + // at the same path/size, and powers the VM back on. diskIndex is the 0-based + // disk index (disk 0 is the OS disk; RAID member disks start at 1). + FailAndReplaceDataDisk(diskIndex uint) error } func (s *TridentE2EScenario) setupTestHost(tc storm.TestCase) error { @@ -182,3 +192,79 @@ func (t *testHostVirtDeploy) SerialLogPath() (string, error) { return "", fmt.Errorf("failed to find a serial device with a log backend in VM definition %s", t.vm.Name) } + +// storagePoolName returns the libvirt storage pool name for this VM's +// namespace, matching virtdeploy's naming convention (-pool). +func (t *testHostVirtDeploy) storagePoolName() string { + return t.namespace + "-pool" +} + +// FailAndReplaceDataDisk simulates a failed RAID member disk: it forcibly +// powers off the VM, deletes the data disk volume at diskIndex, recreates it +// blank at the same path and size, and powers the VM back on. This drives the +// rebuild-raid test the same way the legacy helper did (via virsh + qemu-img), +// but through the libvirt API. +func (t *testHostVirtDeploy) FailAndReplaceDataDisk(diskIndex uint) error { + // virtdeploy names disk volumes "-volume-.qcow2". + volName := fmt.Sprintf("%s-volume-%d.qcow2", t.vm.Name, diskIndex) + + pool, err := t.lv.StoragePoolLookupByName(t.storagePoolName()) + if err != nil { + return fmt.Errorf("failed to look up storage pool %q: %w", t.storagePoolName(), err) + } + + vol, err := t.lv.StorageVolLookupByName(pool, volName) + if err != nil { + return fmt.Errorf("failed to look up disk volume %q: %w", volName, err) + } + volPath, err := t.lv.StorageVolGetPath(vol) + if err != nil { + return fmt.Errorf("failed to get path of disk volume %q: %w", volName, err) + } + + // Determine the disk's declared size (GB). virtdeploy creates the VM's data + // disks at a fixed size, so use that constant for the replacement. + diskSizeGB := uint(defaultDataDiskSizeGB) + + // Force the VM off (a failed disk is an abrupt event, not a clean shutdown). + active, err := t.lv.DomainIsActive(t.vm.Domain) + if err != nil { + return fmt.Errorf("failed to check whether VM %q is active: %w", t.vm.Name, err) + } + if active != 0 { + logrus.Infof("Powering off VM %q to replace data disk %q", t.vm.Name, volName) + if err := t.lv.DomainDestroy(t.vm.Domain); err != nil { + return fmt.Errorf("failed to power off VM %q: %w", t.vm.Name, err) + } + } + + logrus.Infof("Deleting data disk volume %q", volName) + if err := t.lv.StorageVolDelete(vol, 0); err != nil { + return fmt.Errorf("failed to delete disk volume %q: %w", volName, err) + } + + newVolXml := libvirtxml.StorageVolume{ + Name: volName, + Capacity: &libvirtxml.StorageVolumeSize{Unit: "G", Value: uint64(diskSizeGB)}, + Target: &libvirtxml.StorageVolumeTarget{ + Path: volPath, + Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"}, + Permissions: &libvirtxml.StorageVolumeTargetPermissions{Mode: "0644"}, + }, + } + xml, err := newVolXml.Marshal() + if err != nil { + return fmt.Errorf("failed to marshal replacement volume XML: %w", err) + } + logrus.Infof("Recreating blank data disk volume %q (%d GB)", volName, diskSizeGB) + if _, err := t.lv.StorageVolCreateXML(pool, xml, 0); err != nil { + return fmt.Errorf("failed to recreate disk volume %q: %w", volName, err) + } + + logrus.Infof("Powering VM %q back on", t.vm.Name) + if err := t.lv.DomainCreate(t.vm.Domain); err != nil { + return fmt.Errorf("failed to power on VM %q: %w", t.vm.Name, err) + } + + return nil +} diff --git a/tools/storm/e2e/scenario/trident.go b/tools/storm/e2e/scenario/trident.go index 875f718423..0b4e591066 100644 --- a/tools/storm/e2e/scenario/trident.go +++ b/tools/storm/e2e/scenario/trident.go @@ -7,6 +7,7 @@ import ( "tridenttools/pkg/hostconfig" "tridenttools/storm/e2e/testrings" "tridenttools/storm/utils/sshutils" + "tridenttools/storm/utils/sysinspect" "tridenttools/storm/utils/trident" "github.com/microsoft/storm" @@ -63,6 +64,11 @@ type TridentE2EScenario struct { DumpSshKeyFile string `name:"dump-ssh-key" help:"If set, the SSH private key used for VM access will be dumped to the specified file."` VmWaitForLoginTimeout int `name:"vm-wait-for-login-timeout" help:"Time in seconds to wait for the VM to reach login prompt." default:"600"` TestRing testrings.TestRing `name:"test-ring" help:"The test ring in which this scenario is being executed. Defaults to lowest ring for this scenario." env:"TEST_RING"` + SysextOciUrl string `name:"sysext-oci-url" help:"OCI URL of a system extension image to inject into the Host Configuration (os.sysexts)."` + SysextSha384 string `name:"sysext-sha384" help:"SHA384 of the system extension image referenced by --sysext-oci-url."` + ConfextOciUrl string `name:"confext-oci-url" help:"OCI URL of a configuration extension image to inject into the Host Configuration (os.confexts)."` + ConfextSha384 string `name:"confext-sha384" help:"SHA384 of the configuration extension image referenced by --confext-oci-url."` + OciImageUrl string `name:"oci-image-url" help:"If set, overwrites the Host Configuration image.url with this OCI URL (ACR-hosted COSI)."` } // Runtime variables @@ -79,6 +85,11 @@ type TridentE2EScenario struct { // Version of the image, used for AB update tests version uint + // Expected active A/B volume after the most recent servicing operation. + // Initialized to volume-a on clean install and flipped after each + // successful A/B update. Read by validation cases. + expectedActiveVolume trident.AbVolumeSelection + // Working copy of the host configuration, modified during test execution to // reflect changes such as AB updates. config hostconfig.HostConfig @@ -115,6 +126,9 @@ func (s *TridentE2EScenario) Args() any { } func (s *TridentE2EScenario) Setup(storm.SetupCleanupContext) error { + // A clean install boots the A volume; A/B updates flip this. + s.expectedActiveVolume = trident.AbVolumeA + if s.args.TestRing == testrings.TestRingEmpty { // Default to lowest ring lowestRing, err := s.testRings.Lowest() @@ -170,13 +184,46 @@ func (s *TridentE2EScenario) RegisterTestCases(r storm.TestRegistrar) error { } r.RegisterTestCase("prepare-hc", s.prepareHostConfig) + // Ensure the versioned test images the A/B updates will request exist + // (folds the versioning half of the legacy prepare-images helper). No-op + // for non-A/B and OCI-hosted images. + r.RegisterTestCase("prepare-test-images", s.prepareTestImages) r.RegisterTestCase("setup-test-host", s.setupTestHost) r.RegisterTestCase("install-os", s.installOs) r.RegisterTestCase("check-trident-ssh", s.checkTridentViaSshAfterInstall) + r.RegisterTestCase("validate-install", s.validateHostState) + // Host-only SELinux + tracing diagnostics, scoped to the clean install + // (mirrors legacy check-selinux/check-tracing). Self-skips on container. + r.RegisterTestCase("validate-host-diagnostics", s.validateHostDiagnostics) if s.originalConfig.HasABUpdate() { s.addAbUpdateTests(r, "ab-update-1") + r.RegisterTestCase("validate-ab-update-1", s.validateHostState) + // Auto-rollback: force a failing A/B update and confirm the host rolls + // back to the current volume. Legacy runs this for every A/B config. + s.addAutoRollbackTests(r) + // Second A/B update: the return update into OS A after the rollback. + s.addSecondAbUpdateTests(r) s.addSplitABUpdateTests(r, "ab-update-split") + // Validation of the split A/B update must skip on the same rings its + // update cases do, otherwise it would run (and likely fail) against a + // host that never underwent the split update. + r.RegisterTestCase("validate-ab-update-split", func(tc storm.TestCase) error { + s.skipIfSplitTestsDisabled(tc) + return s.validateHostState(tc) + }) + + // Manual rollback: after the A/B updates, explicitly roll back to the + // previously-committed volume and validate. Legacy runs this VM-only. + if s.hardware.IsVM() { + s.addManualRollbackTests(r) + } + } + + // Rebuild-raid self-selects independently of A/B: any config with software + // RAID that is not usr-verity. Legacy runs it VM-only (BM not yet ported). + if s.hardware.IsVM() && s.originalConfig.HasRebuildableRaid() { + s.addRebuildRaidTests(r) } return nil } @@ -222,5 +269,38 @@ func (s *TridentE2EScenario) populateSshClient(ctx context.Context) error { s.sshClient = client + // For the container runtime, prepare the freshly-connected host the same way + // the pytest suite did at connection time: disable SELinux enforcement and + // load the Trident container image into Docker. This runs on every new + // connection (initial and post-reboot reconnects) because setenforce does + // not persist across reboots. + if s.runtime == trident.RuntimeTypeContainer { + if err := s.prepareContainerRuntime(); err != nil { + return fmt.Errorf("failed to prepare container runtime: %w", err) + } + } + + return nil +} + +// prepareContainerRuntime readies a container-runtime host for Trident commands: +// it disables SELinux enforcement (the container runtime requires permissive +// mode) and loads the Trident container image into Docker. Mirrors the container +// handling in the pytest suite's connection fixture. +func (s *TridentE2EScenario) prepareContainerRuntime() error { + mode, err := sysinspect.Getenforce(s.sshClient) + if err != nil { + return fmt.Errorf("failed to query SELinux mode: %w", err) + } + if mode != "Disabled" { + if err := sysinspect.Setenforce(s.sshClient, false); err != nil { + return fmt.Errorf("failed to set SELinux permissive: %w", err) + } + } + + if err := trident.LoadTridentContainer(s.sshClient); err != nil { + return fmt.Errorf("failed to load Trident container image: %w", err) + } + return nil } diff --git a/tools/storm/e2e/scenario/validate.go b/tools/storm/e2e/scenario/validate.go new file mode 100644 index 0000000000..71fcce8dde --- /dev/null +++ b/tools/storm/e2e/scenario/validate.go @@ -0,0 +1,159 @@ +package scenario + +import ( + "context" + "time" + + "github.com/microsoft/storm" + "github.com/sirupsen/logrus" + + "tridenttools/storm/e2e/validate" + "tridenttools/storm/utils/trident" +) + +// hasRollbackIntent reports whether this scenario is expected to trigger a +// health-check rollback, signalled by a top-level `health` section in the Host +// Configuration. Such scenarios expect the install to fail and roll back rather +// than commit successfully. +func (s *TridentE2EScenario) hasRollbackIntent() bool { + return s.config.Exists("health") +} + +// validateHostState is the storm test case that validates the installed host's +// state against the Host Configuration and Host Status. It is registered after +// clean install and after each A/B update. It ports the pytest E2E validation +// suite (tests/e2e_tests/*.py). +// +// All applicable sub-checks run and accumulate their failures (interim +// soft-assert approach) so a single case reports every mismatch it finds; the +// case fails once at the end if any sub-check failed. +func (s *TridentE2EScenario) validateHostState(tc storm.TestCase) error { + connCtx, cancel := context.WithTimeout(tc.Context(), time.Minute) + defer cancel() + if err := s.populateSshClient(connCtx); err != nil { + // The host is expected to be up by now, so a connection failure is an + // infrastructure error rather than a product failure. + return err + } + + hs, err := trident.GetHostStatus(s.runtime, s.sshClient) + if err != nil { + return err + } + + var sa validate.SoftAsserter + + if validate.HasRollbackIntent(hs) { + // Health-check rollback scenarios (e.g. health-checks-install) replace + // base validation with rollback validation and expect the host to have + // rolled back rather than reached the provisioned state. + validate.ValidateRollback(&sa, s.sshClient, hs, + trident.ServicingStateNotProvisioned, s.expectedActiveVolume) + } else { + // `base` validation always applies. + validate.ValidateBase(&sa, s.sshClient, hs, trident.ServicingStateProvisioned, s.expectedActiveVolume) + + // `extensions` validation self-selects when the Host Config declares + // sysexts/confexts. + if validate.HasExtensions(hs) { + validate.ValidateExtensions(&sa, s.sshClient, hs) + } + + // `verity` validation self-selects when the Host Config declares a + // verity device. + if validate.HasVerity(hs) { + validate.ValidateVerity(&sa, s.sshClient, hs, s.expectedActiveVolume) + } + + // `encryption` validation self-selects when the Host Config declares + // encryption volumes. + if validate.HasEncryption(hs) { + validate.ValidateEncryption(&sa, s.sshClient, hs, s.configParams.IsUki, s.expectedActiveVolume) + } + } + + if err := sa.Err(); err != nil { + tc.FailFromError(err) + } + + // Always log the full ordered PASS/FAIL breakdown so a single validate case + // surfaces exactly which sub-checks ran, even when it passes. + logrus.Infof("Host state validation summary:\n%s", sa.Summary()) + + return nil +} + +// defaultCleanInstallMetricsFile is the trace-stream file netlisten writes for +// the clean install when no explicit tracestream file is configured. Kept in +// sync with installOs. +const defaultCleanInstallMetricsFile = "trident-clean-install-metrics.jsonl" + +// cleanInstallTraceFile returns the local path of the trace-stream file netlisten +// captured for the clean install. +func (s *TridentE2EScenario) cleanInstallTraceFile() string { + if s.args.TracestreamFile != "" { + return s.args.TracestreamFile + } + return defaultCleanInstallMetricsFile +} + +// validateHostDiagnostics ports the host-only check-selinux and check-tracing +// steps that legacy ran after clean install: it confirms no SELinux denials +// were logged (surfaced via audit2allow), that Trident's commit tracing metric +// reached journald, and that the servicing feature-usage metric was captured in +// the install trace-stream file. It self-skips on the container runtime, where +// these host-side concerns do not apply. +func (s *TridentE2EScenario) validateHostDiagnostics(tc storm.TestCase) error { + if s.runtime != trident.RuntimeTypeHost { + tc.Skip("Host diagnostics (SELinux + tracing) only apply to the host runtime") + } + + connCtx, cancel := context.WithTimeout(tc.Context(), time.Minute) + defer cancel() + if err := s.populateSshClient(connCtx); err != nil { + return err + } + + var sa validate.SoftAsserter + validate.ValidateSelinuxDenials(&sa, s.sshClient) + validate.ValidateJournaldTracing(&sa, s.sshClient) + validate.ValidateTraceFileMetric(&sa, s.cleanInstallTraceFile()) + + if err := sa.Err(); err != nil { + tc.FailFromError(err) + } + logrus.Infof("Host diagnostics validation summary:\n%s", sa.Summary()) + + return nil +} + +// Unlike validateHostState (which self-selects rollback validation only for +// scenarios whose Host Config declares a top-level `health` section), this case +// always asserts the rollback outcome: the failed update rolled back onto the +// current volume, so the host stays provisioned with the active volume +// unchanged. It ports rollback_test.py for the auto-rollback (provisioned) +// case. +func (s *TridentE2EScenario) validateAutoRollback(tc storm.TestCase) error { + connCtx, cancel := context.WithTimeout(tc.Context(), time.Minute) + defer cancel() + if err := s.populateSshClient(connCtx); err != nil { + return err + } + + hs, err := trident.GetHostStatus(s.runtime, s.sshClient) + if err != nil { + return err + } + + var sa validate.SoftAsserter + validate.ValidateRollback(&sa, s.sshClient, hs, + trident.ServicingStateProvisioned, s.expectedActiveVolume) + + if err := sa.Err(); err != nil { + tc.FailFromError(err) + } + + logrus.Infof("Auto-rollback validation summary:\n%s", sa.Summary()) + + return nil +} diff --git a/tools/storm/e2e/validate/base.go b/tools/storm/e2e/validate/base.go new file mode 100644 index 0000000000..facd548f8e --- /dev/null +++ b/tools/storm/e2e/validate/base.go @@ -0,0 +1,179 @@ +package validate + +import ( + "math" + "strconv" + "strings" + + "tridenttools/pkg/hostconfig" + tridentutil "tridenttools/storm/utils/trident" +) + +// sizeUnits maps a single-letter size suffix to its multiplier (powers of 1024), +// matching base_test.py's SizeUnit enum. +var sizeUnits = map[byte]float64{ + 'B': 1, + 'K': math.Pow(1024, 1), + 'M': math.Pow(1024, 2), + 'G': math.Pow(1024, 3), + 'T': math.Pow(1024, 4), + 'P': math.Pow(1024, 5), +} + +// ParseSizeToBytes converts a Host Configuration partition size string (e.g. +// "8G", "192M", "1024") into bytes. The final character may be a unit suffix +// (B/K/M/G/T/P); a bare number is treated as bytes. Non-numeric sizes such as +// "grow" return ok=false so callers can skip the size expectation. +func ParseSizeToBytes(size string) (int64, bool) { + size = strings.TrimSpace(size) + if size == "" { + return 0, false + } + + last := size[len(size)-1] + numPart := size + multiplier := 1.0 + + if unit, isUnit := sizeUnits[last]; isUnit { + multiplier = unit + numPart = size[:len(size)-1] + } else if last < '0' || last > '9' { + // Trailing non-digit, non-unit character (e.g. "grow"): not a size. + return 0, false + } + + value, err := strconv.ParseFloat(numPart, 64) + if err != nil { + return 0, false + } + + return int64(value * multiplier), true +} + +// PartitionExpectation is a partition declared in the Host Configuration. +type PartitionExpectation struct { + ID string + SizeBytes int64 + HasSize bool +} + +// ExpectedPartitions extracts the partitions declared across all disks in the +// given Host Configuration spec. +func ExpectedPartitions(spec hostconfig.HostConfig) []PartitionExpectation { + var result []PartitionExpectation + for _, disk := range spec.S("storage", "disks").Children() { + for _, part := range disk.S("partitions").Children() { + id, ok := part.S("id").Data().(string) + if !ok { + continue + } + exp := PartitionExpectation{ID: id} + if sizeStr, ok := part.S("size").Data().(string); ok { + exp.SizeBytes, exp.HasSize = ParseSizeToBytes(sizeStr) + } + result = append(result, exp) + } + } + return result +} + +// IsPartition reports whether the block device ID corresponds to a disk +// partition declared in the spec. +func IsPartition(spec hostconfig.HostConfig, deviceID string) bool { + for _, disk := range spec.S("storage", "disks").Children() { + for _, part := range disk.S("partitions").Children() { + if id, ok := part.S("id").Data().(string); ok && id == deviceID { + return true + } + } + } + return false +} + +// IsRaid reports whether the block device ID corresponds to a software RAID +// array declared in the spec. +func IsRaid(spec hostconfig.HostConfig, deviceID string) bool { + for _, raid := range spec.S("storage", "raid", "software").Children() { + if id, ok := raid.S("id").Data().(string); ok && id == deviceID { + return true + } + } + return false +} + +// mountPointPath extracts the "/" path from a filesystem's mountPoint field, +// which may be either a plain string or an object with a "path" key. Returns +// ("", false) if no mount point is set. +func mountPointPath(fs *hostconfig.HostConfig) (string, bool) { + mp := fs.S("mountPoint") + if mp == nil || mp.Data() == nil { + return "", false + } + if str, ok := mp.Data().(string); ok { + return str, true + } + if path, ok := mp.S("path").Data().(string); ok { + return path, true + } + return "", false +} + +// RootFilesystemDeviceID returns the deviceId of the filesystem mounted at "/" +// in the spec, and whether one was found. +func RootFilesystemDeviceID(spec hostconfig.HostConfig) (string, bool) { + for _, fs := range spec.S("storage", "filesystems").Children() { + path, ok := mountPointPath(&hostconfig.HostConfig{Container: fs}) + if !ok || path != "/" { + continue + } + if id, ok := fs.S("deviceId").Data().(string); ok { + return id, true + } + } + return "", false +} + +// ActiveVolumeID resolves the active volume's block-device ID for the A/B +// volume pair whose `id` equals volumePairID, given the active A/B selection. +// Returns ("", false) if no matching volume pair exists. +func ActiveVolumeID(spec hostconfig.HostConfig, volumePairID string, active tridentutil.AbVolumeSelection) (string, bool) { + for _, pair := range spec.S("storage", "abUpdate", "volumePairs").Children() { + id, ok := pair.S("id").Data().(string) + if !ok || id != volumePairID { + continue + } + field := "volumeAId" + if active == tridentutil.AbVolumeB { + field = "volumeBId" + } + if vid, ok := pair.S(field).Data().(string); ok { + return vid, true + } + } + return "", false +} + +// verityDataDeviceID returns the dataDeviceId of the verity device whose id +// matches the given device ID, and whether such a verity device exists. +func verityDataDeviceID(spec hostconfig.HostConfig, deviceID string) (string, bool) { + for _, v := range spec.S("storage", "verity").Children() { + if id, ok := v.S("id").Data().(string); ok && id == deviceID { + if data, ok := v.S("dataDeviceId").Data().(string); ok { + return data, true + } + return "", true + } + } + return "", false +} + +// AbVolumePairID returns the A/B volume-pair ID backing the root filesystem. +// If root sits on a verity device, the pair ID is the verity data device; +// otherwise it is the root filesystem's device ID directly. The second return +// value is true when root is a verity device. +func AbVolumePairID(spec hostconfig.HostConfig, rootDeviceID string) (pairID string, rootIsVerity bool) { + if dataID, isVerity := verityDataDeviceID(spec, rootDeviceID); isVerity { + return dataID, true + } + return rootDeviceID, false +} diff --git a/tools/storm/e2e/validate/base_test.go b/tools/storm/e2e/validate/base_test.go new file mode 100644 index 0000000000..d8e35b2a75 --- /dev/null +++ b/tools/storm/e2e/validate/base_test.go @@ -0,0 +1,140 @@ +package validate + +import ( + "testing" + + "tridenttools/pkg/hostconfig" + tridentutil "tridenttools/storm/utils/trident" +) + +func TestParseSizeToBytes(t *testing.T) { + cases := []struct { + in string + want int64 + wantOk bool + }{ + {"8G", 8 * 1024 * 1024 * 1024, true}, + {"192M", 192 * 1024 * 1024, true}, + {"1K", 1024, true}, + {"512B", 512, true}, + {"1024", 1024, true}, + {"grow", 0, false}, + {"", 0, false}, + {"4.5G", int64(4.5 * 1024 * 1024 * 1024), true}, + } + for _, c := range cases { + got, ok := ParseSizeToBytes(c.in) + if ok != c.wantOk || (ok && got != c.want) { + t.Errorf("ParseSizeToBytes(%q) = (%d,%v), want (%d,%v)", c.in, got, ok, c.want, c.wantOk) + } + } +} + +const specYaml = ` +storage: + disks: + - id: os + partitions: + - id: root-a + size: 8G + - id: root-b + size: 8G + - id: esp + size: 1G + raid: + software: + - id: md-root + name: root + abUpdate: + volumePairs: + - id: root + volumeAId: root-a + volumeBId: root-b + filesystems: + - deviceId: root + mountPoint: / + - deviceId: esp + mountPoint: + path: /boot/efi + options: umask=0077 + verity: + - id: root + name: root + dataDeviceId: root-data + hashDeviceId: root-hash +` + +func mustSpec(t *testing.T) hostconfig.HostConfig { + t.Helper() + hc, err := hostconfig.NewHostConfigFromYaml([]byte(specYaml)) + if err != nil { + t.Fatalf("failed to parse spec: %v", err) + } + return hc +} + +func TestExpectedPartitions(t *testing.T) { + parts := ExpectedPartitions(mustSpec(t)) + if len(parts) != 3 { + t.Fatalf("got %d partitions, want 3", len(parts)) + } + byID := map[string]PartitionExpectation{} + for _, p := range parts { + byID[p.ID] = p + } + if !byID["root-a"].HasSize || byID["root-a"].SizeBytes != 8*1024*1024*1024 { + t.Errorf("root-a = %+v", byID["root-a"]) + } +} + +func TestIsPartitionIsRaid(t *testing.T) { + spec := mustSpec(t) + if !IsPartition(spec, "root-a") { + t.Error("root-a should be a partition") + } + if IsPartition(spec, "md-root") { + t.Error("md-root should not be a partition") + } + if !IsRaid(spec, "md-root") { + t.Error("md-root should be a raid array") + } + if IsRaid(spec, "root-a") { + t.Error("root-a should not be raid") + } +} + +func TestRootFilesystemDeviceID(t *testing.T) { + id, ok := RootFilesystemDeviceID(mustSpec(t)) + if !ok || id != "root" { + t.Errorf("got (%q,%v), want (root,true)", id, ok) + } +} + +func TestActiveVolumeID(t *testing.T) { + spec := mustSpec(t) + a, ok := ActiveVolumeID(spec, "root", tridentutil.AbVolumeA) + if !ok || a != "root-a" { + t.Errorf("volume-a: got (%q,%v), want (root-a,true)", a, ok) + } + b, ok := ActiveVolumeID(spec, "root", tridentutil.AbVolumeB) + if !ok || b != "root-b" { + t.Errorf("volume-b: got (%q,%v), want (root-b,true)", b, ok) + } + if _, ok := ActiveVolumeID(spec, "nonexistent", tridentutil.AbVolumeA); ok { + t.Error("nonexistent pair should return ok=false") + } +} + +func TestAbVolumePairID(t *testing.T) { + spec := mustSpec(t) + // root is a verity device in specYaml -> pair id is its dataDeviceId. + pairID, isVerity := AbVolumePairID(spec, "root") + if !isVerity || pairID != "root-data" { + t.Errorf("verity root: got (%q,%v), want (root-data,true)", pairID, isVerity) + } + // esp is not a verity device -> pair id is the device id itself. + pairID, isVerity = AbVolumePairID(spec, "esp") + if isVerity || pairID != "esp" { + t.Errorf("non-verity: got (%q,%v), want (esp,false)", pairID, isVerity) + } +} diff --git a/tools/storm/e2e/validate/diagnostics.go b/tools/storm/e2e/validate/diagnostics.go new file mode 100644 index 0000000000..87a1fd2b9c --- /dev/null +++ b/tools/storm/e2e/validate/diagnostics.go @@ -0,0 +1,106 @@ +package validate + +import ( + "bufio" + "encoding/json" + "os" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// Diagnostic metric identifiers, mirroring the defaults of the legacy +// check-tracing / check-selinux helpers. These validations only apply to the +// host runtime (SELinux enforcement and Trident's journald tracing are host +// concerns), and are scoped to the clean install. +const ( + // tridentTracingSyslogIdentifier is the journald syslog identifier Trident + // tags its tracing metrics with. + tridentTracingSyslogIdentifier = "trident-tracing" + // tridentStartMetric is a metric emitted by Trident's commit that must + // appear in the journald tracing stream. + tridentStartMetric = "trident_start" + // hostConfigFeatureUsageMetric is a metric collected throughout servicing + // that must appear in the captured trace-stream file. + hostConfigFeatureUsageMetric = "host_config_feature_usage" + + auditLogPath = "/var/log/audit/audit.log" +) + +// ValidateSelinuxDenials ports the check-selinux helper. It runs `audit2allow` +// against the host's audit log and surfaces any SELinux denials. Matching the +// legacy helper, it fails only when the command cannot be run (not when +// denials are present) — the denials are logged for human inspection. +func ValidateSelinuxDenials(sa *SoftAsserter, client *ssh.Client) { + out, err := sshutils.RunCommand(client, "sudo audit2allow -i "+auditLogPath) + if err != nil { + sa.Fail("selinux/audit2allow", err) + return + } + if strings.TrimSpace(out.Stdout) != "" { + sa.Passf("selinux/audit2allow", "audit2allow reported potential denials:\n%s", out.Stdout) + } else { + sa.Pass("selinux/audit2allow") + } +} + +// ValidateJournaldTracing ports check-tracing's check-journald. It confirms the +// Trident tracing metric emitted by commit (trident_start) is present in the +// host's journald logs under the trident-tracing syslog identifier. +func ValidateJournaldTracing(sa *SoftAsserter, client *ssh.Client) { + out, err := sshutils.RunCommand(client, "sudo journalctl -t "+tridentTracingSyslogIdentifier+" -o json") + if err != nil { + sa.Fail("tracing/journald", err) + return + } + + scanner := bufio.NewScanner(strings.NewReader(out.Stdout)) + scanner.Buffer(make([]byte, 0, 1024*1024), 8*1024*1024) + for scanner.Scan() { + var entry map[string]interface{} + if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil { + continue + } + if entry["F_METRIC_NAME"] == tridentStartMetric { + sa.Pass("tracing/journald") + return + } + } + sa.Failf("tracing/journald", "metric %q not found in journald logs for identifier %q", + tridentStartMetric, tridentTracingSyslogIdentifier) +} + +// ValidateTraceFileMetric ports check-tracing's check-trace-file. It confirms +// the feature-usage metric collected during servicing is present in the local +// trace-stream file that netlisten captured for the install. An empty path +// (no trace file configured) is skipped rather than failed, matching the +// helper. +func ValidateTraceFileMetric(sa *SoftAsserter, traceFilePath string) { + if traceFilePath == "" { + return + } + + f, err := os.Open(traceFilePath) + if err != nil { + sa.Fail("tracing/trace-file", err) + return + } + defer f.Close() + + dec := json.NewDecoder(f) + for dec.More() { + var entry map[string]interface{} + if err := dec.Decode(&entry); err != nil { + sa.Fail("tracing/trace-file", err) + return + } + if entry["metric_name"] == hostConfigFeatureUsageMetric { + sa.Pass("tracing/trace-file") + return + } + } + sa.Failf("tracing/trace-file", "metric %q not found in trace file %q", + hostConfigFeatureUsageMetric, traceFilePath) +} diff --git a/tools/storm/e2e/validate/diagnostics_test.go b/tools/storm/e2e/validate/diagnostics_test.go new file mode 100644 index 0000000000..8c1998a3dc --- /dev/null +++ b/tools/storm/e2e/validate/diagnostics_test.go @@ -0,0 +1,57 @@ +package validate + +import ( + "os" + "path/filepath" + "testing" +) + +func writeTempFile(t *testing.T, content string) string { + t.Helper() + dir := t.TempDir() + p := filepath.Join(dir, "metrics.jsonl") + if err := os.WriteFile(p, []byte(content), 0644); err != nil { + t.Fatalf("write temp file: %v", err) + } + return p +} + +func TestValidateTraceFileMetric_Found(t *testing.T) { + // Concatenated JSON objects, matching netlisten's trace-stream format. + path := writeTempFile(t, + `{"metric_name":"something_else","value":1}`+"\n"+ + `{"metric_name":"host_config_feature_usage","value":42}`+"\n") + var sa SoftAsserter + ValidateTraceFileMetric(&sa, path) + if sa.HasFailures() { + t.Errorf("expected pass, got failures: %v", sa.Err()) + } +} + +func TestValidateTraceFileMetric_NotFound(t *testing.T) { + path := writeTempFile(t, `{"metric_name":"other","value":1}`+"\n") + var sa SoftAsserter + ValidateTraceFileMetric(&sa, path) + if !sa.HasFailures() { + t.Error("expected a failure when the feature-usage metric is absent") + } +} + +func TestValidateTraceFileMetric_EmptyPathSkips(t *testing.T) { + var sa SoftAsserter + ValidateTraceFileMetric(&sa, "") + if sa.HasFailures() || sa.Failures() != 0 { + t.Error("empty trace file path should be skipped, not failed") + } + if len(sa.results) != 0 { + t.Error("empty trace file path should record no sub-check") + } +} + +func TestValidateTraceFileMetric_MissingFileFails(t *testing.T) { + var sa SoftAsserter + ValidateTraceFileMetric(&sa, filepath.Join(t.TempDir(), "does-not-exist.jsonl")) + if !sa.HasFailures() { + t.Error("a configured but missing trace file should fail") + } +} diff --git a/tools/storm/e2e/validate/encryption.go b/tools/storm/e2e/validate/encryption.go new file mode 100644 index 0000000000..ece6be6c68 --- /dev/null +++ b/tools/storm/e2e/validate/encryption.go @@ -0,0 +1,395 @@ +package validate + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/pkg/hostconfig" + "tridenttools/storm/utils/sshutils" + "tridenttools/storm/utils/sysinspect" + tridentutil "tridenttools/storm/utils/trident" +) + +const ( + expectedCipher = "aes-xts-plain64" + expectedKeysize = "512 bits" + expectedFsType = "ext4" + expectedLuksType = "crypto_LUKS" + expectedDigestType = "pbkdf2" + expectedDigestHash = "sha512" +) + +// HasEncryption reports whether the spec declares encryption volumes, used to +// self-select the encryption validation. +func HasEncryption(hs tridentutil.HostStatus) bool { + return hs.Spec().Exists("storage", "encryption") +} + +// ValidateEncryption ports encryption_test.py::test_encryption. For each +// configured encryption volume it validates the backing device is LUKS, the +// LUKS metadata, the device-mapper state, and the mount/swap/active status +// (accounting for A/B update pairs). isUki selects the expected TPM2 policy. +func ValidateEncryption( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + isUki bool, + abActive tridentutil.AbVolumeSelection, +) { + spec := hs.Spec() + + blockDevs, err := sysinspect.BlkidExport(client) + if err != nil { + sa.Fail("encryption/blkid-export", err) + return + } + + for _, crypt := range spec.S("storage", "encryption", "volumes").Children() { + cryptID, _ := crypt.S("id").Data().(string) + cryptDevName, _ := crypt.S("deviceName").Data().(string) + cryptDevID, _ := crypt.S("deviceId").Data().(string) + checkCryptDevice(sa, client, spec, isUki, abActive, blockDevs, cryptID, cryptDevName, cryptDevID) + } +} + +func checkCryptDevice( + sa *SoftAsserter, + client *ssh.Client, + spec hostconfig.HostConfig, + isUki bool, + abActive tridentutil.AbVolumeSelection, + blockDevs map[string]map[string]string, + cryptID, cryptDevName, cryptDevID string, +) { + cryptDevicePath := "/dev/mapper/" + cryptDevName + + checkParentDevices(sa, client, spec, isUki, blockDevs, cryptDevID) + + swap := false + isInUse := true + + if pair, isVolumeA, ok := childAbUpdateVolumePair(spec, cryptID); ok { + // Encryption volume is an A/B pair member: it is in use only when its + // side matches the active volume. + isInUse = (abActive == tridentutil.AbVolumeA && isVolumeA) || + (abActive == tridentutil.AbVolumeB && !isVolumeA) + + pairID, _ := pair.S("id").Data().(string) + fs, ok := filesystemByDeviceID(spec, pairID) + if !ok { + sa.Failf("encryption/ab-fs", "no filesystem for A/B volume pair %q", pairID) + } else if mp, ok := mountPointPath(&fs); ok { + checkExists(sa, client, mp) + checkFindmnt(sa, client, mp, cryptDevicePath, isInUse) + } else { + sa.Failf("encryption/ab-mount", "no mount point for A/B volume pair %q", pairID) + } + } else if isSwapDevice(spec, cryptID) { + swap = true + swaps, err := sysinspect.ActiveSwaps(client) + if err != nil { + sa.Fail("encryption/swaps", err) + } else { + realPath, err := sysinspect.ReadlinkF(client, cryptDevicePath) + if err != nil { + sa.Fail("encryption/swap-readlink", err) + } else if _, active := swaps[realPath]; !active { + sa.Failf("encryption/swap-active", "swap %q not in active swaps %v", realPath, swaps) + } + } + } else { + fs, ok := filesystemByDeviceID(spec, cryptID) + if !ok { + sa.Failf("encryption/fs", "no filesystem for encryption volume %q", cryptID) + } else if mp, ok := mountPointPath(&fs); ok { + checkExists(sa, client, mp) + checkFindmnt(sa, client, mp, cryptDevicePath, isInUse) + } else { + sa.Failf("encryption/mount", "encryption volume %q filesystem is not mounted", cryptID) + } + } + + checkExists(sa, client, cryptDevicePath) + checkCryptsetupStatus(sa, client, cryptDevName, isInUse) + checkDmsetupInfo(sa, client, cryptDevName, swap) +} + +// checkParentDevices confirms the crypt device's backing block device is LUKS +// and validates its LUKS metadata. +func checkParentDevices( + sa *SoftAsserter, + client *ssh.Client, + spec hostconfig.HostConfig, + isUki bool, + blockDevs map[string]map[string]string, + cryptDevID string, +) { + var cryptDevPath string + if IsPartition(spec, cryptDevID) { + path, ok := blockDevPathByPartlabel(blockDevs, cryptDevID) + if !ok { + sa.Failf("encryption/parent-partition", "no device with PARTLABEL %q", cryptDevID) + return + } + cryptDevPath = path + } else { + raidName, ok := raidSoftwareArrayName(spec, cryptDevID) + if !ok { + sa.Failf("encryption/parent-kind", "%q is neither a partition nor a RAID array", cryptDevID) + return + } + path, err := sysinspect.ReadlinkF(client, "/dev/md/"+raidName) + if err != nil { + sa.Fail("encryption/parent-raid", err) + return + } + cryptDevPath = path + } + + dev, ok := blockDevs[cryptDevPath] + if !ok { + sa.Failf("encryption/parent-blkid", "no blkid entry for %q", cryptDevPath) + return + } + sa.Assert("encryption/parent-type", dev["TYPE"] == expectedLuksType, + "device %q TYPE = %q, want %q", cryptDevPath, dev["TYPE"], expectedLuksType) + + checkLuksDump(sa, client, cryptDevPath, isUki) +} + +// checkLuksDump validates the LUKS2 metadata from cryptsetup luksDump. +func checkLuksDump(sa *SoftAsserter, client *ssh.Client, cryptDevPath string, isUki bool) { + // luksDump needs an SELinux permission the Trident policy intentionally + // omits; temporarily drop to permissive (a testing-infra quirk), matching + // encryption_test.py. + enforcing := false + if mode, err := sysinspect.Getenforce(client); err == nil && mode == "Enforcing" { + enforcing = true + if err := sysinspect.Setenforce(client, false); err != nil { + sa.Fail("encryption/selinux", err) + } + } + + dump, err := sysinspect.CryptsetupLuksDump(client, cryptDevPath) + + if enforcing { + // Best-effort restore of enforcing mode. + if restoreErr := sysinspect.Setenforce(client, true); restoreErr != nil { + sa.Fail("encryption/selinux-restore", restoreErr) + } + } + + if err != nil { + sa.Fail("encryption/luks-dump", err) + return + } + + digest, ok := dump.Digests["0"] + if !ok { + sa.Failf("encryption/luks-digest", "luksDump missing digest 0") + } else { + sa.Assert("encryption/luks-digest-type", digest.Type == expectedDigestType, + "digest type = %q, want %q", digest.Type, expectedDigestType) + sa.Assert("encryption/luks-digest-hash", digest.Hash == expectedDigestHash, + "digest hash = %q, want %q", digest.Hash, expectedDigestHash) + } + + token, ok := dump.Tokens["0"] + if !ok { + sa.Failf("encryption/luks-token", "luksDump missing token 0") + } else { + sa.Assert("encryption/luks-token-count", len(dump.Tokens) == 1, + "expected 1 token, got %d", len(dump.Tokens)) + sa.Assert("encryption/luks-token-keyslots", len(token.Keyslots) == 1 && contains(token.Keyslots, "1"), + "expected token keyslot [1], got %v", token.Keyslots) + sa.Assert("encryption/luks-token-type", token.Type == "systemd-tpm2", + "token type = %q, want systemd-tpm2", token.Type) + if isUki { + sa.Assert("encryption/luks-pcrlock", token.Tpm2Pcrlock, + "expected tpm2_pcrlock=true for UKI image") + sa.Assert("encryption/luks-pcrs", len(token.Tpm2Pcrs) == 0, + "expected empty tpm2-pcrs for UKI image, got %v", token.Tpm2Pcrs) + } else { + sa.Assert("encryption/luks-pcrlock", !token.Tpm2Pcrlock, + "expected tpm2_pcrlock=false for non-UKI image") + sa.Assert("encryption/luks-pcrs", len(token.Tpm2Pcrs) == 1 && token.Tpm2Pcrs[0] == 7, + "expected tpm2-pcrs=[7] for non-UKI image, got %v", token.Tpm2Pcrs) + } + } + + keyslot, ok := dump.Keyslots["1"] + if !ok { + sa.Failf("encryption/luks-keyslot", "luksDump missing keyslot 1") + } else { + sa.Assert("encryption/luks-keyslot-count", len(dump.Keyslots) == 1, + "expected 1 keyslot, got %d", len(dump.Keyslots)) + sa.Assert("encryption/luks-keyslot-type", keyslot.Type == "luks2", + "keyslot type = %q, want luks2", keyslot.Type) + sa.Assert("encryption/luks-kdf-type", keyslot.Kdf.Type == "pbkdf2", + "keyslot kdf type = %q, want pbkdf2", keyslot.Kdf.Type) + sa.Assert("encryption/luks-kdf-hash", keyslot.Kdf.Hash == "sha512", + "keyslot kdf hash = %q, want sha512", keyslot.Kdf.Hash) + sa.Assert("encryption/luks-area-enc", keyslot.Area.Encryption == expectedCipher, + "keyslot area encryption = %q, want %q", keyslot.Area.Encryption, expectedCipher) + } +} + +func checkCryptsetupStatus(sa *SoftAsserter, client *ssh.Client, name string, isInUse bool) { + status, err := sysinspect.Cryptsetup(client, name) + if err != nil { + sa.Fail("encryption/cryptsetup-status", err) + return + } + if isInUse { + sa.Assert("encryption/cryptsetup-inuse", status.InUse, + "expected %q to be active and in use", name) + } else { + sa.Assert("encryption/cryptsetup-active", status.Active && !status.InUse, + "expected %q to be active but not in use", name) + } + if cipher, _ := status.Get("cipher"); cipher != expectedCipher { + sa.Failf("encryption/cryptsetup-cipher", "cipher = %q, want %q", cipher, expectedCipher) + } + if keysize, _ := status.Get("keysize"); keysize != expectedKeysize { + sa.Failf("encryption/cryptsetup-keysize", "keysize = %q, want %q", keysize, expectedKeysize) + } +} + +func checkDmsetupInfo(sa *SoftAsserter, client *ssh.Client, name string, swap bool) { + info, err := sysinspect.DmsetupInfo(client, name) + if err != nil { + sa.Fail("encryption/dmsetup", err) + return + } + sa.Assert("encryption/dmsetup-name", info["Name"] == name, + "dmsetup Name = %q, want %q", info["Name"], name) + sa.Assert("encryption/dmsetup-state", info["State"] == "ACTIVE", + "dmsetup State = %q, want ACTIVE", info["State"]) + sa.Assert("encryption/dmsetup-tables", info["Tables present"] == "LIVE", + "dmsetup Tables present = %q, want LIVE", info["Tables present"]) + + cryptKind := "LUKS2" + if swap { + cryptKind = "PLAIN" + } + prefix := fmt.Sprintf("CRYPT-%s-", cryptKind) + suffix := "-" + name + uuid := info["UUID"] + sa.Assert("encryption/dmsetup-uuid-prefix", strings.HasPrefix(uuid, prefix), + "dmsetup UUID %q does not start with %q", uuid, prefix) + sa.Assert("encryption/dmsetup-uuid-suffix", strings.HasSuffix(uuid, suffix), + "dmsetup UUID %q does not end with %q", uuid, suffix) +} + +// checkExists runs `sudo ls ` and records a failure if it does not exist. +func checkExists(sa *SoftAsserter, client *ssh.Client, path string) { + out, err := sshutils.RunCommand(client, fmt.Sprintf("sudo ls %s", path)) + if err != nil { + sa.Fail("encryption/exists", err) + return + } + sa.Assert("encryption/exists", out.Status == 0, "path does not exist: %s", path) +} + +// checkFindmnt validates the findmnt row for target: when active, SOURCE must be +// the crypt device; when inactive, SOURCE must differ. FSTYPE is always ext4. +func checkFindmnt(sa *SoftAsserter, client *ssh.Client, target, source string, isActive bool) { + rows, err := sysinspect.Findmnt(client, target) + if err != nil { + sa.Fail("encryption/findmnt", err) + return + } + if len(rows) != 1 { + sa.Failf("encryption/findmnt-rows", "expected 1 findmnt row for %q, got %d", target, len(rows)) + return + } + row := rows[0] + sa.Assert("encryption/findmnt-target", row.Target == target, + "findmnt TARGET = %q, want %q", row.Target, target) + sa.Assert("encryption/findmnt-fstype", row.FsType == expectedFsType, + "findmnt FSTYPE = %q, want %q", row.FsType, expectedFsType) + if isActive { + sa.Assert("encryption/findmnt-source", row.Source == source, + "findmnt SOURCE = %q, want %q (active)", row.Source, source) + } else { + sa.Assert("encryption/findmnt-source", row.Source != source, + "findmnt SOURCE = %q, expected different from %q (inactive)", row.Source, source) + } +} + +// --- Host Configuration helpers (operate on the Host Status spec) --- + +// childAbUpdateVolumePair returns the A/B volume pair that has cryptID as one of +// its volumes, whether cryptID is the A side, and whether such a pair exists. +func childAbUpdateVolumePair(spec hostconfig.HostConfig, cryptID string) (*hostconfig.HostConfig, bool, bool) { + for _, pair := range spec.S("storage", "abUpdate", "volumePairs").Children() { + if a, _ := pair.S("volumeAId").Data().(string); a == cryptID { + hc := hostconfig.NewHostConfigFromContainer(pair) + return &hc, true, true + } + if b, _ := pair.S("volumeBId").Data().(string); b == cryptID { + hc := hostconfig.NewHostConfigFromContainer(pair) + return &hc, false, true + } + } + return nil, false, false +} + +// filesystemByDeviceID returns the filesystem with the given deviceId. +func filesystemByDeviceID(spec hostconfig.HostConfig, deviceID string) (hostconfig.HostConfig, bool) { + for _, fs := range spec.S("storage", "filesystems").Children() { + if id, _ := fs.S("deviceId").Data().(string); id == deviceID { + return hostconfig.NewHostConfigFromContainer(fs), true + } + } + return hostconfig.HostConfig{}, false +} + +// isSwapDevice reports whether devID is configured as swap. storage.swap entries +// may be plain device-id strings or objects with a deviceId field. +func isSwapDevice(spec hostconfig.HostConfig, devID string) bool { + for _, swap := range spec.S("storage", "swap").Children() { + if s, ok := swap.Data().(string); ok && s == devID { + return true + } + if id, ok := swap.S("deviceId").Data().(string); ok && id == devID { + return true + } + } + return false +} + +// raidSoftwareArrayName returns the name of the software RAID array with the +// given id. +func raidSoftwareArrayName(spec hostconfig.HostConfig, id string) (string, bool) { + for _, raid := range spec.S("storage", "raid", "software").Children() { + if rid, _ := raid.S("id").Data().(string); rid == id { + if name, ok := raid.S("name").Data().(string); ok { + return name, true + } + } + } + return "", false +} + +// blockDevPathByPartlabel returns the device path whose PARTLABEL matches label. +func blockDevPathByPartlabel(blockDevs map[string]map[string]string, label string) (string, bool) { + for path, dev := range blockDevs { + if dev["PARTLABEL"] == label { + return path, true + } + } + return "", false +} + +// contains reports whether s contains v. +func contains(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} diff --git a/tools/storm/e2e/validate/encryption_test.go b/tools/storm/e2e/validate/encryption_test.go new file mode 100644 index 0000000000..4373a284fd --- /dev/null +++ b/tools/storm/e2e/validate/encryption_test.go @@ -0,0 +1,156 @@ +package validate + +import ( + "testing" + + "tridenttools/pkg/hostconfig" + tridentutil "tridenttools/storm/utils/trident" +) + +const encSpecYaml = ` +storage: + encryption: + volumes: + - id: enc-root + deviceName: root + deviceId: root-luks + - id: enc-swap + deviceName: swapdev + deviceId: swap-part + disks: + - id: os + partitions: + - id: swap-part + size: 2G + raid: + software: + - id: root-luks + name: rootarray + abUpdate: + volumePairs: + - id: root + volumeAId: enc-root + volumeBId: enc-root-b + filesystems: + - deviceId: root + mountPoint: / + swap: + - enc-swap +` + +func encSpec(t *testing.T) hostconfig.HostConfig { + t.Helper() + hc, err := hostconfig.NewHostConfigFromYaml([]byte(encSpecYaml)) + if err != nil { + t.Fatalf("parse: %v", err) + } + return hc +} + +func TestHasEncryption(t *testing.T) { + hs, _ := tridentutil.NewHostStatusFromYaml([]byte("spec:\n" + indent(encSpecYaml))) + if !HasEncryption(hs) { + t.Error("expected HasEncryption=true") + } + plain, _ := tridentutil.NewHostStatusFromYaml([]byte("spec:\n storage:\n disks: []\n")) + if HasEncryption(plain) { + t.Error("expected HasEncryption=false") + } +} + +func TestChildAbUpdateVolumePair(t *testing.T) { + spec := encSpec(t) + pair, isA, ok := childAbUpdateVolumePair(spec, "enc-root") + if !ok || !isA { + t.Fatalf("enc-root: ok=%v isA=%v, want true,true", ok, isA) + } + if id, _ := pair.S("id").Data().(string); id != "root" { + t.Errorf("pair id = %q, want root", id) + } + if _, isA, ok := childAbUpdateVolumePair(spec, "enc-root-b"); !ok || isA { + t.Errorf("enc-root-b: ok=%v isA=%v, want true,false", ok, isA) + } + if _, _, ok := childAbUpdateVolumePair(spec, "nope"); ok { + t.Error("nope should not be found") + } +} + +func TestFilesystemByDeviceID(t *testing.T) { + spec := encSpec(t) + fs, ok := filesystemByDeviceID(spec, "root") + if !ok { + t.Fatal("root fs not found") + } + if mp, ok := mountPointPath(&fs); !ok || mp != "/" { + t.Errorf("mount = %q ok=%v, want /", mp, ok) + } + if _, ok := filesystemByDeviceID(spec, "missing"); ok { + t.Error("missing fs should not be found") + } +} + +func TestIsSwapDevice(t *testing.T) { + spec := encSpec(t) + if !isSwapDevice(spec, "enc-swap") { + t.Error("enc-swap should be swap") + } + if isSwapDevice(spec, "enc-root") { + t.Error("enc-root should not be swap") + } +} + +func TestRaidSoftwareArrayName(t *testing.T) { + spec := encSpec(t) + name, ok := raidSoftwareArrayName(spec, "root-luks") + if !ok || name != "rootarray" { + t.Errorf("got (%q,%v), want (rootarray,true)", name, ok) + } + if _, ok := raidSoftwareArrayName(spec, "swap-part"); ok { + t.Error("swap-part is a partition, not raid") + } +} + +func TestBlockDevPathByPartlabel(t *testing.T) { + devs := map[string]map[string]string{ + "/dev/sda4": {"PARTLABEL": "swap-part", "TYPE": "crypto_LUKS"}, + "/dev/sda1": {"PARTLABEL": "esp"}, + } + path, ok := blockDevPathByPartlabel(devs, "swap-part") + if !ok || path != "/dev/sda4" { + t.Errorf("got (%q,%v), want (/dev/sda4,true)", path, ok) + } + if _, ok := blockDevPathByPartlabel(devs, "nope"); ok { + t.Error("nope should not match") + } +} + +// indent prefixes every non-empty line with two spaces (to nest encSpecYaml +// under a `spec:` key). +func indent(s string) string { + out := "" + for _, line := range splitLines(s) { + if line == "" { + out += "\n" + } else { + out += " " + line + "\n" + } + } + return out +} + +func splitLines(s string) []string { + var lines []string + cur := "" + for _, r := range s { + if r == '\n' { + lines = append(lines, cur) + cur = "" + } else { + cur += string(r) + } + } + if cur != "" { + lines = append(lines, cur) + } + return lines +} diff --git a/tools/storm/e2e/validate/extensions.go b/tools/storm/e2e/validate/extensions.go new file mode 100644 index 0000000000..2c5d67f88e --- /dev/null +++ b/tools/storm/e2e/validate/extensions.go @@ -0,0 +1,87 @@ +package validate + +import ( + "fmt" + "path/filepath" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" + "tridenttools/storm/utils/sysinspect" + tridentutil "tridenttools/storm/utils/trident" +) + +// extensionKinds maps a Host Configuration `os` extension list key to the +// systemd extension type used on the command line. +var extensionKinds = map[string]string{ + "sysexts": "sysext", + "confexts": "confext", +} + +// HasExtensions reports whether the Host Status spec declares any system +// extensions (sysexts or confexts), used to self-select the extensions +// validation. +func HasExtensions(hs tridentutil.HostStatus) bool { + os := hs.Spec().S("os") + for key := range extensionKinds { + if os.Exists(key) { + return true + } + } + return false +} + +// ValidateExtensions ports extensions_test.py::test_extensions. For each +// configured sysext/confext it confirms the extension path exists on the host +// and that the extension is active per `systemd- status`. +func ValidateExtensions(sa *SoftAsserter, client *ssh.Client, hs tridentutil.HostStatus) { + os := hs.Spec().S("os") + + for listKey, extType := range extensionKinds { + configured := os.S(listKey).Children() + if len(configured) == 0 { + continue + } + + active, err := sysinspect.SystemdExtStatus(client, extType) + if err != nil { + sa.Fail(fmt.Sprintf("extensions/%s-status", extType), err) + continue + } + + for _, ext := range configured { + path, ok := ext.S("path").Data().(string) + if !ok { + sa.Failf(fmt.Sprintf("extensions/%s-path", extType), + "configured %s entry has no path", extType) + continue + } + + // Verify the extension path exists on the target OS. + out, err := sshutils.RunCommand(client, fmt.Sprintf("test -e %s", path)) + if err != nil { + sa.Fail(fmt.Sprintf("extensions/%s-exists", extType), err) + } else { + sa.Assert(fmt.Sprintf("extensions/%s-exists", extType), + out.Status == 0, "%s path does not exist: %s", extType, path) + } + + // The active extension name is the file stem (basename minus its + // final extension), matching Python's Path.stem. + name := extensionStem(path) + _, isActive := active[name] + sa.Assert(fmt.Sprintf("extensions/%s-active", extType), + isActive, "%s %q not found in 'systemd-%s status'", extType, name, extType) + } + } +} + +// extensionStem returns the basename of a path with its final extension +// removed, matching Python's pathlib.Path.stem (e.g. "/x/foo.raw" -> "foo", +// "/x/foo.bar.raw" -> "foo.bar"). +func extensionStem(path string) string { + base := filepath.Base(path) + ext := filepath.Ext(base) + return strings.TrimSuffix(base, ext) +} diff --git a/tools/storm/e2e/validate/extensions_test.go b/tools/storm/e2e/validate/extensions_test.go new file mode 100644 index 0000000000..eb31de952f --- /dev/null +++ b/tools/storm/e2e/validate/extensions_test.go @@ -0,0 +1,17 @@ +package validate + +import "testing" + +func TestExtensionStem(t *testing.T) { + cases := map[string]string{ + "/var/lib/ext/foo.raw": "foo", + "/var/lib/ext/foo.bar.raw": "foo.bar", + "foo": "foo", + "/a/b/c.sysext.raw": "c.sysext", + } + for in, want := range cases { + if got := extensionStem(in); got != want { + t.Errorf("extensionStem(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/tools/storm/e2e/validate/orchestrate.go b/tools/storm/e2e/validate/orchestrate.go new file mode 100644 index 0000000000..1779ae6907 --- /dev/null +++ b/tools/storm/e2e/validate/orchestrate.go @@ -0,0 +1,285 @@ +package validate + +import ( + "fmt" + "strings" + + "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" + + "tridenttools/pkg/hostconfig" + "tridenttools/storm/utils/sshutils" + "tridenttools/storm/utils/sysinspect" + tridentutil "tridenttools/storm/utils/trident" +) + +// ValidateBase runs the full `base` marker validation (partitions, users, UEFI +// fallback) against the installed host, accumulating all sub-check failures in +// sa. It ports tests/e2e_tests/base_test.py. +// +// expectedState is the servicing state the host is expected to report (normally +// "provisioned"). abActive is the expected active A/B volume; it is only used +// when the host configuration declares an A/B update. +func ValidateBase( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + expectedState tridentutil.ServicingState, + abActive tridentutil.AbVolumeSelection, +) { + spec := hs.Spec() + + ValidatePartitions(sa, client, hs, expectedState, abActive) + ValidateUsers(sa, client, spec) + ValidateUefiFallback(sa, client, spec) +} + +// ValidatePartitions ports base_test.py::test_partitions. It confirms the +// servicing state, that every configured partition is present both in the Host +// Status and on the running system, and (for A/B configs on non-verity roots) +// that the active volume's device path matches the mounted root device. +func ValidatePartitions( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + expectedState tridentutil.ServicingState, + abActive tridentutil.AbVolumeSelection, +) { + spec := hs.Spec() + + // Gather system state. + blkid, err := sysinspect.Blkid(client) + if err != nil { + sa.Fail("partitions/blkid", err) + return + } + if _, err := sysinspect.Lsblk(client); err != nil { + sa.Fail("partitions/lsblk", err) + return + } + + // Set of PARTLABELs present on the system (partitions_system_info keys in + // base_test.py). + presentPartlabels := make(map[string]struct{}) + for _, entry := range blkid { + if label, ok := entry.Get("PARTLABEL"); ok { + presentPartlabels[label] = struct{}{} + } + } + + // Servicing state. + sa.Assert("partitions/servicing-state", + hs.ServicingState() == expectedState, + "expected servicingState %q, got %q", expectedState, hs.ServicingState()) + + // Every configured partition must appear in the Host Status partitionPaths + // and on the system (as a PARTLABEL). + partitionPaths := hs.PartitionPaths() + for _, part := range ExpectedPartitions(spec) { + if _, ok := partitionPaths[part.ID]; !ok { + sa.Failf("partitions/status-path", + "partition %q missing from Host Status partitionPaths", part.ID) + } + if _, ok := presentPartlabels[part.ID]; !ok { + sa.Failf("partitions/system-present", + "partition %q (PARTLABEL) not found on system", part.ID) + } + } + + // A/B active-volume device-path cross-check (non-verity root only; verity + // A/B is covered by verity validation). + if spec.HasABUpdate() { + validateActiveVolumePath(sa, client, hs, spec, blkid, abActive) + } +} + +// validateActiveVolumePath ports the A/B branch of base_test.py::test_partitions +// for non-verity roots. +func validateActiveVolumePath( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + spec hostconfig.HostConfig, + blkid map[string]sysinspect.BlkidEntry, + abActive tridentutil.AbVolumeSelection, +) { + rootDeviceID, ok := RootFilesystemDeviceID(spec) + if !ok { + sa.Failf("partitions/ab-root", "root mount point not found in Host Status spec") + return + } + + pairID, rootIsVerity := AbVolumePairID(spec, rootDeviceID) + if rootIsVerity { + // Verity root A/B validation lives in the verity validation, not base. + return + } + + activeVolumeID, ok := ActiveVolumeID(spec, pairID, abActive) + if !ok { + sa.Failf("partitions/ab-active", "no volume pair with id %q for %s", pairID, abActive) + return + } + + isPart := IsPartition(spec, activeVolumeID) + isRaid := IsRaid(spec, activeVolumeID) + if isPart == isRaid { + sa.Failf("partitions/ab-kind", + "active volume %q must be exactly one of partition/raid (partition=%v raid=%v)", + activeVolumeID, isPart, isRaid) + return + } + + // Resolve the device path we expect Host Status to report for the active + // volume, based on the actual mounted root device. + rootMountDevice, ok := getRootMountDevice(client) + if !ok { + sa.Failf("partitions/ab-mount", "could not determine device mounted at /") + return + } + rootBasename := rootMountDevice[strings.LastIndex(rootMountDevice, "/")+1:] + + var expectedPath string + switch { + case isPart: + entry, ok := blkid[rootBasename] + if !ok { + sa.Failf("partitions/ab-blkid", "no blkid entry for root device %q", rootBasename) + return + } + partuuid, ok := entry.Get("PARTUUID") + if !ok { + sa.Failf("partitions/ab-partuuid", "root device %q has no PARTUUID", rootBasename) + return + } + expectedPath = "/dev/disk/by-partuuid/" + partuuid + case isRaid: + name, found, err := sysinspect.RaidNameForDevice(client, rootMountDevice) + if err != nil { + sa.Fail("partitions/ab-raid", err) + return + } + if !found { + sa.Failf("partitions/ab-raid", "could not resolve RAID name for %q", rootMountDevice) + return + } + expectedPath = name + } + + if actual, ok := hs.PartitionPaths()[activeVolumeID]; ok { + sa.Assert("partitions/ab-path-match", + actual == expectedPath, + "active volume %q path mismatch: Host Status has %q, expected %q", + activeVolumeID, actual, expectedPath) + } else { + // base_test.py only asserts the path when the active volume ID appears + // in partitionPaths; when it does not (e.g. combined, where root sits on + // the A/B pair but Trident reports the mounted device under a different + // key), it is a no-op rather than a failure. Match that tolerance. + logrus.Infof("Active volume %q not present in Host Status partitionPaths; skipping path match", activeVolumeID) + } + + // Active volume selection must match expectation. + actualVol, present := hs.AbActiveVolume() + sa.Assert("partitions/ab-active-volume", + present && actualVol == abActive, + "expected abActiveVolume %q, got %q (present=%v)", abActive, actualVol, present) +} + +// getRootMountDevice returns the device mounted at "/" from `mount` output. +func getRootMountDevice(client *ssh.Client) (string, bool) { + entries, err := sysinspect.Mount(client) + if err != nil { + return "", false + } + return sysinspect.RootDevice(entries) +} + +// ValidateUsers ports base_test.py::test_users. It confirms that every user and +// group declared in the Host Configuration exists on the system. +func ValidateUsers(sa *SoftAsserter, client *ssh.Client, spec hostconfig.HostConfig) { + systemUsers, err := sysinspect.Users(client) + if err != nil { + sa.Fail("users/passwd", err) + return + } + systemGroups, err := sysinspect.Groups(client) + if err != nil { + sa.Fail("users/group", err) + return + } + + for _, user := range spec.S("os", "users").Children() { + name, ok := user.S("name").Data().(string) + if !ok { + continue + } + if _, present := systemUsers[name]; !present { + sa.Failf("users/present", "configured user %q not found in /etc/passwd", name) + } + + for _, group := range user.S("groups").Children() { + groupName, ok := group.Data().(string) + if !ok { + continue + } + members, present := systemGroups[groupName] + if !present { + sa.Failf("users/group-present", "configured group %q not found in /etc/group", groupName) + continue + } + if _, ok := members[name]; !ok { + sa.Failf("users/group-member", "user %q not a member of group %q", name, groupName) + } + } + } +} + +// ValidateUefiFallback ports base_test.py::test_uefi_fallback. It validates the +// UEFI fallback boot entries according to the configured mode (disabled, +// conservative, optimistic; defaulting to conservative). +func ValidateUefiFallback(sa *SoftAsserter, client *ssh.Client, spec hostconfig.HostConfig) { + mode := "conservative" + if m, ok := spec.S("os", "uefiFallback").Data().(string); ok { + mode = m + } + + switch mode { + case "disabled": + // /efi/boot/EFI/BOOT should be empty. + out, err := sshutils.RunCommand(client, "sudo find /efi/boot/EFI/BOOT/* && exit 1 || exit 0") + if err != nil { + sa.Fail("uefi/disabled", err) + return + } + sa.Assert("uefi/disabled", out.Status == 0, + "/efi/boot/EFI/BOOT is not empty for disabled uefiFallback") + return + case "conservative", "optimistic": + // handled below + default: + sa.Failf("uefi/mode", "unknown uefiFallback mode: %q", mode) + return + } + + info, err := sysinspect.EfiBootMgr(client) + if err != nil { + sa.Fail("uefi/efibootmgr", err) + return + } + currentName, ok := info.CurrentName() + if !ok { + sa.Failf("uefi/current", "could not determine current boot entry name (BootCurrent=%q)", info.BootCurrent) + return + } + + // Fallback boot files should match the current boot's files. + cmd := fmt.Sprintf("sudo diff /efi/boot/EFI/BOOT/* /efi/azl/EFI/%s/* && exit 1 || exit 0", currentName) + out, err := sshutils.RunCommand(client, cmd) + if err != nil { + sa.Fail("uefi/diff", err) + return + } + sa.Assert("uefi/diff", out.Status == 0, + "UEFI fallback files differ from current boot entry %q", currentName) +} diff --git a/tools/storm/e2e/validate/rollback.go b/tools/storm/e2e/validate/rollback.go new file mode 100644 index 0000000000..e45e4fc33c --- /dev/null +++ b/tools/storm/e2e/validate/rollback.go @@ -0,0 +1,102 @@ +package validate + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" + tridentutil "tridenttools/storm/utils/trident" +) + +// Health-check rollback failure log location and the messages the +// health-checks-install scenario is expected to produce. These strings are +// properties of that scenario's Host Configuration health checks (scripts +// `invoke-rollback-from-script` referencing two non-existent services), mirrored +// from rollback_test.py. +const ( + healthCheckFailureLogGlob = "/var/lib/trident/trident-health-check-failure-*.log" + rollbackFailedHealthError = "Failed health check(s)" +) + +var expectedRollbackLogMessages = []string{ + "Script 'invoke-rollback-from-script' failed", + "Unit non-existent-service1.service could not be found", + "Unit non-existent-service2.service could not be found", +} + +// HasRollbackIntent reports whether the scenario is expected to trigger a +// health-check rollback, signalled by a top-level `health` section in the Host +// Configuration. Such scenarios replace `base` validation with rollback +// validation. +func HasRollbackIntent(hs tridentutil.HostStatus) bool { + return hs.Spec().Exists("health") +} + +// ValidateRollback ports rollback_test.py::test_rollback. It confirms the host +// reached the expected (rolled-back) servicing state, that the last error +// reflects a failed health check, that the active volume is unchanged (or +// absent when not provisioned), and that the health-check failure log records +// the expected script/service failures. +func ValidateRollback( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + expectedState tridentutil.ServicingState, + abActive tridentutil.AbVolumeSelection, +) { + sa.Assert("rollback/servicing-state", + hs.ServicingState() == expectedState, + "expected servicingState %q, got %q", expectedState, hs.ServicingState()) + + if lastErr, ok := hs.LastError(); ok { + sa.Assert("rollback/last-error", + strings.Contains(lastErr, rollbackFailedHealthError), + "lastError does not contain %q: %s", rollbackFailedHealthError, lastErr) + } else { + sa.Failf("rollback/last-error", "expected a lastError reflecting a failed health check") + } + + if expectedState == tridentutil.ServicingStateNotProvisioned { + if _, present := hs.AbActiveVolume(); present { + sa.Failf("rollback/active-volume", "abActiveVolume should be absent when not provisioned") + } + } else { + actual, present := hs.AbActiveVolume() + sa.Assert("rollback/active-volume", + present && actual == abActive, + "expected abActiveVolume %q, got %q (present=%v)", abActive, actual, present) + } + + validateRollbackLogs(sa, client) +} + +// validateRollbackLogs checks that exactly one health-check failure log exists +// and that it records the expected failure messages. +func validateRollbackLogs(sa *SoftAsserter, client *ssh.Client) { + listOut, err := sshutils.CommandOutput(client, "sudo ls "+healthCheckFailureLogGlob) + if err != nil { + sa.Fail("rollback/log-list", err) + return + } + + logFiles := strings.Fields(strings.TrimSpace(listOut)) + if len(logFiles) != 1 { + sa.Failf("rollback/log-count", "expected exactly 1 health-check failure log, found %d: %v", + len(logFiles), logFiles) + return + } + + content, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo cat %s", logFiles[0])) + if err != nil { + sa.Fail("rollback/log-read", err) + return + } + + for _, want := range expectedRollbackLogMessages { + sa.Assert("rollback/log-message", + strings.Contains(content, want), + "health-check failure log missing message %q", want) + } +} diff --git a/tools/storm/e2e/validate/rollback_test.go b/tools/storm/e2e/validate/rollback_test.go new file mode 100644 index 0000000000..dfa0f373a2 --- /dev/null +++ b/tools/storm/e2e/validate/rollback_test.go @@ -0,0 +1,67 @@ +package validate + +import ( + "strings" + "testing" + + tridentutil "tridenttools/storm/utils/trident" +) + +func TestHasRollbackIntent(t *testing.T) { + withHealth, _ := tridentutil.NewHostStatusFromYaml([]byte("spec:\n health:\n healthChecks: []\n")) + if !HasRollbackIntent(withHealth) { + t.Error("expected rollback intent when spec.health present") + } + withoutHealth, _ := tridentutil.NewHostStatusFromYaml([]byte("spec:\n storage: {}\n")) + if HasRollbackIntent(withoutHealth) { + t.Error("did not expect rollback intent without spec.health") + } +} + +// TestRollbackHostStatusChecks exercises the host-status portion of the rollback +// contract (state, absent active volume, health-check lastError) that +// ValidateRollback asserts; the log-file portion requires SSH and is covered by +// integration runs. +func TestRollbackHostStatusChecks(t *testing.T) { + hs, _ := tridentutil.NewHostStatusFromYaml([]byte( + "servicingState: not-provisioned\nlastError:\n message: Failed health check(s)\nspec:\n health: {}\n")) + + if hs.ServicingState() != tridentutil.ServicingStateNotProvisioned { + t.Errorf("state = %q, want not-provisioned", hs.ServicingState()) + } + if _, present := hs.AbActiveVolume(); present { + t.Error("abActiveVolume should be absent when not provisioned") + } + le, ok := hs.LastError() + if !ok || !strings.Contains(le, rollbackFailedHealthError) { + t.Errorf("lastError = %q, want it to contain %q", le, rollbackFailedHealthError) + } +} + +func TestValidateAbUpdateStaged(t *testing.T) { + hs, _ := tridentutil.NewHostStatusFromYaml([]byte( + "servicingState: ab-update-staged\nabActiveVolume: volume-a\n")) + var sa SoftAsserter + ValidateAbUpdateStaged(&sa, hs, tridentutil.AbVolumeA) + if sa.HasFailures() { + t.Errorf("expected no failures, got: %v", sa.Err()) + } + + // Wrong state -> failure. + bad, _ := tridentutil.NewHostStatusFromYaml([]byte( + "servicingState: provisioned\nabActiveVolume: volume-a\n")) + var sa2 SoftAsserter + ValidateAbUpdateStaged(&sa2, bad, tridentutil.AbVolumeA) + if !sa2.HasFailures() { + t.Error("expected failure for non-staged state") + } + + // Volume already flipped -> failure. + flipped, _ := tridentutil.NewHostStatusFromYaml([]byte( + "servicingState: ab-update-staged\nabActiveVolume: volume-b\n")) + var sa3 SoftAsserter + ValidateAbUpdateStaged(&sa3, flipped, tridentutil.AbVolumeA) + if !sa3.HasFailures() { + t.Error("expected failure when active volume already changed") + } +} diff --git a/tools/storm/e2e/validate/softassert.go b/tools/storm/e2e/validate/softassert.go new file mode 100644 index 0000000000..5dd701e902 --- /dev/null +++ b/tools/storm/e2e/validate/softassert.go @@ -0,0 +1,144 @@ +// Package validate holds the E2E host-state validation logic ported from the +// Python pytest suite (tests/e2e_tests/*.py). Validations run over SSH against +// the installed VM and assert that the real host state matches both the Host +// Configuration and what Trident reports in its Host Status. +package validate + +import ( + "errors" + "fmt" + "strings" + + "github.com/sirupsen/logrus" +) + +// SoftAsserter accumulates sub-check results within a single storm test case so +// that every sub-check runs (and is logged individually) even after an earlier +// one fails, instead of bailing on the first failure via runtime.Goexit(). At +// the end of the case, call Summary() to log every sub-check that ran and +// Err() to pass the combined failure to tc.FailFromError. +// +// This is the interim approach (stage 1) chosen while storm lacks native +// soft-assert / subtest support: all sub-checks are reported as a single test +// case rather than per-sub-check JUnit rows, but every sub-check (pass and +// fail) is recorded and surfaced through Summary() so the exact set of checks +// performed is visible even when the case passes. See the E2E storm-port plan +// for the deferred per-subtest reporting enhancement. +type SoftAsserter struct { + results []subCheck +} + +// subCheck is the outcome of a single named sub-check. A nil err means the +// sub-check passed. +type subCheck struct { + name string + err error +} + +// Check runs fn and records its outcome under name. fn is always executed; a +// failure never stops subsequent checks. +func (s *SoftAsserter) Check(name string, fn func() error) { + if err := fn(); err != nil { + s.record(name, err) + } else { + s.pass(name) + } +} + +// Pass records and logs a passing sub-check. Use it to make an explicitly +// verified condition visible in the summary when the check is not expressed via +// Check/Assert (e.g. after a guard-style branch). +func (s *SoftAsserter) Pass(name string) { + s.pass(name) +} + +// Passf records a passing sub-check and logs an accompanying informational +// message (e.g. diagnostic output that is surfaced but not treated as a +// failure). +func (s *SoftAsserter) Passf(name, format string, args ...any) { + logrus.Infof("validation sub-check passed (%s): "+format, append([]any{name}, args...)...) + s.results = append(s.results, subCheck{name: name}) +} + +// Fail records and logs a failure for the named sub-check. +func (s *SoftAsserter) Fail(name string, err error) { + s.record(name, err) +} + +// Failf records and logs a formatted failure for the named sub-check. +func (s *SoftAsserter) Failf(name, format string, args ...any) { + s.record(name, fmt.Errorf(format, args...)) +} + +// Assert records a pass if cond is true, otherwise a failure with the given +// message. +func (s *SoftAsserter) Assert(name string, cond bool, msgFormat string, args ...any) { + if cond { + s.pass(name) + } else { + s.record(name, fmt.Errorf(msgFormat, args...)) + } +} + +func (s *SoftAsserter) pass(name string) { + logrus.Debugf("validation sub-check passed: %s", name) + s.results = append(s.results, subCheck{name: name}) +} + +func (s *SoftAsserter) record(name string, err error) { + logrus.Errorf("validation sub-check failed: %s: %v", name, err) + s.results = append(s.results, subCheck{name: name, err: err}) +} + +// failures returns each failed sub-check as an error prefixed with its name. +func (s *SoftAsserter) failures() []error { + var errs []error + for _, r := range s.results { + if r.err != nil { + errs = append(errs, fmt.Errorf("%s: %w", r.name, r.err)) + } + } + return errs +} + +// HasFailures reports whether any sub-check has failed. +func (s *SoftAsserter) HasFailures() bool { + return len(s.failures()) > 0 +} + +// Failures returns the number of failed sub-checks. +func (s *SoftAsserter) Failures() int { + return len(s.failures()) +} + +// Summary returns a human-readable, ordered report of every sub-check that ran, +// each marked PASS or FAIL, prefixed with an aggregate count. It is intended to +// be logged at the end of a validation case so the exact set of checks +// performed is visible even when the case passes. +func (s *SoftAsserter) Summary() string { + if len(s.results) == 0 { + return "no validation sub-checks ran" + } + failed := len(s.failures()) + var b strings.Builder + fmt.Fprintf(&b, "%d validation sub-check(s): %d passed, %d failed", + len(s.results), len(s.results)-failed, failed) + for _, r := range s.results { + if r.err != nil { + fmt.Fprintf(&b, "\n FAIL %s: %v", r.name, r.err) + } else { + fmt.Fprintf(&b, "\n PASS %s", r.name) + } + } + return b.String() +} + +// Err returns the combined error of all failed sub-checks, or nil if none +// failed. The result is suitable to pass directly to tc.FailFromError. +func (s *SoftAsserter) Err() error { + errs := s.failures() + if len(errs) == 0 { + return nil + } + return fmt.Errorf("%d validation sub-check(s) failed: %w", len(errs), errors.Join(errs...)) +} diff --git a/tools/storm/e2e/validate/softassert_test.go b/tools/storm/e2e/validate/softassert_test.go new file mode 100644 index 0000000000..bb7a0ecf1c --- /dev/null +++ b/tools/storm/e2e/validate/softassert_test.go @@ -0,0 +1,81 @@ +package validate + +import ( + "errors" + "strings" + "testing" +) + +func TestSoftAsserter_AllPass(t *testing.T) { + var sa SoftAsserter + ran := 0 + sa.Check("a", func() error { ran++; return nil }) + sa.Check("b", func() error { ran++; return nil }) + sa.Assert("c", true, "should not fire") + + if ran != 2 { + t.Errorf("ran = %d, want 2", ran) + } + if sa.HasFailures() { + t.Error("HasFailures() = true, want false") + } + if sa.Err() != nil { + t.Errorf("Err() = %v, want nil", sa.Err()) + } +} + +// TestSoftAsserter_Summary verifies the ordered PASS/FAIL breakdown surfaced so +// a single validate case shows exactly which sub-checks ran. +func TestSoftAsserter_Summary(t *testing.T) { + var sa SoftAsserter + sa.Check("a", func() error { return nil }) + sa.Assert("b", false, "b broke") + sa.Pass("c") + + summary := sa.Summary() + for _, want := range []string{ + "3 validation sub-check(s): 2 passed, 1 failed", + "PASS a", + "FAIL b: b broke", + "PASS c", + } { + if !strings.Contains(summary, want) { + t.Errorf("Summary() = %q, missing %q", summary, want) + } + } + + var empty SoftAsserter + if got := empty.Summary(); got != "no validation sub-checks ran" { + t.Errorf("empty Summary() = %q", got) + } +} + +func TestSoftAsserter_ContinuesAfterFailure(t *testing.T) { + var sa SoftAsserter + ran := 0 + sa.Check("first", func() error { ran++; return errors.New("boom") }) + sa.Check("second", func() error { ran++; return nil }) // must still run + sa.Failf("third", "value %d bad", 7) + sa.Assert("fourth", false, "cond false") + + if ran != 2 { + t.Errorf("ran = %d, want 2 (both Check fns must execute)", ran) + } + if !sa.HasFailures() { + t.Fatal("HasFailures() = false, want true") + } + if sa.Failures() != 3 { + t.Errorf("Failures() = %d, want 3", sa.Failures()) + } + + err := sa.Err() + if err == nil { + t.Fatal("Err() = nil, want combined error") + } + msg := err.Error() + for _, want := range []string{"first: boom", "third: value 7 bad", "fourth: cond false", "3 validation sub-check(s) failed"} { + if !strings.Contains(msg, want) { + t.Errorf("Err() = %q, missing %q", msg, want) + } + } +} diff --git a/tools/storm/e2e/validate/staged.go b/tools/storm/e2e/validate/staged.go new file mode 100644 index 0000000000..89c37c8f8a --- /dev/null +++ b/tools/storm/e2e/validate/staged.go @@ -0,0 +1,25 @@ +package validate + +import ( + tridentutil "tridenttools/storm/utils/trident" +) + +// ValidateAbUpdateStaged ports ab_update_staged_test.py::test_ab_update_staged. +// It confirms that, after staging an A/B update but before finalizing, the host +// reports the staged servicing state and that the active volume has not yet +// changed. abActive is the volume expected to still be active before finalize. +func ValidateAbUpdateStaged( + sa *SoftAsserter, + hs tridentutil.HostStatus, + abActive tridentutil.AbVolumeSelection, +) { + sa.Assert("ab-staged/servicing-state", + hs.ServicingState() == tridentutil.ServicingStateAbUpdateStaged, + "expected servicingState %q, got %q", + tridentutil.ServicingStateAbUpdateStaged, hs.ServicingState()) + + actual, present := hs.AbActiveVolume() + sa.Assert("ab-staged/active-volume", + present && actual == abActive, + "expected abActiveVolume %q (unchanged), got %q (present=%v)", abActive, actual, present) +} diff --git a/tools/storm/e2e/validate/verity.go b/tools/storm/e2e/validate/verity.go new file mode 100644 index 0000000000..321c32483d --- /dev/null +++ b/tools/storm/e2e/validate/verity.go @@ -0,0 +1,227 @@ +package validate + +import ( + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/pkg/hostconfig" + "tridenttools/storm/utils/sysinspect" + tridentutil "tridenttools/storm/utils/trident" +) + +// VerityDevice describes a verity device declared in the Host Status spec. +type VerityDevice struct { + ID string + Name string + DataDeviceID string + HashDeviceID string +} + +// HasVerity reports whether the ROOT filesystem is backed by a verity device, +// used to self-select verity validation. This matches the scope of the ported +// verity_test.py::test_verity_root, which only validates root verity: configs +// where verity protects a non-root path (e.g. usr-verity) are intentionally not +// selected (they receive only base validation, as under pytest). +func HasVerity(hs tridentutil.HostStatus) bool { + spec := hs.Spec() + rootID, ok := RootFilesystemDeviceID(spec) + if !ok { + return false + } + _, ok = verityForRoot(spec, rootID) + return ok +} + +// verityForRoot returns the verity device whose id matches the root +// filesystem's device ID, and whether it was found. +func verityForRoot(spec hostconfig.HostConfig, rootDeviceID string) (VerityDevice, bool) { + for _, v := range spec.S("storage", "verity").Children() { + id, _ := v.S("id").Data().(string) + if id != rootDeviceID { + continue + } + dev := VerityDevice{ID: id} + dev.Name, _ = v.S("name").Data().(string) + dev.DataDeviceID, _ = v.S("dataDeviceId").Data().(string) + dev.HashDeviceID, _ = v.S("hashDeviceId").Data().(string) + return dev, true + } + return VerityDevice{}, false +} + +// ValidateVerity ports verity_test.py::test_verity_root. It confirms the root +// verity device mapper exists and is active/verified/read-only, then validates +// that its data and hash devices correspond to the expected block devices +// (accounting for A/B updates and RAID arrays). +func ValidateVerity( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + abActive tridentutil.AbVolumeSelection, +) { + spec := hs.Spec() + + blkid, err := sysinspect.Blkid(client) + if err != nil { + sa.Fail("verity/blkid", err) + return + } + if !blkidHasPath(blkid, "/dev/mapper/root") { + sa.Failf("verity/mapper", "/dev/mapper/root not present in blkid output") + } + + // Locate the root verity device from the Host Status. + rootDeviceID, ok := RootFilesystemDeviceID(spec) + if !ok { + sa.Failf("verity/root-mount", "root mount point not found in Host Status spec") + return + } + verity, ok := verityForRoot(spec, rootDeviceID) + if !ok || verity.HashDeviceID == "" { + sa.Failf("verity/config", "no verity configuration found for root device %q", rootDeviceID) + return + } + + name := verity.Name + if name == "" { + name = "root" + } + + // veritysetup status must show an active, verified, read-only device. + status, err := sysinspect.VeritySetup(client, name) + if err != nil { + sa.Fail("verity/status", err) + return + } + sa.Assert("verity/active", status.Active, "verity device %q is not active/in-use", name) + assertVerityField(sa, status, "type", "VERITY") + assertVerityField(sa, status, "status", "verified") + assertVerityField(sa, status, "mode", "readonly") + + dataDevice, dataOk := status.Get("data device") + hashDevice, hashOk := status.Get("hash device") + if !dataOk || !hashOk { + sa.Failf("verity/devices", "veritysetup status missing data/hash device fields") + return + } + + if spec.HasABUpdate() { + validateVerityAbDevices(sa, client, spec, blkid, verity, dataDevice, hashDevice, abActive) + } else { + validateVerityNonAbDevices(sa, client, blkid, verity, dataDevice, hashDevice) + } +} + +// validateVerityAbDevices validates the A/B branch: the veritysetup data/hash +// devices must correspond to the active volume of the data/hash A/B pairs. +func validateVerityAbDevices( + sa *SoftAsserter, + client *ssh.Client, + spec hostconfig.HostConfig, + blkid map[string]sysinspect.BlkidEntry, + verity VerityDevice, + dataDevice, hashDevice string, + abActive tridentutil.AbVolumeSelection, +) { + activeDataID, dataFound := ActiveVolumeID(spec, verity.DataDeviceID, abActive) + activeHashID, hashFound := ActiveVolumeID(spec, verity.HashDeviceID, abActive) + if !dataFound || !hashFound { + sa.Failf("verity/ab-pair", "could not resolve active data/hash volume for %s", abActive) + return + } + + dataRaid, _, _ := sysinspect.RaidNameForDevice(client, dataDevice) + hashRaid, _, _ := sysinspect.RaidNameForDevice(client, hashDevice) + dataIsRaid := dataRaid != "" + hashIsRaid := hashRaid != "" + if dataIsRaid != hashIsRaid { + sa.Failf("verity/ab-raid-parity", + "data/hash RAID mismatch: data raid=%v hash raid=%v", dataIsRaid, hashIsRaid) + return + } + + if dataIsRaid { + sa.Assert("verity/ab-data-raid", basename(dataRaid) == activeDataID, + "active data volume %q != raid %q", activeDataID, basename(dataRaid)) + sa.Assert("verity/ab-hash-raid", basename(hashRaid) == activeHashID, + "active hash volume %q != raid %q", activeHashID, basename(hashRaid)) + return + } + + // Partition case: PARTLABEL of the block device must equal the active ID. + assertPartlabelEquals(sa, blkid, dataDevice, activeDataID, "verity/ab-data-partlabel") + assertPartlabelEquals(sa, blkid, hashDevice, activeHashID, "verity/ab-hash-partlabel") +} + +// validateVerityNonAbDevices validates the non-A/B branch: the veritysetup +// data/hash devices must correspond to the configured data/hash device IDs. +func validateVerityNonAbDevices( + sa *SoftAsserter, + client *ssh.Client, + blkid map[string]sysinspect.BlkidEntry, + verity VerityDevice, + dataDevice, hashDevice string, +) { + dataRaid, _, _ := sysinspect.RaidNameForDevice(client, dataDevice) + hashRaid, _, _ := sysinspect.RaidNameForDevice(client, hashDevice) + dataIsRaid := dataRaid != "" + hashIsRaid := hashRaid != "" + if dataIsRaid != hashIsRaid { + sa.Failf("verity/raid-parity", + "data/hash RAID mismatch: data raid=%v hash raid=%v", dataIsRaid, hashIsRaid) + return + } + + if dataIsRaid { + sa.Assert("verity/data-raid", basename(dataRaid) == verity.DataDeviceID, + "data device id %q != raid %q", verity.DataDeviceID, basename(dataRaid)) + sa.Assert("verity/hash-raid", basename(hashRaid) == verity.HashDeviceID, + "hash device id %q != raid %q", verity.HashDeviceID, basename(hashRaid)) + return + } + + // Partition case: base_test's non-A/B branch only asserts presence in blkid. + if !blkidHasDevice(blkid, basename(dataDevice)) { + sa.Failf("verity/data-present", "verity data device %q not present in blkid", dataDevice) + } + if !blkidHasDevice(blkid, basename(hashDevice)) { + sa.Failf("verity/hash-present", "verity hash device %q not present in blkid", hashDevice) + } +} + +func assertVerityField(sa *SoftAsserter, status sysinspect.VeritySetupStatus, field, want string) { + got, ok := status.Get(field) + sa.Assert("verity/"+field, ok && got == want, + "veritysetup %s = %q, want %q", field, got, want) +} + +func assertPartlabelEquals(sa *SoftAsserter, blkid map[string]sysinspect.BlkidEntry, devicePath, wantLabel, checkName string) { + entry, ok := blkid[basename(devicePath)] + if !ok { + sa.Failf(checkName, "device %q not present in blkid", devicePath) + return + } + label, ok := entry.Get("PARTLABEL") + sa.Assert(checkName, ok && label == wantLabel, + "device %q PARTLABEL = %q, want %q", devicePath, label, wantLabel) +} + +func blkidHasDevice(blkid map[string]sysinspect.BlkidEntry, deviceBasename string) bool { + _, ok := blkid[deviceBasename] + return ok +} + +// blkidHasPath reports whether any blkid entry has the given full device path. +func blkidHasPath(blkid map[string]sysinspect.BlkidEntry, path string) bool { + for _, entry := range blkid { + if entry.Path == path { + return true + } + } + return false +} + +func basename(path string) string { + return path[strings.LastIndex(path, "/")+1:] +} diff --git a/tools/storm/e2e/validate/verity_test.go b/tools/storm/e2e/validate/verity_test.go new file mode 100644 index 0000000000..525ef954b6 --- /dev/null +++ b/tools/storm/e2e/validate/verity_test.go @@ -0,0 +1,76 @@ +package validate + +import ( + "testing" + + tridentutil "tridenttools/storm/utils/trident" +) + +const verityStatusYaml = ` +servicingState: provisioned +spec: + storage: + verity: + - id: root + name: root + dataDeviceId: root-data + hashDeviceId: root-hash + filesystems: + - deviceId: root + mountPoint: / +` + +func TestHasVerity(t *testing.T) { + // Root is a verity device -> selected. + hs, _ := tridentutil.NewHostStatusFromYaml([]byte(verityStatusYaml)) + if !HasVerity(hs) { + t.Error("expected HasVerity=true when root is a verity device") + } + // No verity at all -> not selected. + plain, _ := tridentutil.NewHostStatusFromYaml([]byte("spec:\n storage:\n disks: []\n")) + if HasVerity(plain) { + t.Error("expected HasVerity=false without verity") + } + // usr-verity: verity exists but protects /usr, not root -> NOT selected + // (root-verity validation must not run and false-fail). + usrVerity, _ := tridentutil.NewHostStatusFromYaml([]byte(` +spec: + storage: + verity: + - id: usr + name: usr + dataDeviceId: usr-data + hashDeviceId: usr-hash + filesystems: + - deviceId: root + mountPoint: / + - deviceId: usr + mountPoint: /usr +`)) + if HasVerity(usrVerity) { + t.Error("expected HasVerity=false for usr-verity (root is not a verity device)") + } +} + +func TestVerityForRoot(t *testing.T) { + hs, _ := tridentutil.NewHostStatusFromYaml([]byte(verityStatusYaml)) + dev, ok := verityForRoot(hs.Spec(), "root") + if !ok { + t.Fatal("verityForRoot not found") + } + if dev.Name != "root" || dev.DataDeviceID != "root-data" || dev.HashDeviceID != "root-hash" { + t.Errorf("verity device = %+v", dev) + } + if _, ok := verityForRoot(hs.Spec(), "nonexistent"); ok { + t.Error("expected not found for nonexistent root") + } +} + +func TestBasename(t *testing.T) { + if basename("/dev/md/root-a") != "root-a" { + t.Error("basename /dev/md/root-a") + } + if basename("sda1") != "sda1" { + t.Error("basename sda1") + } +} diff --git a/tools/storm/rollback/tests/prepare_extensions.go b/tools/storm/rollback/tests/prepare_extensions.go new file mode 100644 index 0000000000..91a361bda8 --- /dev/null +++ b/tools/storm/rollback/tests/prepare_extensions.go @@ -0,0 +1,75 @@ +package tests + +import ( + "fmt" + "os" + "path/filepath" + + stormrollbackconfig "tridenttools/storm/rollback/utils/config" + buildextensions "tridenttools/storm/scripts/build_extension_images" + stormvmconfig "tridenttools/storm/utils/vm/config" + + "github.com/sirupsen/logrus" +) + +// sysextCloneCount is the number of test sysext images the rollback flow needs. +// The extension update test walks through the clones one version at a time. +const sysextCloneCount = 3 + +// PrepareExtensions makes sure the test sysext images the rollback flow needs +// exist in the artifacts directory. +// +// These used to be produced by a pipeline step that ran the +// build-extension-images script and moved the results into place, which meant a +// local run silently failed later with "failed to find extension file". Owning +// it here keeps the scenario self-sufficient on a dev box, and skips the work +// entirely for flavors that do not exercise extensions. +func PrepareExtensions(testConfig stormrollbackconfig.TestConfig, vmConfig stormvmconfig.AllVMConfig) error { + if testConfig.SkipExtensionTesting { + logrus.Infof("Skipping extension image preparation since SkipExtensionTesting is set") + return nil + } + + if err := os.MkdirAll(testConfig.ArtifactsDir, 0755); err != nil { + return fmt.Errorf("failed to create artifacts directory %s: %w", testConfig.ArtifactsDir, err) + } + + if missing, err := missingSysexts(testConfig); err != nil { + return err + } else if len(missing) == 0 { + logrus.Infof("All %d test sysext images already present in %s", sysextCloneCount, testConfig.ArtifactsDir) + return nil + } else { + logrus.Infof("Building %d test sysext images in %s (missing: %v)", sysextCloneCount, testConfig.ArtifactsDir, missing) + } + + if err := buildextensions.BuildSysextImages(testConfig.ArtifactsDir, sysextCloneCount); err != nil { + return fmt.Errorf("failed to build test sysext images: %w", err) + } + + // A partial build would otherwise surface much later as a confusing + // "failed to find extension file" during qcow2 preparation. + if missing, err := missingSysexts(testConfig); err != nil { + return err + } else if len(missing) > 0 { + return fmt.Errorf("test sysext images still missing after build: %v", missing) + } + + return nil +} + +// missingSysexts returns the sysext image paths that are not yet present. +func missingSysexts(testConfig stormrollbackconfig.TestConfig) ([]string, error) { + var missing []string + for i := 1; i <= sysextCloneCount; i++ { + path := filepath.Join(testConfig.ArtifactsDir, fmt.Sprintf("%s-%d.raw", testConfig.ExtensionName, i)) + switch _, err := os.Stat(path); { + case err == nil: + case os.IsNotExist(err): + missing = append(missing, path) + default: + return nil, fmt.Errorf("failed to stat %s: %w", path, err) + } + } + return missing, nil +} diff --git a/tools/storm/rollback/trident.go b/tools/storm/rollback/trident.go index ed2c460c44..d974521793 100644 --- a/tools/storm/rollback/trident.go +++ b/tools/storm/rollback/trident.go @@ -1,7 +1,6 @@ package rollback import ( - "fmt" "os" "path/filepath" @@ -25,7 +24,6 @@ type TridentRollbackScenarioArgs struct { stormvmconfig.VMConfig `embed:""` stormvmqemu.QemuConfig `embed:""` stormvmazure.AzureConfig `embed:""` - TestCaseToRun string `help:"Name of the test case to run. If not specified, all test cases will be run." default:"all"` } func (s *TridentRollbackScenario) Name() string { @@ -45,6 +43,7 @@ func (s *TridentRollbackScenario) StagePaths() []string { } func (s *TridentRollbackScenario) RegisterTestCases(r storm.TestRegistrar) error { + r.RegisterTestCase("prepare-extensions", s.prepareExtensions) r.RegisterTestCase("prepare-qcow2", s.prepareQcow2) r.RegisterTestCase("deploy-vm", s.deployVm) r.RegisterTestCase("check-deployment", s.checkDeployment) @@ -60,7 +59,27 @@ func (s *TridentRollbackScenario) RequiredFiles() []string { return nil } -func (s TridentRollbackScenario) Setup(ctx storm.SetupCleanupContext) error { +func (s *TridentRollbackScenario) Setup(ctx storm.SetupCleanupContext) error { + profile, err := s.args.TestConfig.ApplyFlavor() + if err != nil { + return err + } + + // A flavor that cannot enroll the test signing keys into firmware must not + // run with secure boot, even when the caller asked for it. + s.args.QemuConfig.SecureBoot = s.args.QemuConfig.SecureBoot && profile.SupportsSecureBoot + + logrus.Infof( + "Rollback flavor %q: uki=%t secure-boot=%t skip-extensions=%t skip-runtime-updates=%t skip-netplan=%t skip-manual-rollbacks=%t", + s.args.TestConfig.Flavor, + s.args.TestConfig.Uki, + s.args.QemuConfig.SecureBoot, + s.args.TestConfig.SkipExtensionTesting, + s.args.TestConfig.SkipRuntimeUpdates, + s.args.TestConfig.SkipNetplanRuntimeTesting, + s.args.TestConfig.SkipManualRollbacks, + ) + return nil } @@ -79,34 +98,33 @@ func (s *TridentRollbackScenario) Cleanup(ctx storm.SetupCleanupContext) error { } func (s *TridentRollbackScenario) runTestCase(tc storm.TestCase, testFunc func(stormrollbackconfig.TestConfig, stormvmconfig.AllVMConfig) error) error { - if tc.Name() != s.args.TestCaseToRun && s.args.TestCaseToRun != "all" { - tc.Skip(fmt.Sprintf("Test case '%s' does not align to TestCaseToRun '%s'", tc.Name(), s.args.TestCaseToRun)) - } else { - logrus.Infof("Running test case '%s'", tc.Name()) - // create test-specific output directory - testCaseSpecificConfig := s.args.TestConfig - testCaseSpecificConfig.OutputPath = s.args.TestConfig.OutputPath - if testCaseSpecificConfig.OutputPath != "" { - testCaseSpecificConfig.OutputPath = filepath.Join(testCaseSpecificConfig.OutputPath, tc.Name()) - if err := os.MkdirAll(testCaseSpecificConfig.OutputPath, 0755); err != nil { - tc.FailFromError(err) - } - } - err := testFunc( - testCaseSpecificConfig, - stormvmconfig.AllVMConfig{ - VMConfig: s.args.VMConfig, - QemuConfig: s.args.QemuConfig, - AzureConfig: s.args.AzureConfig, - }) - if err != nil { - logrus.Infof("test case '%s' failed", tc.Name()) + logrus.Infof("Running test case '%s'", tc.Name()) + // create test-specific output directory + testCaseSpecificConfig := s.args.TestConfig + testCaseSpecificConfig.OutputPath = s.args.TestConfig.OutputPath + if testCaseSpecificConfig.OutputPath != "" { + testCaseSpecificConfig.OutputPath = filepath.Join(testCaseSpecificConfig.OutputPath, tc.Name()) + if err := os.MkdirAll(testCaseSpecificConfig.OutputPath, 0755); err != nil { tc.FailFromError(err) } - logrus.Infof("test case '%s' passed", tc.Name()) } + err := testFunc( + testCaseSpecificConfig, + stormvmconfig.AllVMConfig{ + VMConfig: s.args.VMConfig, + QemuConfig: s.args.QemuConfig, + AzureConfig: s.args.AzureConfig, + }) + if err != nil { + logrus.Infof("test case '%s' failed", tc.Name()) + tc.FailFromError(err) + } + logrus.Infof("test case '%s' passed", tc.Name()) return nil +} +func (s *TridentRollbackScenario) prepareExtensions(tc storm.TestCase) error { + return s.runTestCase(tc, stormrollbacktests.PrepareExtensions) } func (s *TridentRollbackScenario) prepareQcow2(tc storm.TestCase) error { diff --git a/tools/storm/rollback/utils/config/config.go b/tools/storm/rollback/utils/config/config.go index 726dcb721a..8a04473068 100644 --- a/tools/storm/rollback/utils/config/config.go +++ b/tools/storm/rollback/utils/config/config.go @@ -4,6 +4,7 @@ type TestConfig struct { ArtifactsDir string `help:"Directory containing artifacts for the VM" default:"/tmp"` OutputPath string `help:"Path to the output directory for logs and artifacts" default:"./output"` Verbose bool `help:"Enable verbose logging" default:"false"` + Flavor string `help:"Image flavor under test. Determines which sub-tests run, whether the image is UKI, and whether secure boot is supported." enum:"qemu-grub,qemu,uki" default:"qemu-grub"` HostConfig string `help:"Host Configuration to use for updates" default:"./input/trident.yaml"` ExtensionName string `help:"Extension Name to test" default:"test-sysext"` FileServerPort int `help:"Port for the cosi and extension file server" default:"8000"` diff --git a/tools/storm/rollback/utils/config/flavor.go b/tools/storm/rollback/utils/config/flavor.go new file mode 100644 index 0000000000..5c93f86214 --- /dev/null +++ b/tools/storm/rollback/utils/config/flavor.go @@ -0,0 +1,80 @@ +package config + +import "fmt" + +// Flavor identifies the image variant under test. +// +// The pipeline previously derived the skip/uki/secure-boot flags from this +// value with a chain of bash conditionals spread across two YAML templates. +// The mapping lives here instead, so a single --flavor argument fully +// determines the test profile and the same profile applies to local runs. +type Flavor string + +const ( + FlavorQemuGrub Flavor = "qemu-grub" + FlavorQemu Flavor = "qemu" + FlavorUki Flavor = "uki" +) + +// FlavorProfile is the set of test behaviors implied by a Flavor. +type FlavorProfile struct { + SkipExtensionTesting bool + SkipRuntimeUpdates bool + SkipNetplanRuntimeTesting bool + Uki bool + + // SupportsSecureBoot reports whether secure boot may be enabled for this + // flavor. Note this is deliberately not the same as !Uki: it mirrors the + // pipeline's long-standing behavior of gating secure boot on the flavor + // name alone, so the "qemu" flavor keeps secure boot even though it sets + // Uki. Preserved as-is to avoid changing what CI exercises. + SupportsSecureBoot bool +} + +// Profile returns the behaviors implied by f, or an error if f is unknown. +func (f Flavor) Profile() (FlavorProfile, error) { + switch f { + case FlavorQemuGrub: + return FlavorProfile{ + SupportsSecureBoot: true, + }, nil + + case FlavorQemu: + // Root-verity image: extensions cannot be added to the qcow2 and the + // netplan config cannot be modified on a verity rootfs, so a runtime + // update would have nothing to service. + return FlavorProfile{ + SkipExtensionTesting: true, + SkipRuntimeUpdates: true, + SkipNetplanRuntimeTesting: true, + Uki: true, + SupportsSecureBoot: true, + }, nil + + case FlavorUki: + return FlavorProfile{ + Uki: true, + }, nil + } + + return FlavorProfile{}, fmt.Errorf("unknown image flavor %q", f) +} + +// ApplyFlavor folds the flavor profile into c and returns the profile. +// +// Skips are additive: a flavor may only add a skip, never clear one the caller +// asked for explicitly. This keeps flags like --skip-manual-rollbacks usable on +// top of any flavor. +func (c *TestConfig) ApplyFlavor() (FlavorProfile, error) { + profile, err := Flavor(c.Flavor).Profile() + if err != nil { + return FlavorProfile{}, err + } + + c.SkipExtensionTesting = c.SkipExtensionTesting || profile.SkipExtensionTesting + c.SkipRuntimeUpdates = c.SkipRuntimeUpdates || profile.SkipRuntimeUpdates + c.SkipNetplanRuntimeTesting = c.SkipNetplanRuntimeTesting || profile.SkipNetplanRuntimeTesting + c.Uki = c.Uki || profile.Uki + + return profile, nil +} diff --git a/tools/storm/rollback/utils/config/flavor_test.go b/tools/storm/rollback/utils/config/flavor_test.go new file mode 100644 index 0000000000..1a21f421da --- /dev/null +++ b/tools/storm/rollback/utils/config/flavor_test.go @@ -0,0 +1,98 @@ +package config + +import "testing" + +// The expectations below mirror what the pipeline templates passed before the +// flavor mapping moved into Go, so a regression here means CI would start +// exercising a different set of tests than it does today. +func TestFlavorProfile(t *testing.T) { + tests := []struct { + flavor Flavor + want FlavorProfile + }{ + { + flavor: FlavorQemuGrub, + want: FlavorProfile{ + SupportsSecureBoot: true, + }, + }, + { + flavor: FlavorQemu, + want: FlavorProfile{ + SkipExtensionTesting: true, + SkipRuntimeUpdates: true, + SkipNetplanRuntimeTesting: true, + Uki: true, + SupportsSecureBoot: true, + }, + }, + { + flavor: FlavorUki, + want: FlavorProfile{ + Uki: true, + }, + }, + } + + for _, tt := range tests { + t.Run(string(tt.flavor), func(t *testing.T) { + got, err := tt.flavor.Profile() + if err != nil { + t.Fatalf("Profile() returned error: %v", err) + } + if got != tt.want { + t.Errorf("Profile() = %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestFlavorProfileUnknown(t *testing.T) { + if _, err := Flavor("nonsense").Profile(); err == nil { + t.Fatal("Profile() accepted an unknown flavor, want error") + } +} + +func TestApplyFlavorSetsSkips(t *testing.T) { + cfg := TestConfig{Flavor: string(FlavorQemu)} + + if _, err := cfg.ApplyFlavor(); err != nil { + t.Fatalf("ApplyFlavor() returned error: %v", err) + } + + if !cfg.SkipExtensionTesting || !cfg.SkipRuntimeUpdates || !cfg.SkipNetplanRuntimeTesting || !cfg.Uki { + t.Errorf("ApplyFlavor() did not apply the qemu profile: %+v", cfg) + } +} + +// A flavor may add skips but must never clear one the caller asked for, so +// flags like --skip-manual-rollbacks stay usable on top of any flavor. +func TestApplyFlavorIsAdditive(t *testing.T) { + cfg := TestConfig{ + Flavor: string(FlavorQemuGrub), + SkipExtensionTesting: true, + SkipManualRollbacks: true, + } + + if _, err := cfg.ApplyFlavor(); err != nil { + t.Fatalf("ApplyFlavor() returned error: %v", err) + } + + if !cfg.SkipExtensionTesting { + t.Error("ApplyFlavor() cleared an explicitly requested SkipExtensionTesting") + } + if !cfg.SkipManualRollbacks { + t.Error("ApplyFlavor() cleared an explicitly requested SkipManualRollbacks") + } + if cfg.Uki { + t.Error("ApplyFlavor() set Uki for the qemu-grub flavor") + } +} + +func TestApplyFlavorUnknownFails(t *testing.T) { + cfg := TestConfig{Flavor: "nonsense"} + + if _, err := cfg.ApplyFlavor(); err == nil { + t.Fatal("ApplyFlavor() accepted an unknown flavor, want error") + } +} diff --git a/tools/storm/scripts/build_extension_images/build_extension_images.go b/tools/storm/scripts/build_extension_images/build_extension_images.go index 91d7d2ea88..6b579b710c 100644 --- a/tools/storm/scripts/build_extension_images/build_extension_images.go +++ b/tools/storm/scripts/build_extension_images/build_extension_images.go @@ -26,13 +26,13 @@ func (s *BuildExtensionImagesScript) Run() error { } if s.BuildSysexts { - err := buildImage("sysext", s.NumClones) + err := buildImage("sysext", s.NumClones, ".") if err != nil { return fmt.Errorf("failed to build sysext images: %w", err) } } if s.BuildConfexts { - err := buildImage("confext", s.NumClones) + err := buildImage("confext", s.NumClones, ".") if err != nil { return fmt.Errorf("failed to build confext images: %w", err) } @@ -56,22 +56,38 @@ func (s *BuildExtensionImagesScript) Run() error { return nil } -func buildImage(extType string, numClones int) error { +// BuildSysextImages builds numClones test sysext images into outputDir. +// +// Exposed so scenarios can provision their own extension images instead of +// depending on a pipeline step having built them and moved them into place, +// which is what makes a local dev-box run possible. +func BuildSysextImages(outputDir string, numClones int) error { + return buildImage("sysext", numClones, outputDir) +} + +// buildImage writes images into outputDir. The intermediate +// directory tree is staged in a temp dir so it does not litter outputDir. +func buildImage(extType string, numClones int, outputDir string) error { + stagingRoot, err := os.MkdirTemp("", "storm-extension-build-") + if err != nil { + return fmt.Errorf("failed to create staging directory: %w", err) + } + defer os.RemoveAll(stagingRoot) + for i := 1; i <= numClones; i++ { extName := fmt.Sprintf("test-%s-%d", extType, i) // Create extension-release file var dir string var fileContent string - var err error if extType == "sysext" { - dir = fmt.Sprintf("%s-image-%d/usr/lib/extension-release.d", extType, i) + dir = filepath.Join(stagingRoot, fmt.Sprintf("%s-image-%d", extType, i), "usr/lib/extension-release.d") err = os.MkdirAll(dir, 0755) if err != nil { return fmt.Errorf("failed to create sysext directory %s: %w", dir, err) } fileContent = fmt.Sprintf("ID=_any\nSYSEXT_ID=test-sysext\nSYSEXT_VERSION_ID=%d.0.0\nARCHITECTURE=x86-64\n", i) } else { - dir = fmt.Sprintf("%s-image-%d/etc/extension-release.d", extType, i) + dir = filepath.Join(stagingRoot, fmt.Sprintf("%s-image-%d", extType, i), "etc/extension-release.d") err = os.MkdirAll(dir, 0755) if err != nil { return fmt.Errorf("failed to create confext directory %s: %w", dir, err) @@ -86,7 +102,7 @@ func buildImage(extType string, numClones int) error { if extType == "sysext" { // Create script that outputs version - binDir := fmt.Sprintf("%s-image-%d/usr/bin", extType, i) + binDir := filepath.Join(stagingRoot, fmt.Sprintf("%s-image-%d", extType, i), "usr/bin") err := os.MkdirAll(binDir, 0755) if err != nil { return fmt.Errorf("failed to create sysext directory %s: %w", binDir, err) @@ -103,12 +119,11 @@ func buildImage(extType string, numClones int) error { } // Create DDI files using mksquashfs - imageDir := fmt.Sprintf("%s-image-%d", extType, i) - rawFile := fmt.Sprintf("%s.raw", extName) + imageDir := filepath.Join(stagingRoot, fmt.Sprintf("%s-image-%d", extType, i)) + rawFile := filepath.Join(outputDir, fmt.Sprintf("%s.raw", extName)) cmd := exec.Command("mksquashfs", imageDir, rawFile, "-comp", "xz", "-Xbcj", "x86", "-noappend", "-no-xattrs") - err = cmd.Run() - if err != nil { - return fmt.Errorf("failed to create raw file %s: %w", rawFile, err) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create raw file %s: %w: %s", rawFile, err, string(output)) } } return nil diff --git a/tools/storm/utils/sysinspect/blkid.go b/tools/storm/utils/sysinspect/blkid.go new file mode 100644 index 0000000000..a38185bf95 --- /dev/null +++ b/tools/storm/utils/sysinspect/blkid.go @@ -0,0 +1,121 @@ +// Package sysinspect provides parsers that gather structured host state from a +// running system over SSH. Each helper runs a standard inspection tool (blkid, +// lsblk, mount, cryptsetup, veritysetup, ...) and returns typed data for E2E +// validations. These replace the ad-hoc string parsing previously done in the +// Python E2E suite (tests/e2e_tests/*.py). +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// BlkidEntry holds the tag=value fields blkid reports for a single block +// device, e.g. UUID, TYPE, PARTLABEL, PARTUUID, LABEL, BLOCK_SIZE. +type BlkidEntry struct { + // Device is the kernel device name (e.g. "sda1"), i.e. the basename of the + // device path reported by blkid. + Device string + // Path is the full device path reported by blkid (e.g. "/dev/sda1", + // "/dev/mapper/root"). + Path string + // Fields holds the parsed tag=value pairs (quotes stripped). + Fields map[string]string +} + +// Get returns the value of a blkid field and whether it was present. +func (e BlkidEntry) Get(field string) (string, bool) { + v, ok := e.Fields[field] + return v, ok +} + +// Blkid runs `sudo blkid` on the host and returns the parsed entries keyed by +// kernel device name (basename of the device path). +// +// Example line: +// +// /dev/sda2: UUID="04267584-..." BLOCK_SIZE="4096" TYPE="ext4" PARTLABEL="root-a" PARTUUID="f1be3a27-..." +func Blkid(client *ssh.Client) (map[string]BlkidEntry, error) { + out, err := sshutils.CommandOutput(client, "sudo blkid") + if err != nil { + return nil, fmt.Errorf("failed to run blkid: %w", err) + } + return ParseBlkid(out), nil +} + +// ParseBlkid parses the stdout of `blkid` into entries keyed by kernel device +// name. It is separated from Blkid so it can be unit-tested without SSH. +func ParseBlkid(stdout string) map[string]BlkidEntry { + entries := make(map[string]BlkidEntry) + + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // Split "device: field=val field=val ..." into device and the rest. + devicePart, rest, found := strings.Cut(line, ": ") + if !found { + continue + } + + device := devicePart[strings.LastIndex(devicePart, "/")+1:] + entry := BlkidEntry{Device: device, Path: devicePart, Fields: make(map[string]string)} + + for _, field := range strings.Fields(rest) { + key, value, ok := strings.Cut(field, "=") + if !ok { + continue + } + entry.Fields[key] = strings.Trim(value, "\"") + } + + entries[device] = entry + } + + return entries +} + +// BlkidExport runs `sudo blkid --output export` and returns a map from full +// device path (DEVNAME) to its properties. This export format groups each +// device's fields as `KEY=value` lines separated by blank lines, and is used by +// the encryption validation (mirrors encryption_test.py::get_blkid_output). +func BlkidExport(client *ssh.Client) (map[string]map[string]string, error) { + out, err := sshutils.CommandOutput(client, "sudo blkid --output export") + if err != nil { + return nil, fmt.Errorf("failed to run blkid --output export: %w", err) + } + return ParseBlkidExport(out), nil +} + +// ParseBlkidExport parses `blkid --output export` output into a map keyed by +// DEVNAME (full device path). +func ParseBlkidExport(stdout string) map[string]map[string]string { + devices := make(map[string]map[string]string) + var current string + + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if line == "" { + current = "" + continue + } + key, value, ok := strings.Cut(line, "=") + if !ok { + continue + } + if key == "DEVNAME" { + current = value + devices[current] = make(map[string]string) + } else if current != "" { + devices[current][key] = value + } + } + + return devices +} diff --git a/tools/storm/utils/sysinspect/cryptsetup.go b/tools/storm/utils/sysinspect/cryptsetup.go new file mode 100644 index 0000000000..1473441090 --- /dev/null +++ b/tools/storm/utils/sysinspect/cryptsetup.go @@ -0,0 +1,125 @@ +package sysinspect + +import ( + "encoding/json" + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// CryptsetupStatus is the parsed output of `cryptsetup status `. +type CryptsetupStatus struct { + // Active is true when the device is active (LUKS2 volumes are always open + // and thus active). + Active bool + // InUse is true when the first line reports the device is "active and is in + // use" (mounted); false when it is merely "active". + InUse bool + Fields map[string]string +} + +// Get returns a status field value and whether it was present. +func (s CryptsetupStatus) Get(field string) (string, bool) { + v, ok := s.Fields[field] + return v, ok +} + +// Cryptsetup runs `sudo cryptsetup status ` and returns the parsed status. +func Cryptsetup(client *ssh.Client, name string) (CryptsetupStatus, error) { + out, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo cryptsetup status %s", name)) + if err != nil { + return CryptsetupStatus{}, fmt.Errorf("failed to run cryptsetup status %s: %w", name, err) + } + return ParseCryptsetupStatus(out, name), nil +} + +// ParseCryptsetupStatus parses `cryptsetup status ` output. The first line +// is the header (" is active[ and is in use]."); subsequent lines are +// "key: value" pairs. +func ParseCryptsetupStatus(stdout, name string) CryptsetupStatus { + status := CryptsetupStatus{Fields: make(map[string]string)} + + lines := strings.Split(strings.TrimSpace(stdout), "\n") + if len(lines) == 0 { + return status + } + + header := strings.TrimSpace(lines[0]) + inUseHeader := fmt.Sprintf("/dev/mapper/%s is active and is in use.", name) + activeHeader := fmt.Sprintf("/dev/mapper/%s is active.", name) + switch header { + case inUseHeader: + status.Active = true + status.InUse = true + case activeHeader: + status.Active = true + } + + for _, line := range lines[1:] { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + if key != "" { + status.Fields[key] = value + } + } + + return status +} + +// LuksDump is the subset of `cryptsetup luksDump --dump-json-metadata` output +// that the encryption validation inspects. +type LuksDump struct { + Keyslots map[string]LuksKeyslot `json:"keyslots"` + Tokens map[string]LuksToken `json:"tokens"` + Digests map[string]LuksDigest `json:"digests"` +} + +type LuksKeyslot struct { + Type string `json:"type"` + Kdf struct { + Type string `json:"type"` + Hash string `json:"hash"` + } `json:"kdf"` + Area struct { + Encryption string `json:"encryption"` + } `json:"area"` +} + +type LuksToken struct { + Type string `json:"type"` + Keyslots []string `json:"keyslots"` + Tpm2Pcrlock bool `json:"tpm2_pcrlock"` + Tpm2Pcrs []int `json:"tpm2-pcrs"` +} + +type LuksDigest struct { + Type string `json:"type"` + Hash string `json:"hash"` +} + +// CryptsetupLuksDump runs `cryptsetup luksDump --dump-json-metadata ` and +// returns the parsed metadata. +func CryptsetupLuksDump(client *ssh.Client, devicePath string) (LuksDump, error) { + out, err := sshutils.CommandOutput(client, + fmt.Sprintf("sudo cryptsetup luksDump --dump-json-metadata %s", devicePath)) + if err != nil { + return LuksDump{}, fmt.Errorf("failed to run cryptsetup luksDump %s: %w", devicePath, err) + } + return ParseLuksDump(out) +} + +// ParseLuksDump parses cryptsetup luksDump JSON metadata. +func ParseLuksDump(stdout string) (LuksDump, error) { + var dump LuksDump + if err := json.Unmarshal([]byte(stdout), &dump); err != nil { + return LuksDump{}, fmt.Errorf("failed to parse luksDump JSON: %w", err) + } + return dump, nil +} diff --git a/tools/storm/utils/sysinspect/dmsetup.go b/tools/storm/utils/sysinspect/dmsetup.go new file mode 100644 index 0000000000..16f288d444 --- /dev/null +++ b/tools/storm/utils/sysinspect/dmsetup.go @@ -0,0 +1,38 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// DmsetupInfo runs `sudo dmsetup info ` and returns the parsed key:value +// fields (Name, State, Tables present, UUID, ...). +func DmsetupInfo(client *ssh.Client, name string) (map[string]string, error) { + out, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo dmsetup info %s", name)) + if err != nil { + return nil, fmt.Errorf("failed to run dmsetup info %s: %w", name, err) + } + return ParseKeyValueLines(out), nil +} + +// ParseKeyValueLines parses lines of the form "key: value" into a map. Keys may +// contain spaces (e.g. "Tables present"); the split is on the first colon. +func ParseKeyValueLines(stdout string) map[string]string { + result := make(map[string]string) + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + if key != "" { + result[key] = value + } + } + return result +} diff --git a/tools/storm/utils/sysinspect/efi.go b/tools/storm/utils/sysinspect/efi.go new file mode 100644 index 0000000000..9dd5e1e137 --- /dev/null +++ b/tools/storm/utils/sysinspect/efi.go @@ -0,0 +1,77 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// EfiBootInfo is the parsed result of `efibootmgr`. +type EfiBootInfo struct { + // BootCurrent is the active boot entry number (e.g. "0001"). + BootCurrent string + // EntryNames maps a boot entry number (e.g. "0001") to its label (the token + // following "Boot####*" on the entry line). + EntryNames map[string]string +} + +// CurrentName returns the label of the currently-booted entry and whether both +// BootCurrent and its label were found. +func (e EfiBootInfo) CurrentName() (string, bool) { + if e.BootCurrent == "" { + return "", false + } + name, ok := e.EntryNames[e.BootCurrent] + return name, ok +} + +// EfiBootMgr runs `sudo efibootmgr` and returns the parsed boot info. +func EfiBootMgr(client *ssh.Client) (EfiBootInfo, error) { + out, err := sshutils.CommandOutput(client, "sudo efibootmgr") + if err != nil { + return EfiBootInfo{}, fmt.Errorf("failed to run efibootmgr: %w", err) + } + return ParseEfiBootMgr(out), nil +} + +// ParseEfiBootMgr parses `efibootmgr` output, extracting BootCurrent and the +// label of each Boot#### entry. Mirrors base_test.py::test_uefi_fallback. +// +// Example: +// +// BootCurrent: 0001 +// BootOrder: 0001,0000 +// Boot0000* UiApp +// Boot0001* azl HD(1,GPT,...) +func ParseEfiBootMgr(stdout string) EfiBootInfo { + info := EfiBootInfo{EntryNames: make(map[string]string)} + + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + line = strings.TrimSpace(line) + switch { + case strings.HasPrefix(line, "BootCurrent:"): + _, value, _ := strings.Cut(line, ":") + info.BootCurrent = strings.TrimSpace(value) + + case strings.HasPrefix(line, "Boot"): + // Entry lines look like "Boot0001* label ...". Skip non-entry + // "Boot*" lines such as BootOrder/BootNext. + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + // fields[0] = "Boot0001*" (or "Boot0001"); extract the 4-hex number. + head := strings.TrimPrefix(fields[0], "Boot") + head = strings.TrimSuffix(head, "*") + if len(head) != 4 { + continue + } + info.EntryNames[head] = fields[1] + } + } + + return info +} diff --git a/tools/storm/utils/sysinspect/encryption_parsers_test.go b/tools/storm/utils/sysinspect/encryption_parsers_test.go new file mode 100644 index 0000000000..b5454df986 --- /dev/null +++ b/tools/storm/utils/sysinspect/encryption_parsers_test.go @@ -0,0 +1,101 @@ +package sysinspect + +import "testing" + +func TestParseCryptsetupStatus_InUse(t *testing.T) { + sample := `/dev/mapper/web is active and is in use. + type: n/a + cipher: aes-xts-plain64 + keysize: 512 bits + device: /dev/md127 + mode: read/write` + s := ParseCryptsetupStatus(sample, "web") + if !s.Active || !s.InUse { + t.Errorf("Active=%v InUse=%v, want both true", s.Active, s.InUse) + } + if v, _ := s.Get("cipher"); v != "aes-xts-plain64" { + t.Errorf("cipher = %q", v) + } + if v, _ := s.Get("keysize"); v != "512 bits" { + t.Errorf("keysize = %q", v) + } +} + +func TestParseCryptsetupStatus_ActiveNotInUse(t *testing.T) { + s := ParseCryptsetupStatus("/dev/mapper/web is active.\n cipher: aes-xts-plain64", "web") + if !s.Active || s.InUse { + t.Errorf("Active=%v InUse=%v, want active-not-inuse", s.Active, s.InUse) + } +} + +func TestParseLuksDump(t *testing.T) { + sample := `{ + "keyslots": {"1": {"type":"luks2","kdf":{"type":"pbkdf2","hash":"sha512"},"area":{"encryption":"aes-xts-plain64"}}}, + "tokens": {"0": {"type":"systemd-tpm2","keyslots":["1"],"tpm2_pcrlock":false,"tpm2-pcrs":[7]}}, + "digests": {"0": {"type":"pbkdf2","hash":"sha512"}} + }` + d, err := ParseLuksDump(sample) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if d.Keyslots["1"].Type != "luks2" || d.Keyslots["1"].Kdf.Hash != "sha512" { + t.Errorf("keyslot 1 = %+v", d.Keyslots["1"]) + } + if d.Keyslots["1"].Area.Encryption != "aes-xts-plain64" { + t.Errorf("area encryption = %q", d.Keyslots["1"].Area.Encryption) + } + tok := d.Tokens["0"] + if tok.Type != "systemd-tpm2" || tok.Tpm2Pcrlock != false || len(tok.Tpm2Pcrs) != 1 || tok.Tpm2Pcrs[0] != 7 { + t.Errorf("token 0 = %+v", tok) + } + if d.Digests["0"].Type != "pbkdf2" || d.Digests["0"].Hash != "sha512" { + t.Errorf("digest 0 = %+v", d.Digests["0"]) + } +} + +func TestParseKeyValueLines(t *testing.T) { + sample := `Name: web +State: ACTIVE +Tables present: LIVE +UUID: CRYPT-LUKS2-475f03514bb749bbb9af1f53f94b91cb-web` + m := ParseKeyValueLines(sample) + if m["Name"] != "web" || m["State"] != "ACTIVE" || m["Tables present"] != "LIVE" { + t.Errorf("parsed = %v", m) + } + if m["UUID"] != "CRYPT-LUKS2-475f03514bb749bbb9af1f53f94b91cb-web" { + t.Errorf("UUID = %q", m["UUID"]) + } +} + +func TestParseFindmnt(t *testing.T) { + sample := `TARGET SOURCE FSTYPE OPTIONS +/mnt/web /dev/mapper/web ext4 rw,relatime` + rows := ParseFindmnt(sample) + if len(rows) != 1 { + t.Fatalf("got %d rows, want 1", len(rows)) + } + r := rows[0] + if r.Target != "/mnt/web" || r.Source != "/dev/mapper/web" || r.FsType != "ext4" { + t.Errorf("row = %+v", r) + } +} + +func TestParseBlkidExport(t *testing.T) { + sample := `DEVNAME=/dev/md127 +UUID=475f0351-4bb7-49bb-b9af-1f53f94b91cb +TYPE=crypto_LUKS +PARTLABEL=web + +DEVNAME=/dev/sr0 +TYPE=iso9660` + devs := ParseBlkidExport(sample) + if len(devs) != 2 { + t.Fatalf("got %d devices, want 2", len(devs)) + } + if devs["/dev/md127"]["TYPE"] != "crypto_LUKS" { + t.Errorf("md127 TYPE = %q", devs["/dev/md127"]["TYPE"]) + } + if devs["/dev/md127"]["PARTLABEL"] != "web" { + t.Errorf("md127 PARTLABEL = %q", devs["/dev/md127"]["PARTLABEL"]) + } +} diff --git a/tools/storm/utils/sysinspect/findmnt.go b/tools/storm/utils/sysinspect/findmnt.go new file mode 100644 index 0000000000..22ccb84c29 --- /dev/null +++ b/tools/storm/utils/sysinspect/findmnt.go @@ -0,0 +1,54 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// FindmntRow is one row of `findmnt ` output. +type FindmntRow struct { + Target string + Source string + FsType string + Options string +} + +// Findmnt runs `sudo findmnt ` and returns the parsed rows. +func Findmnt(client *ssh.Client, target string) ([]FindmntRow, error) { + out, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo findmnt %s", target)) + if err != nil { + return nil, fmt.Errorf("failed to run findmnt %s: %w", target, err) + } + return ParseFindmnt(out), nil +} + +// ParseFindmnt parses `findmnt ` table output. The first line is the +// header (TARGET SOURCE FSTYPE OPTIONS); columns are whitespace-separated. +func ParseFindmnt(stdout string) []FindmntRow { + lines := strings.Split(strings.TrimSpace(stdout), "\n") + if len(lines) < 2 { + return nil + } + + var rows []FindmntRow + for _, line := range lines[1:] { + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + row := FindmntRow{ + Target: fields[0], + Source: fields[1], + FsType: fields[2], + } + if len(fields) > 3 { + row.Options = fields[3] + } + rows = append(rows, row) + } + return rows +} diff --git a/tools/storm/utils/sysinspect/lsblk.go b/tools/storm/utils/sysinspect/lsblk.go new file mode 100644 index 0000000000..4461f025cd --- /dev/null +++ b/tools/storm/utils/sysinspect/lsblk.go @@ -0,0 +1,63 @@ +package sysinspect + +import ( + "encoding/json" + "fmt" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// LsblkDevice is a single node in the `lsblk -J` tree. Sizes are in bytes when +// lsblk is invoked with -b. +type LsblkDevice struct { + Name string `json:"name"` + MajMin string `json:"maj:min"` + RM bool `json:"rm"` + Size int64 `json:"size"` + RO bool `json:"ro"` + Type string `json:"type"` + Mountpoints []*string `json:"mountpoints"` + Children []LsblkDevice `json:"children,omitempty"` +} + +// LsblkOutput is the top-level structure of `lsblk -J` output. +type LsblkOutput struct { + Blockdevices []LsblkDevice `json:"blockdevices"` +} + +// Partitions flattens the block-device tree into the set of leaf devices, +// treating any block device without children as a partition (mirrors the +// flattening done in base_test.py). +func (o LsblkOutput) Partitions() []LsblkDevice { + var partitions []LsblkDevice + for _, bd := range o.Blockdevices { + if len(bd.Children) == 0 { + partitions = append(partitions, bd) + continue + } + partitions = append(partitions, bd.Children...) + } + return partitions +} + +// Lsblk runs `lsblk -J -b` on the host and returns the parsed tree with sizes +// in bytes. +func Lsblk(client *ssh.Client) (LsblkOutput, error) { + out, err := sshutils.CommandOutput(client, "lsblk -J -b") + if err != nil { + return LsblkOutput{}, fmt.Errorf("failed to run lsblk: %w", err) + } + return ParseLsblk(out) +} + +// ParseLsblk parses `lsblk -J -b` JSON output. Separated from Lsblk for unit +// testing without SSH. +func ParseLsblk(stdout string) (LsblkOutput, error) { + var parsed LsblkOutput + if err := json.Unmarshal([]byte(stdout), &parsed); err != nil { + return LsblkOutput{}, fmt.Errorf("failed to parse lsblk JSON: %w", err) + } + return parsed, nil +} diff --git a/tools/storm/utils/sysinspect/mount.go b/tools/storm/utils/sysinspect/mount.go new file mode 100644 index 0000000000..81fff8d2a9 --- /dev/null +++ b/tools/storm/utils/sysinspect/mount.go @@ -0,0 +1,58 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// MountEntry describes one line of `mount` output. +type MountEntry struct { + Device string + MountPoint string + FsType string +} + +// Mount runs `mount` on the host and returns the parsed entries. +func Mount(client *ssh.Client) ([]MountEntry, error) { + out, err := sshutils.CommandOutput(client, "mount") + if err != nil { + return nil, fmt.Errorf("failed to run mount: %w", err) + } + return ParseMount(out), nil +} + +// ParseMount parses `mount` output lines of the form +// "device on mount_point type fs_type (options)". +func ParseMount(stdout string) []MountEntry { + var entries []MountEntry + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + entry := MountEntry{ + Device: fields[0], + MountPoint: fields[2], + FsType: "unknown", + } + if len(fields) > 4 { + entry.FsType = fields[4] + } + entries = append(entries, entry) + } + return entries +} + +// RootDevice returns the device mounted at "/" and whether one was found. +func RootDevice(entries []MountEntry) (string, bool) { + for _, e := range entries { + if e.MountPoint == "/" { + return e.Device, true + } + } + return "", false +} diff --git a/tools/storm/utils/sysinspect/parsers2_test.go b/tools/storm/utils/sysinspect/parsers2_test.go new file mode 100644 index 0000000000..ddd96a5ca9 --- /dev/null +++ b/tools/storm/utils/sysinspect/parsers2_test.go @@ -0,0 +1,89 @@ +package sysinspect + +import "testing" + +const mdSample = `total 0 +lrwxrwxrwx 1 root root 8 Apr 1 22:42 home -> ../md124 +lrwxrwxrwx 1 root root 8 Apr 1 22:42 root-a -> ../md127 +lrwxrwxrwx 1 root root 8 Apr 1 22:42 root-b -> ../md125 +lrwxrwxrwx 1 root root 8 Apr 1 22:42 trident -> ../md126` + +func TestParseRaidName(t *testing.T) { + name, ok := parseRaidName(mdSample, "/dev/md127") + if !ok { + t.Fatal("expected to resolve /dev/md127") + } + if name != "/dev/md/root-a" { + t.Errorf("got %q, want /dev/md/root-a", name) + } + + // Bare name form. + name, ok = parseRaidName(mdSample, "md125") + if !ok || name != "/dev/md/root-b" { + t.Errorf("got %q (ok=%v), want /dev/md/root-b", name, ok) + } + + // Unknown device. + if _, ok := parseRaidName(mdSample, "/dev/md999"); ok { + t.Error("md999 should not resolve") + } +} + +const passwdSample = `root:x:0:0:root:/root:/bin/bash +bin:x:1:1:bin:/dev/null:/bin/false +testing-user:x:1001:1001::/home/testing-user:/bin/bash` + +const groupSample = `root:x:0: +wheel:x:10:testing-user,admin +bin:x:1:daemon +empty:x:99:` + +func TestParsePasswd(t *testing.T) { + users := ParsePasswd(passwdSample) + if len(users) != 3 { + t.Fatalf("got %d users, want 3", len(users)) + } + if _, ok := users["testing-user"]; !ok { + t.Error("testing-user missing") + } +} + +func TestParseGroup(t *testing.T) { + groups := ParseGroup(groupSample) + wheel, ok := groups["wheel"] + if !ok { + t.Fatal("wheel group missing") + } + if _, ok := wheel["testing-user"]; !ok { + t.Error("testing-user not in wheel") + } + if _, ok := wheel["admin"]; !ok { + t.Error("admin not in wheel") + } + if len(groups["empty"]) != 0 { + t.Errorf("empty group should have no members, got %v", groups["empty"]) + } +} + +const efiSample = `BootCurrent: 0001 +Timeout: 0 seconds +BootOrder: 0001,0000 +Boot0000* UiApp FvVol(...) +Boot0001* azl HD(1,GPT,abc)/File(...)` + +func TestParseEfiBootMgr(t *testing.T) { + info := ParseEfiBootMgr(efiSample) + if info.BootCurrent != "0001" { + t.Errorf("BootCurrent = %q, want 0001", info.BootCurrent) + } + name, ok := info.CurrentName() + if !ok { + t.Fatal("CurrentName not found") + } + if name != "azl" { + t.Errorf("CurrentName = %q, want azl", name) + } + if info.EntryNames["0000"] != "UiApp" { + t.Errorf("entry 0000 = %q, want UiApp", info.EntryNames["0000"]) + } +} diff --git a/tools/storm/utils/sysinspect/parsers_test.go b/tools/storm/utils/sysinspect/parsers_test.go new file mode 100644 index 0000000000..5b21360b1a --- /dev/null +++ b/tools/storm/utils/sysinspect/parsers_test.go @@ -0,0 +1,108 @@ +package sysinspect + +import "testing" + +const blkidSample = `/dev/sr0: BLOCK_SIZE="2048" UUID="2023-12-16-00-55-13-99" LABEL="TRIDENT_CDROM" TYPE="iso9660" +/dev/sda4: LABEL="3e9cecef-5a01-4" UUID="37a7b4fa-87f0-4887-895b-393f46c345a0" TYPE="swap" PARTLABEL="swap" PARTUUID="3e9cecef-5a01-43d6-a1ae-58bf24f42521" +/dev/sda2: UUID="04267584-7e18-4612-a649-c71e1811bd82" BLOCK_SIZE="4096" TYPE="ext4" PARTLABEL="root-a" PARTUUID="f1be3a27-36e2-4d4b-b8ec-5b0b5909cbf9" +/dev/sda1: SEC_TYPE="msdos" UUID="D920-8BA4" BLOCK_SIZE="512" TYPE="vfat" PARTLABEL="esp" PARTUUID="6fcc7c57-b21c-46e5-bc79-041c7fc53f34" +/dev/sda3: PARTLABEL="root-b" PARTUUID="573fdf4c-9133-4a9f-8cf5-aff7b74d1aeb"` + +func TestParseBlkid(t *testing.T) { + entries := ParseBlkid(blkidSample) + if len(entries) != 5 { + t.Fatalf("got %d entries, want 5", len(entries)) + } + + sda2, ok := entries["sda2"] + if !ok { + t.Fatal("missing sda2 entry") + } + if v, _ := sda2.Get("TYPE"); v != "ext4" { + t.Errorf("sda2 TYPE = %q, want ext4", v) + } + if v, _ := sda2.Get("PARTLABEL"); v != "root-a" { + t.Errorf("sda2 PARTLABEL = %q, want root-a", v) + } + if v, _ := sda2.Get("PARTUUID"); v != "f1be3a27-36e2-4d4b-b8ec-5b0b5909cbf9" { + t.Errorf("sda2 PARTUUID = %q", v) + } + + // sda3 has no TYPE (unformatted B volume) - Get should report absent. + sda3 := entries["sda3"] + if _, ok := sda3.Get("TYPE"); ok { + t.Error("sda3 should not have TYPE") + } + if v, _ := sda3.Get("PARTLABEL"); v != "root-b" { + t.Errorf("sda3 PARTLABEL = %q, want root-b", v) + } +} + +const lsblkSample = `{ + "blockdevices": [ + {"name":"sda","maj:min":"8:0","rm":false,"size":34359738368,"ro":false,"type":"disk","mountpoints":[null], + "children":[ + {"name":"sda1","maj:min":"8:1","rm":false,"size":1073741824,"ro":false,"type":"part","mountpoints":["/boot/efi"]}, + {"name":"sda2","maj:min":"8:2","rm":false,"size":8589934592,"ro":false,"type":"part","mountpoints":["/"]} + ]}, + {"name":"sdb","maj:min":"8:16","rm":false,"size":34359738368,"ro":false,"type":"disk","mountpoints":[null], + "children":[ + {"name":"sdb1","maj:min":"8:17","rm":false,"size":10485760,"ro":false,"type":"part","mountpoints":[null]} + ]}, + {"name":"sr0","maj:min":"11:0","rm":true,"size":501121024,"ro":false,"type":"rom","mountpoints":[null]} + ] +}` + +func TestParseLsblk(t *testing.T) { + out, err := ParseLsblk(lsblkSample) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(out.Blockdevices) != 3 { + t.Fatalf("got %d block devices, want 3", len(out.Blockdevices)) + } + + parts := out.Partitions() + // sda1, sda2, sdb1 (children) + sr0 (no children) = 4 + if len(parts) != 4 { + t.Fatalf("got %d partitions, want 4", len(parts)) + } + + byName := map[string]LsblkDevice{} + for _, p := range parts { + byName[p.Name] = p + } + if byName["sda2"].Size != 8589934592 { + t.Errorf("sda2 size = %d, want 8589934592", byName["sda2"].Size) + } + if got := byName["sda2"].Mountpoints; len(got) != 1 || got[0] == nil || *got[0] != "/" { + t.Errorf("sda2 mountpoint unexpected: %v", got) + } + if _, ok := byName["sr0"]; !ok { + t.Error("sr0 should be treated as a leaf partition") + } +} + +const mountSample = `/dev/sda3 on / type ext4 (rw,relatime) +devtmpfs on /dev type devtmpfs (rw,nosuid) +/dev/sda5 on /home type ext4 (rw,relatime) +/dev/sda1 on /boot/efi type vfat (rw,relatime)` + +func TestParseMountAndRootDevice(t *testing.T) { + entries := ParseMount(mountSample) + if len(entries) != 4 { + t.Fatalf("got %d mount entries, want 4", len(entries)) + } + + root, ok := RootDevice(entries) + if !ok { + t.Fatal("root device not found") + } + if root != "/dev/sda3" { + t.Errorf("root device = %q, want /dev/sda3", root) + } + + if entries[0].FsType != "ext4" { + t.Errorf("first entry fstype = %q, want ext4", entries[0].FsType) + } +} diff --git a/tools/storm/utils/sysinspect/raid.go b/tools/storm/utils/sysinspect/raid.go new file mode 100644 index 0000000000..efc91db6e9 --- /dev/null +++ b/tools/storm/utils/sysinspect/raid.go @@ -0,0 +1,54 @@ +package sysinspect + +import ( + "fmt" + "regexp" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// mdSymlinkRe matches an `ls -l /dev/md` line, capturing the RAID array name +// and the md device it links to, e.g. "root-a -> ../md127". +var mdSymlinkRe = regexp.MustCompile(`(\S+)\s+->\s+\.\./(md\d+)`) + +// RaidNameForDevice resolves a kernel md device path (e.g. "/dev/md127") to its +// friendly RAID array path (e.g. "/dev/md/root-a") by inspecting `ls -l /dev/md`. +// Returns ("", false) if /dev/md does not exist or no matching array is found. +// +// Mirrors base_test.py::get_raid_name_from_device_name. +func RaidNameForDevice(client *ssh.Client, deviceName string) (string, bool, error) { + // Tolerate a missing /dev/md directory (non-RAID configs) without error. + out, err := sshutils.RunCommand(client, "ls -l /dev/md") + if err != nil { + return "", false, fmt.Errorf("failed to run ls -l /dev/md: %w", err) + } + if out.Status != 0 { + // Directory absent or empty: device is not a RAID array. + return "", false, nil + } + + name, found := parseRaidName(out.Stdout, deviceName) + return name, found, nil +} + +// parseRaidName extracts the friendly RAID path for the given md device from +// `ls -l /dev/md` output. deviceName may be a full path ("/dev/md127") or bare +// name ("md127"). +func parseRaidName(stdout, deviceName string) (string, bool) { + mdName := deviceName[strings.LastIndex(deviceName, "/")+1:] + + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + matches := mdSymlinkRe.FindStringSubmatch(line) + if matches == nil { + continue + } + // matches[1] = array name, matches[2] = md device (e.g. md127) + if matches[2] == mdName { + return "/dev/md/" + matches[1], true + } + } + return "", false +} diff --git a/tools/storm/utils/sysinspect/swap.go b/tools/storm/utils/sysinspect/swap.go new file mode 100644 index 0000000000..5dfbf2f7d9 --- /dev/null +++ b/tools/storm/utils/sysinspect/swap.go @@ -0,0 +1,61 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// ActiveSwaps returns the set of active swap device paths, canonicalized via +// `readlink -f`. Mirrors encryption_test.py::get_active_swaps. +func ActiveSwaps(client *ssh.Client) (map[string]struct{}, error) { + cmd := "swapon --show=NAME --raw --bytes --noheadings | xargs -r -I @ readlink -f @" + out, err := sshutils.CommandOutput(client, cmd) + if err != nil { + return nil, fmt.Errorf("failed to list active swaps: %w", err) + } + + swaps := make(map[string]struct{}) + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + line = strings.TrimSpace(line) + if line != "" { + swaps[line] = struct{}{} + } + } + return swaps, nil +} + +// ReadlinkF resolves a path to its canonical absolute form via `readlink -f`. +func ReadlinkF(client *ssh.Client, path string) (string, error) { + out, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo readlink -f %s", path)) + if err != nil { + return "", fmt.Errorf("failed to readlink -f %s: %w", path, err) + } + return strings.TrimSpace(out), nil +} + +// Getenforce returns the current SELinux enforcement mode ("Enforcing", +// "Permissive", or "Disabled"). +func Getenforce(client *ssh.Client) (string, error) { + out, err := sshutils.CommandOutput(client, "sudo getenforce") + if err != nil { + return "", fmt.Errorf("failed to run getenforce: %w", err) + } + return strings.TrimSpace(out), nil +} + +// Setenforce sets the SELinux enforcement mode (0 = permissive, 1 = enforcing). +func Setenforce(client *ssh.Client, enforcing bool) error { + mode := "0" + if enforcing { + mode = "1" + } + _, err := sshutils.CommandOutput(client, "sudo setenforce "+mode) + if err != nil { + return fmt.Errorf("failed to run setenforce %s: %w", mode, err) + } + return nil +} diff --git a/tools/storm/utils/sysinspect/systemd_ext.go b/tools/storm/utils/sysinspect/systemd_ext.go new file mode 100644 index 0000000000..fd57043358 --- /dev/null +++ b/tools/storm/utils/sysinspect/systemd_ext.go @@ -0,0 +1,85 @@ +package sysinspect + +import ( + "encoding/json" + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// systemdExtHierarchy is one entry of `systemd-sysext status --json` / +// `systemd-confext status --json` output: a hierarchy (e.g. /usr, /opt, /etc) +// with the list of extensions currently merged into it. +type systemdExtHierarchy struct { + Hierarchy string `json:"hierarchy"` + Extensions stringOrSlice `json:"extensions"` +} + +// stringOrSlice decodes a JSON value that systemd emits either as an array of +// strings, a single bare string (e.g. "none" when a hierarchy has no +// extensions merged), or null. It normalizes all three into a []string. +type stringOrSlice []string + +func (s *stringOrSlice) UnmarshalJSON(data []byte) error { + trimmed := strings.TrimSpace(string(data)) + if trimmed == "null" { + *s = nil + return nil + } + // Array form: ["a","b"]. + if strings.HasPrefix(trimmed, "[") { + var list []string + if err := json.Unmarshal(data, &list); err != nil { + return err + } + *s = list + return nil + } + // Scalar string form: "none" / "some-ext". + var single string + if err := json.Unmarshal(data, &single); err != nil { + return err + } + *s = []string{single} + return nil +} + +// SystemdExtStatus runs `systemd- status --json=pretty` on the host and +// returns the set of active extension names across all hierarchies. extType is +// "sysext" or "confext". +func SystemdExtStatus(client *ssh.Client, extType string) (map[string]struct{}, error) { + cmd := fmt.Sprintf("sudo systemd-%s status --json=pretty --no-pager", extType) + out, err := sshutils.RunCommand(client, cmd) + if err != nil { + return nil, fmt.Errorf("failed to run systemd-%s status: %w", extType, err) + } + if err := out.Check(); err != nil { + return nil, fmt.Errorf("systemd-%s status failed: %s", extType, out.Report()) + } + return ParseSystemdExtStatus(out.Stdout) +} + +// ParseSystemdExtStatus parses `systemd-sysext/confext status --json` output +// into the set of active extension names. Separated for unit testing. +func ParseSystemdExtStatus(stdout string) (map[string]struct{}, error) { + var hierarchies []systemdExtHierarchy + if err := json.Unmarshal([]byte(stdout), &hierarchies); err != nil { + return nil, fmt.Errorf("failed to parse systemd extension status JSON: %w", err) + } + + active := make(map[string]struct{}) + for _, h := range hierarchies { + for _, ext := range h.Extensions { + // systemd reports "none" (as a bare string) for a hierarchy with no + // extensions merged; it is a sentinel, not an extension name. + if strings.EqualFold(ext, "none") { + continue + } + active[ext] = struct{}{} + } + } + return active, nil +} diff --git a/tools/storm/utils/sysinspect/systemd_ext_test.go b/tools/storm/utils/sysinspect/systemd_ext_test.go new file mode 100644 index 0000000000..a2c4e199e0 --- /dev/null +++ b/tools/storm/utils/sysinspect/systemd_ext_test.go @@ -0,0 +1,47 @@ +package sysinspect + +import "testing" + +const sysextStatusSample = `[ + {"hierarchy":"/opt","extensions":["myext"]}, + {"hierarchy":"/usr","extensions":["myext","other"]}, + {"hierarchy":"/var","extensions":null} +]` + +func TestParseSystemdExtStatus(t *testing.T) { + active, err := ParseSystemdExtStatus(sysextStatusSample) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(active) != 2 { + t.Fatalf("got %d active exts, want 2 (deduped)", len(active)) + } + for _, want := range []string{"myext", "other"} { + if _, ok := active[want]; !ok { + t.Errorf("missing active ext %q", want) + } + } +} + +// TestParseSystemdExtStatus_ScalarExtensions covers the systemd variant where +// the `extensions` field is a bare string ("none" for empty hierarchies, or a +// single extension name) rather than an array. +func TestParseSystemdExtStatus_ScalarExtensions(t *testing.T) { + const sample = `[ + {"hierarchy":"/opt","extensions":"none"}, + {"hierarchy":"/usr","extensions":"solo"} +]` + active, err := ParseSystemdExtStatus(sample) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(active) != 1 { + t.Fatalf("got %d active exts, want 1 (none filtered)", len(active)) + } + if _, ok := active["solo"]; !ok { + t.Errorf("missing active ext %q", "solo") + } + if _, ok := active["none"]; ok { + t.Error(`"none" sentinel should not be treated as an active extension`) + } +} diff --git a/tools/storm/utils/sysinspect/users.go b/tools/storm/utils/sysinspect/users.go new file mode 100644 index 0000000000..0d095f2eed --- /dev/null +++ b/tools/storm/utils/sysinspect/users.go @@ -0,0 +1,70 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// Users runs `cat /etc/passwd` and returns the set of usernames on the host. +func Users(client *ssh.Client) (map[string]struct{}, error) { + out, err := sshutils.CommandOutput(client, "cat /etc/passwd") + if err != nil { + return nil, fmt.Errorf("failed to read /etc/passwd: %w", err) + } + return ParsePasswd(out), nil +} + +// ParsePasswd parses /etc/passwd content into a set of usernames (the first +// colon-separated field of each line). +func ParsePasswd(stdout string) map[string]struct{} { + users := make(map[string]struct{}) + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + name, _, _ := strings.Cut(line, ":") + users[name] = struct{}{} + } + return users +} + +// Groups runs `cat /etc/group` and returns, for each group, the set of member +// usernames listed in the final field. +func Groups(client *ssh.Client) (map[string]map[string]struct{}, error) { + out, err := sshutils.CommandOutput(client, "cat /etc/group") + if err != nil { + return nil, fmt.Errorf("failed to read /etc/group: %w", err) + } + return ParseGroup(out), nil +} + +// ParseGroup parses /etc/group content into a map of group name -> member set. +// Lines look like "wheel:x:10:testing-user,other". +func ParseGroup(stdout string) map[string]map[string]struct{} { + groups := make(map[string]map[string]struct{}) + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.Split(line, ":") + if len(parts) < 1 { + continue + } + members := make(map[string]struct{}) + if len(parts) >= 4 && parts[len(parts)-1] != "" { + for _, m := range strings.Split(parts[len(parts)-1], ",") { + if m != "" { + members[m] = struct{}{} + } + } + } + groups[parts[0]] = members + } + return groups +} diff --git a/tools/storm/utils/sysinspect/veritysetup.go b/tools/storm/utils/sysinspect/veritysetup.go new file mode 100644 index 0000000000..046236c062 --- /dev/null +++ b/tools/storm/utils/sysinspect/veritysetup.go @@ -0,0 +1,65 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// VeritySetupStatus is the parsed output of `veritysetup status `. +type VeritySetupStatus struct { + // Active is true when the first status line reports the device is active + // and in use. + Active bool + // Fields holds the "key: value" lines below the header (e.g. "type", + // "status", "mode", "data device", "hash device"). + Fields map[string]string +} + +// Get returns a status field value and whether it was present. +func (s VeritySetupStatus) Get(field string) (string, bool) { + v, ok := s.Fields[field] + return v, ok +} + +// VeritySetup runs `sudo veritysetup status ` on the host and returns the +// parsed status. +func VeritySetup(client *ssh.Client, name string) (VeritySetupStatus, error) { + out, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo veritysetup status %s", name)) + if err != nil { + return VeritySetupStatus{}, fmt.Errorf("failed to run veritysetup status %s: %w", name, err) + } + return ParseVeritySetupStatus(out, name), nil +} + +// ParseVeritySetupStatus parses `veritysetup status ` output. The first +// line is the " is active and is in use." header; subsequent lines are +// "key: value" pairs (keys may contain spaces, e.g. "data device"). +func ParseVeritySetupStatus(stdout, name string) VeritySetupStatus { + status := VeritySetupStatus{Fields: make(map[string]string)} + + lines := strings.Split(strings.TrimSpace(stdout), "\n") + if len(lines) == 0 { + return status + } + + header := strings.TrimSpace(lines[0]) + status.Active = header == fmt.Sprintf("/dev/mapper/%s is active and is in use.", name) + + for _, line := range lines[1:] { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + if key != "" && value != "" { + status.Fields[key] = value + } + } + + return status +} diff --git a/tools/storm/utils/sysinspect/veritysetup_test.go b/tools/storm/utils/sysinspect/veritysetup_test.go new file mode 100644 index 0000000000..3f23a6ad91 --- /dev/null +++ b/tools/storm/utils/sysinspect/veritysetup_test.go @@ -0,0 +1,47 @@ +package sysinspect + +import "testing" + +const veritysetupSample = `/dev/mapper/root is active and is in use. + type: VERITY + status: verified + hash type: 1 + data block: 4096 + hash block: 4096 + hash name: sha256 + data device: /dev/sda3 + size: 1377128 sectors + mode: readonly + hash device: /dev/sda4 + hash offset: 8 sectors + root hash: a8c34ed685f365352231db21aa36ff23bf8b658e001afa8e498f57d1755e9a19 + flags: panic_on_corruption` + +func TestParseVeritySetupStatus(t *testing.T) { + s := ParseVeritySetupStatus(veritysetupSample, "root") + if !s.Active { + t.Error("expected Active=true") + } + if v, _ := s.Get("type"); v != "VERITY" { + t.Errorf("type = %q, want VERITY", v) + } + if v, _ := s.Get("status"); v != "verified" { + t.Errorf("status = %q, want verified", v) + } + if v, _ := s.Get("mode"); v != "readonly" { + t.Errorf("mode = %q, want readonly", v) + } + if v, _ := s.Get("data device"); v != "/dev/sda3" { + t.Errorf("data device = %q, want /dev/sda3", v) + } + if v, _ := s.Get("hash device"); v != "/dev/sda4" { + t.Errorf("hash device = %q, want /dev/sda4", v) + } +} + +func TestParseVeritySetupStatus_Inactive(t *testing.T) { + s := ParseVeritySetupStatus("/dev/mapper/root is inactive.", "root") + if s.Active { + t.Error("expected Active=false for inactive device") + } +} diff --git a/tools/storm/utils/trident/hoststatus.go b/tools/storm/utils/trident/hoststatus.go new file mode 100644 index 0000000000..439e089057 --- /dev/null +++ b/tools/storm/utils/trident/hoststatus.go @@ -0,0 +1,131 @@ +package trident + +import ( + "fmt" + + "github.com/Jeffail/gabs/v2" + "golang.org/x/crypto/ssh" + "gopkg.in/yaml.v3" + + "tridenttools/pkg/hostconfig" +) + +// ServicingState mirrors trident_api::status::ServicingState (kebab-case). It +// is the value reported under `servicingState` in `trident get` output. +type ServicingState string + +const ( + ServicingStateNotProvisioned ServicingState = "not-provisioned" + ServicingStateCleanInstallStaged ServicingState = "clean-install-staged" + ServicingStateAbUpdateStaged ServicingState = "ab-update-staged" + ServicingStateManualRollbackAbStaged ServicingState = "manual-rollback-ab-staged" + ServicingStateManualRollbackRuntimeStaged ServicingState = "manual-rollback-runtime-staged" + ServicingStateRuntimeUpdateStaged ServicingState = "runtime-update-staged" + ServicingStateCleanInstallFinalized ServicingState = "clean-install-finalized" + ServicingStateAbUpdateFinalized ServicingState = "ab-update-finalized" + ServicingStateManualRollbackAbFinalized ServicingState = "manual-rollback-ab-finalized" + ServicingStateProvisioned ServicingState = "provisioned" + ServicingStateAbUpdateHealthCheckFailed ServicingState = "ab-update-health-check-failed" +) + +// AbVolumeSelection mirrors trident_api::status::AbVolumeSelection (kebab-case). +// It is the value reported under `abActiveVolume` in `trident get` output. +type AbVolumeSelection string + +const ( + AbVolumeA AbVolumeSelection = "volume-a" + AbVolumeB AbVolumeSelection = "volume-b" +) + +// Other returns the opposite A/B volume selection. +func (v AbVolumeSelection) Other() AbVolumeSelection { + if v == AbVolumeA { + return AbVolumeB + } + return AbVolumeA +} + +// HostStatus is a hybrid view over the YAML emitted by `trident get`. It wraps a +// gabs.Container (escape hatch for rarely-touched corners) and provides typed +// accessors for the stable core fields that E2E validations depend on. +type HostStatus struct { + *gabs.Container +} + +// NewHostStatusFromYaml parses `trident get` YAML output into a HostStatus. +// +// The Host Status embeds a Host Configuration under `spec` whose `contents` +// nodes carry custom YAML tags (e.g. `!image`). yaml.v3 decodes these into +// plain maps when the target is `map[string]any`, so no custom tag constructor +// is required (unlike the Python suite's yaml.add_multi_constructor). +func NewHostStatusFromYaml(yamlData []byte) (HostStatus, error) { + var data map[string]any + if err := yaml.Unmarshal(yamlData, &data); err != nil { + return HostStatus{}, fmt.Errorf("failed to unmarshal Host Status YAML: %w", err) + } + + return HostStatus{Container: gabs.Wrap(data)}, nil +} + +// GetHostStatus runs `trident get` on the host over the provided SSH client and +// parses the result into a HostStatus. +func GetHostStatus(runtime RuntimeType, client *ssh.Client) (HostStatus, error) { + out, err := InvokeTrident(runtime, client, nil, "get") + if err != nil { + return HostStatus{}, fmt.Errorf("failed to invoke 'trident get': %w", err) + } + if err := out.Check(); err != nil { + return HostStatus{}, fmt.Errorf("'trident get' failed: %s", out.Report()) + } + + return NewHostStatusFromYaml([]byte(out.Stdout)) +} + +// ServicingState returns the current servicing state of the host. +func (hs *HostStatus) ServicingState() ServicingState { + s, _ := hs.S("servicingState").Data().(string) + return ServicingState(s) +} + +// AbActiveVolume returns the active A/B volume and whether it is present. It is +// absent on hosts that were never A/B updated (or failed before provisioning). +func (hs *HostStatus) AbActiveVolume() (AbVolumeSelection, bool) { + s, ok := hs.S("abActiveVolume").Data().(string) + if !ok { + return "", false + } + return AbVolumeSelection(s), true +} + +// PartitionPaths returns the device path of each block device, keyed by device +// ID (the `partitionPaths` map). +func (hs *HostStatus) PartitionPaths() map[string]string { + result := make(map[string]string) + for id, child := range hs.S("partitionPaths").ChildrenMap() { + if path, ok := child.Data().(string); ok { + result[id] = path + } + } + return result +} + +// Spec returns the embedded Host Configuration (`spec`) as a HostConfig, reusing +// the existing gabs-backed configuration handling for storage introspection. +func (hs *HostStatus) Spec() hostconfig.HostConfig { + return hostconfig.NewHostConfigFromContainer(hs.S("spec")) +} + +// LastError returns the serialized YAML of the `lastError` field and whether it +// is present. Validations match substrings against it (e.g. rollback checks). +func (hs *HostStatus) LastError() (string, bool) { + container := hs.S("lastError") + if container == nil || container.Data() == nil { + return "", false + } + + raw, err := yaml.Marshal(container.Data()) + if err != nil { + return "", false + } + return string(raw), true +} diff --git a/tools/storm/utils/trident/hoststatus_test.go b/tools/storm/utils/trident/hoststatus_test.go new file mode 100644 index 0000000000..4f7a13c16c --- /dev/null +++ b/tools/storm/utils/trident/hoststatus_test.go @@ -0,0 +1,133 @@ +package trident + +import ( + "strings" + "testing" +) + +const sampleHostStatusYaml = ` +abActiveVolume: volume-a +diskUuids: + disk-0: f4265b47-09cd-4d5e-aa92-684fb783f817 +installIndex: 0 +partitionPaths: + esp: /dev/sda1 + root-a: /dev/sda2 + root-b: /dev/sda3 +servicingState: provisioned +lastError: null +spec: + storage: + abUpdate: + volumePairs: + - id: root + volumeAId: root-a + volumeBId: root-b + disks: + - device: /dev/sda + id: disk-0 + partitions: + - id: esp + size: 8M + type: esp + - id: root-a + size: 4G + type: linux-generic + image: + url: http://blob/regular.cosi + contents: !image + sha256: abc123 + length: 705090048 +` + +func TestNewHostStatusFromYaml_CoreFields(t *testing.T) { + hs, err := NewHostStatusFromYaml([]byte(sampleHostStatusYaml)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if got := hs.ServicingState(); got != ServicingStateProvisioned { + t.Errorf("ServicingState() = %q, want %q", got, ServicingStateProvisioned) + } + + vol, present := hs.AbActiveVolume() + if !present { + t.Fatal("AbActiveVolume() reported absent, want present") + } + if vol != AbVolumeA { + t.Errorf("AbActiveVolume() = %q, want %q", vol, AbVolumeA) + } + + paths := hs.PartitionPaths() + if len(paths) != 3 { + t.Errorf("PartitionPaths() len = %d, want 3", len(paths)) + } + if paths["esp"] != "/dev/sda1" { + t.Errorf("PartitionPaths()[esp] = %q, want /dev/sda1", paths["esp"]) + } + if paths["root-a"] != "/dev/sda2" { + t.Errorf("PartitionPaths()[root-a] = %q, want /dev/sda2", paths["root-a"]) + } +} + +func TestNewHostStatusFromYaml_CustomImageTag(t *testing.T) { + // The `!image` tag on `spec.image.contents` must decode into a plain map, + // reachable via the gabs escape hatch through Spec(). + hs, err := NewHostStatusFromYaml([]byte(sampleHostStatusYaml)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + spec := hs.Spec() + if got := spec.S("image", "contents", "sha256").Data(); got != "abc123" { + t.Errorf("spec image contents sha256 = %v, want abc123", got) + } + if !spec.HasABUpdate() { + t.Error("Spec().HasABUpdate() = false, want true") + } +} + +func TestHostStatus_AbActiveVolumeAbsent(t *testing.T) { + hs, err := NewHostStatusFromYaml([]byte("servicingState: not-provisioned\n")) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if got := hs.ServicingState(); got != ServicingStateNotProvisioned { + t.Errorf("ServicingState() = %q, want %q", got, ServicingStateNotProvisioned) + } + if _, present := hs.AbActiveVolume(); present { + t.Error("AbActiveVolume() reported present, want absent") + } +} + +func TestHostStatus_LastError(t *testing.T) { + hs, err := NewHostStatusFromYaml([]byte(sampleHostStatusYaml)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, present := hs.LastError(); present { + t.Error("null lastError should be reported as absent") + } + + withErr, err := NewHostStatusFromYaml([]byte("lastError:\n message: Failed health check(s)\n")) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + msg, present := withErr.LastError() + if !present { + t.Fatal("LastError() reported absent, want present") + } + if !strings.Contains(msg, "Failed health check(s)") { + t.Errorf("LastError() = %q, want it to contain 'Failed health check(s)'", msg) + } +} + +func TestAbVolumeSelection_Other(t *testing.T) { + if got := AbVolumeA.Other(); got != AbVolumeB { + t.Errorf("AbVolumeA.Other() = %q, want %q", got, AbVolumeB) + } + if got := AbVolumeB.Other(); got != AbVolumeA { + t.Errorf("AbVolumeB.Other() = %q, want %q", got, AbVolumeA) + } +}