From d4233b69f8b37b6a7d68cf6263ebc71d012bdb49 Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 24 Jul 2026 15:41:00 -0700 Subject: [PATCH 01/41] storm/e2e: port base host-state validation to Go (Phase 0 + base) Add the foundations for porting the pytest E2E host-state validation suite into Go storm test cases, plus the first ported marker (base): - storm/utils/trident/hoststatus.go: hybrid Host Status parser (typed core accessors + gabs escape hatch), GetHostStatus over SSH. - storm/utils/sysinspect: base-tier SSH parsers (blkid, lsblk, mount, /dev/md RAID resolution, passwd/group, efibootmgr). - storm/e2e/validate: SoftAsserter (interim soft-assert accumulator) and base validation (partitions, users, uefi-fallback) ported from base_test.py, including the non-verity A/B active-volume path check. - scenario wiring: validate-install / validate-ab-update-* cases and expectedActiveVolume tracking (flipped after each A/B update). All unit-tested (stdlib testing); storm-trident builds and discovers base_vm-host. Not yet integration-tested on a real VM. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/storm/e2e/scenario/ab_update.go | 4 + tools/storm/e2e/scenario/register_test.go | 120 ++++++++ tools/storm/e2e/scenario/trident.go | 11 + tools/storm/e2e/scenario/validate.go | 48 +++ tools/storm/e2e/validate/base.go | 179 +++++++++++ tools/storm/e2e/validate/base_test.go | 140 +++++++++ tools/storm/e2e/validate/orchestrate.go | 280 ++++++++++++++++++ tools/storm/e2e/validate/softassert.go | 77 +++++ tools/storm/e2e/validate/softassert_test.go | 55 ++++ tools/storm/utils/sysinspect/blkid.go | 79 +++++ tools/storm/utils/sysinspect/efi.go | 77 +++++ tools/storm/utils/sysinspect/lsblk.go | 63 ++++ tools/storm/utils/sysinspect/mount.go | 58 ++++ tools/storm/utils/sysinspect/parsers2_test.go | 89 ++++++ tools/storm/utils/sysinspect/parsers_test.go | 108 +++++++ tools/storm/utils/sysinspect/raid.go | 54 ++++ tools/storm/utils/sysinspect/users.go | 70 +++++ tools/storm/utils/trident/hoststatus.go | 131 ++++++++ tools/storm/utils/trident/hoststatus_test.go | 133 +++++++++ 19 files changed, 1776 insertions(+) create mode 100644 tools/storm/e2e/scenario/register_test.go create mode 100644 tools/storm/e2e/scenario/validate.go create mode 100644 tools/storm/e2e/validate/base.go create mode 100644 tools/storm/e2e/validate/base_test.go create mode 100644 tools/storm/e2e/validate/orchestrate.go create mode 100644 tools/storm/e2e/validate/softassert.go create mode 100644 tools/storm/e2e/validate/softassert_test.go create mode 100644 tools/storm/utils/sysinspect/blkid.go create mode 100644 tools/storm/utils/sysinspect/efi.go create mode 100644 tools/storm/utils/sysinspect/lsblk.go create mode 100644 tools/storm/utils/sysinspect/mount.go create mode 100644 tools/storm/utils/sysinspect/parsers2_test.go create mode 100644 tools/storm/utils/sysinspect/parsers_test.go create mode 100644 tools/storm/utils/sysinspect/raid.go create mode 100644 tools/storm/utils/sysinspect/users.go create mode 100644 tools/storm/utils/trident/hoststatus.go create mode 100644 tools/storm/utils/trident/hoststatus_test.go diff --git a/tools/storm/e2e/scenario/ab_update.go b/tools/storm/e2e/scenario/ab_update.go index da563702b4..521b9d9c4c 100644 --- a/tools/storm/e2e/scenario/ab_update.go +++ b/tools/storm/e2e/scenario/ab_update.go @@ -308,6 +308,10 @@ func (s *TridentE2EScenario) abUpdateOs(tc storm.TestCase, split bool) error { tc.FailFromError(err) } + // The A/B update rebooted into the other volume; flip the expected active + // volume so subsequent validation checks the correct one. + s.expectedActiveVolume = s.expectedActiveVolume.Other() + return nil } diff --git a/tools/storm/e2e/scenario/register_test.go b/tools/storm/e2e/scenario/register_test.go new file mode 100644 index 0000000000..d58d657c45 --- /dev/null +++ b/tools/storm/e2e/scenario/register_test.go @@ -0,0 +1,120 @@ +package scenario + +import ( + "slices" + "testing" + + "github.com/microsoft/storm/pkg/storm/core" + + "tridenttools/pkg/hostconfig" + "tridenttools/storm/e2e/testrings" + "tridenttools/storm/utils/trident" +) + +// fakeRegistrar records the order of registered test-case names. +type fakeRegistrar struct { + names []string +} + +func (f *fakeRegistrar) RegisterTestCase(name string, _ core.TestCaseFunction) { + f.names = append(f.names, name) +} + +func newScenarioForTest(t *testing.T, configYaml string) *TridentE2EScenario { + t.Helper() + hc, err := hostconfig.NewHostConfigFromYaml([]byte(configYaml)) + if err != nil { + t.Fatalf("failed to parse config: %v", err) + } + s, err := NewTridentE2EScenario( + "test", []string{"e2e"}, hc, TridentE2EHostConfigParams{}, + HardwareTypeVM, trident.RuntimeTypeHost, testrings.TestRingSet{testrings.TestRingPrE2e}, + ) + if err != nil { + t.Fatalf("failed to create scenario: %v", err) + } + return s +} + +const abConfig = ` +storage: + abUpdate: + volumePairs: + - id: root + volumeAId: root-a + volumeBId: root-b +` + +const noAbConfig = ` +storage: + disks: + - id: os + partitions: + - id: root + size: 8G +` + +func TestRegisterTestCases_ABUpdate_RegistersValidation(t *testing.T) { + s := newScenarioForTest(t, abConfig) + var r fakeRegistrar + if err := s.RegisterTestCases(&r); err != nil { + t.Fatalf("RegisterTestCases error: %v", err) + } + + // validate-install must come right after check-trident-ssh. + assertOrder(t, r.names, "check-trident-ssh", "validate-install") + // Post-A/B-update validations must be registered. + mustContain(t, r.names, "validate-ab-update-1") + mustContain(t, r.names, "validate-ab-update-split") + // validate-ab-update-1 must come after the ab-update-1 update case. + assertOrder(t, r.names, "ab-update-1-ab-update", "validate-ab-update-1") + + assertUnique(t, r.names) +} + +func TestRegisterTestCases_NoABUpdate_OnlyInstallValidation(t *testing.T) { + s := newScenarioForTest(t, noAbConfig) + var r fakeRegistrar + if err := s.RegisterTestCases(&r); err != nil { + t.Fatalf("RegisterTestCases error: %v", err) + } + + mustContain(t, r.names, "validate-install") + if slices.Contains(r.names, "validate-ab-update-1") { + t.Error("validate-ab-update-1 should not be registered without abUpdate") + } + assertUnique(t, r.names) +} + +func assertOrder(t *testing.T, names []string, before, after string) { + t.Helper() + bi := slices.Index(names, before) + ai := slices.Index(names, after) + if bi < 0 { + t.Fatalf("%q not registered (have %v)", before, names) + } + if ai < 0 { + t.Fatalf("%q not registered (have %v)", after, names) + } + if bi >= ai { + t.Errorf("%q (idx %d) should come before %q (idx %d)", before, bi, after, ai) + } +} + +func mustContain(t *testing.T, names []string, want string) { + t.Helper() + if !slices.Contains(names, want) { + t.Errorf("expected %q to be registered, have %v", want, names) + } +} + +func assertUnique(t *testing.T, names []string) { + t.Helper() + seen := map[string]struct{}{} + for _, n := range names { + if _, dup := seen[n]; dup { + t.Errorf("duplicate test case name %q", n) + } + seen[n] = struct{}{} + } +} diff --git a/tools/storm/e2e/scenario/trident.go b/tools/storm/e2e/scenario/trident.go index 875f718423..dd234e1856 100644 --- a/tools/storm/e2e/scenario/trident.go +++ b/tools/storm/e2e/scenario/trident.go @@ -79,6 +79,11 @@ type TridentE2EScenario struct { // Version of the image, used for AB update tests version uint + // Expected active A/B volume after the most recent servicing operation. + // Initialized to volume-a on clean install and flipped after each + // successful A/B update. Read by validation cases. + expectedActiveVolume trident.AbVolumeSelection + // Working copy of the host configuration, modified during test execution to // reflect changes such as AB updates. config hostconfig.HostConfig @@ -115,6 +120,9 @@ func (s *TridentE2EScenario) Args() any { } func (s *TridentE2EScenario) Setup(storm.SetupCleanupContext) error { + // A clean install boots the A volume; A/B updates flip this. + s.expectedActiveVolume = trident.AbVolumeA + if s.args.TestRing == testrings.TestRingEmpty { // Default to lowest ring lowestRing, err := s.testRings.Lowest() @@ -173,10 +181,13 @@ func (s *TridentE2EScenario) RegisterTestCases(r storm.TestRegistrar) error { r.RegisterTestCase("setup-test-host", s.setupTestHost) r.RegisterTestCase("install-os", s.installOs) r.RegisterTestCase("check-trident-ssh", s.checkTridentViaSshAfterInstall) + r.RegisterTestCase("validate-install", s.validateHostState) if s.originalConfig.HasABUpdate() { s.addAbUpdateTests(r, "ab-update-1") + r.RegisterTestCase("validate-ab-update-1", s.validateHostState) s.addSplitABUpdateTests(r, "ab-update-split") + r.RegisterTestCase("validate-ab-update-split", s.validateHostState) } return nil } diff --git a/tools/storm/e2e/scenario/validate.go b/tools/storm/e2e/scenario/validate.go new file mode 100644 index 0000000000..3bd5010180 --- /dev/null +++ b/tools/storm/e2e/scenario/validate.go @@ -0,0 +1,48 @@ +package scenario + +import ( + "context" + "time" + + "github.com/microsoft/storm" + + "tridenttools/storm/e2e/validate" + "tridenttools/storm/utils/trident" +) + +// validateHostState is the storm test case that validates the installed host's +// state against the Host Configuration and Host Status. It is registered after +// clean install and after each A/B update. It ports the pytest E2E validation +// suite (tests/e2e_tests/*.py). +// +// All applicable sub-checks run and accumulate their failures (interim +// soft-assert approach) so a single case reports every mismatch it finds; the +// case fails once at the end if any sub-check failed. +func (s *TridentE2EScenario) validateHostState(tc storm.TestCase) error { + connCtx, cancel := context.WithTimeout(tc.Context(), time.Minute) + defer cancel() + if err := s.populateSshClient(connCtx); err != nil { + // The host is expected to be up by now, so a connection failure is an + // infrastructure error rather than a product failure. + return err + } + + hs, err := trident.GetHostStatus(s.runtime, s.sshClient) + if err != nil { + return err + } + + var sa validate.SoftAsserter + + // `base` validation always applies. + validate.ValidateBase(&sa, s.sshClient, hs, trident.ServicingStateProvisioned, s.expectedActiveVolume) + + // Future markers (encryption, verity, extensions) will self-select here + // based on the Host Configuration. + + if err := sa.Err(); err != nil { + tc.FailFromError(err) + } + + return nil +} diff --git a/tools/storm/e2e/validate/base.go b/tools/storm/e2e/validate/base.go new file mode 100644 index 0000000000..facd548f8e --- /dev/null +++ b/tools/storm/e2e/validate/base.go @@ -0,0 +1,179 @@ +package validate + +import ( + "math" + "strconv" + "strings" + + "tridenttools/pkg/hostconfig" + tridentutil "tridenttools/storm/utils/trident" +) + +// sizeUnits maps a single-letter size suffix to its multiplier (powers of 1024), +// matching base_test.py's SizeUnit enum. +var sizeUnits = map[byte]float64{ + 'B': 1, + 'K': math.Pow(1024, 1), + 'M': math.Pow(1024, 2), + 'G': math.Pow(1024, 3), + 'T': math.Pow(1024, 4), + 'P': math.Pow(1024, 5), +} + +// ParseSizeToBytes converts a Host Configuration partition size string (e.g. +// "8G", "192M", "1024") into bytes. The final character may be a unit suffix +// (B/K/M/G/T/P); a bare number is treated as bytes. Non-numeric sizes such as +// "grow" return ok=false so callers can skip the size expectation. +func ParseSizeToBytes(size string) (int64, bool) { + size = strings.TrimSpace(size) + if size == "" { + return 0, false + } + + last := size[len(size)-1] + numPart := size + multiplier := 1.0 + + if unit, isUnit := sizeUnits[last]; isUnit { + multiplier = unit + numPart = size[:len(size)-1] + } else if last < '0' || last > '9' { + // Trailing non-digit, non-unit character (e.g. "grow"): not a size. + return 0, false + } + + value, err := strconv.ParseFloat(numPart, 64) + if err != nil { + return 0, false + } + + return int64(value * multiplier), true +} + +// PartitionExpectation is a partition declared in the Host Configuration. +type PartitionExpectation struct { + ID string + SizeBytes int64 + HasSize bool +} + +// ExpectedPartitions extracts the partitions declared across all disks in the +// given Host Configuration spec. +func ExpectedPartitions(spec hostconfig.HostConfig) []PartitionExpectation { + var result []PartitionExpectation + for _, disk := range spec.S("storage", "disks").Children() { + for _, part := range disk.S("partitions").Children() { + id, ok := part.S("id").Data().(string) + if !ok { + continue + } + exp := PartitionExpectation{ID: id} + if sizeStr, ok := part.S("size").Data().(string); ok { + exp.SizeBytes, exp.HasSize = ParseSizeToBytes(sizeStr) + } + result = append(result, exp) + } + } + return result +} + +// IsPartition reports whether the block device ID corresponds to a disk +// partition declared in the spec. +func IsPartition(spec hostconfig.HostConfig, deviceID string) bool { + for _, disk := range spec.S("storage", "disks").Children() { + for _, part := range disk.S("partitions").Children() { + if id, ok := part.S("id").Data().(string); ok && id == deviceID { + return true + } + } + } + return false +} + +// IsRaid reports whether the block device ID corresponds to a software RAID +// array declared in the spec. +func IsRaid(spec hostconfig.HostConfig, deviceID string) bool { + for _, raid := range spec.S("storage", "raid", "software").Children() { + if id, ok := raid.S("id").Data().(string); ok && id == deviceID { + return true + } + } + return false +} + +// mountPointPath extracts the "/" path from a filesystem's mountPoint field, +// which may be either a plain string or an object with a "path" key. Returns +// ("", false) if no mount point is set. +func mountPointPath(fs *hostconfig.HostConfig) (string, bool) { + mp := fs.S("mountPoint") + if mp == nil || mp.Data() == nil { + return "", false + } + if str, ok := mp.Data().(string); ok { + return str, true + } + if path, ok := mp.S("path").Data().(string); ok { + return path, true + } + return "", false +} + +// RootFilesystemDeviceID returns the deviceId of the filesystem mounted at "/" +// in the spec, and whether one was found. +func RootFilesystemDeviceID(spec hostconfig.HostConfig) (string, bool) { + for _, fs := range spec.S("storage", "filesystems").Children() { + path, ok := mountPointPath(&hostconfig.HostConfig{Container: fs}) + if !ok || path != "/" { + continue + } + if id, ok := fs.S("deviceId").Data().(string); ok { + return id, true + } + } + return "", false +} + +// ActiveVolumeID resolves the active volume's block-device ID for the A/B +// volume pair whose `id` equals volumePairID, given the active A/B selection. +// Returns ("", false) if no matching volume pair exists. +func ActiveVolumeID(spec hostconfig.HostConfig, volumePairID string, active tridentutil.AbVolumeSelection) (string, bool) { + for _, pair := range spec.S("storage", "abUpdate", "volumePairs").Children() { + id, ok := pair.S("id").Data().(string) + if !ok || id != volumePairID { + continue + } + field := "volumeAId" + if active == tridentutil.AbVolumeB { + field = "volumeBId" + } + if vid, ok := pair.S(field).Data().(string); ok { + return vid, true + } + } + return "", false +} + +// verityDataDeviceID returns the dataDeviceId of the verity device whose id +// matches the given device ID, and whether such a verity device exists. +func verityDataDeviceID(spec hostconfig.HostConfig, deviceID string) (string, bool) { + for _, v := range spec.S("storage", "verity").Children() { + if id, ok := v.S("id").Data().(string); ok && id == deviceID { + if data, ok := v.S("dataDeviceId").Data().(string); ok { + return data, true + } + return "", true + } + } + return "", false +} + +// AbVolumePairID returns the A/B volume-pair ID backing the root filesystem. +// If root sits on a verity device, the pair ID is the verity data device; +// otherwise it is the root filesystem's device ID directly. The second return +// value is true when root is a verity device. +func AbVolumePairID(spec hostconfig.HostConfig, rootDeviceID string) (pairID string, rootIsVerity bool) { + if dataID, isVerity := verityDataDeviceID(spec, rootDeviceID); isVerity { + return dataID, true + } + return rootDeviceID, false +} diff --git a/tools/storm/e2e/validate/base_test.go b/tools/storm/e2e/validate/base_test.go new file mode 100644 index 0000000000..d8e35b2a75 --- /dev/null +++ b/tools/storm/e2e/validate/base_test.go @@ -0,0 +1,140 @@ +package validate + +import ( + "testing" + + "tridenttools/pkg/hostconfig" + tridentutil "tridenttools/storm/utils/trident" +) + +func TestParseSizeToBytes(t *testing.T) { + cases := []struct { + in string + want int64 + wantOk bool + }{ + {"8G", 8 * 1024 * 1024 * 1024, true}, + {"192M", 192 * 1024 * 1024, true}, + {"1K", 1024, true}, + {"512B", 512, true}, + {"1024", 1024, true}, + {"grow", 0, false}, + {"", 0, false}, + {"4.5G", int64(4.5 * 1024 * 1024 * 1024), true}, + } + for _, c := range cases { + got, ok := ParseSizeToBytes(c.in) + if ok != c.wantOk || (ok && got != c.want) { + t.Errorf("ParseSizeToBytes(%q) = (%d,%v), want (%d,%v)", c.in, got, ok, c.want, c.wantOk) + } + } +} + +const specYaml = ` +storage: + disks: + - id: os + partitions: + - id: root-a + size: 8G + - id: root-b + size: 8G + - id: esp + size: 1G + raid: + software: + - id: md-root + name: root + abUpdate: + volumePairs: + - id: root + volumeAId: root-a + volumeBId: root-b + filesystems: + - deviceId: root + mountPoint: / + - deviceId: esp + mountPoint: + path: /boot/efi + options: umask=0077 + verity: + - id: root + name: root + dataDeviceId: root-data + hashDeviceId: root-hash +` + +func mustSpec(t *testing.T) hostconfig.HostConfig { + t.Helper() + hc, err := hostconfig.NewHostConfigFromYaml([]byte(specYaml)) + if err != nil { + t.Fatalf("failed to parse spec: %v", err) + } + return hc +} + +func TestExpectedPartitions(t *testing.T) { + parts := ExpectedPartitions(mustSpec(t)) + if len(parts) != 3 { + t.Fatalf("got %d partitions, want 3", len(parts)) + } + byID := map[string]PartitionExpectation{} + for _, p := range parts { + byID[p.ID] = p + } + if !byID["root-a"].HasSize || byID["root-a"].SizeBytes != 8*1024*1024*1024 { + t.Errorf("root-a = %+v", byID["root-a"]) + } +} + +func TestIsPartitionIsRaid(t *testing.T) { + spec := mustSpec(t) + if !IsPartition(spec, "root-a") { + t.Error("root-a should be a partition") + } + if IsPartition(spec, "md-root") { + t.Error("md-root should not be a partition") + } + if !IsRaid(spec, "md-root") { + t.Error("md-root should be a raid array") + } + if IsRaid(spec, "root-a") { + t.Error("root-a should not be raid") + } +} + +func TestRootFilesystemDeviceID(t *testing.T) { + id, ok := RootFilesystemDeviceID(mustSpec(t)) + if !ok || id != "root" { + t.Errorf("got (%q,%v), want (root,true)", id, ok) + } +} + +func TestActiveVolumeID(t *testing.T) { + spec := mustSpec(t) + a, ok := ActiveVolumeID(spec, "root", tridentutil.AbVolumeA) + if !ok || a != "root-a" { + t.Errorf("volume-a: got (%q,%v), want (root-a,true)", a, ok) + } + b, ok := ActiveVolumeID(spec, "root", tridentutil.AbVolumeB) + if !ok || b != "root-b" { + t.Errorf("volume-b: got (%q,%v), want (root-b,true)", b, ok) + } + if _, ok := ActiveVolumeID(spec, "nonexistent", tridentutil.AbVolumeA); ok { + t.Error("nonexistent pair should return ok=false") + } +} + +func TestAbVolumePairID(t *testing.T) { + spec := mustSpec(t) + // root is a verity device in specYaml -> pair id is its dataDeviceId. + pairID, isVerity := AbVolumePairID(spec, "root") + if !isVerity || pairID != "root-data" { + t.Errorf("verity root: got (%q,%v), want (root-data,true)", pairID, isVerity) + } + // esp is not a verity device -> pair id is the device id itself. + pairID, isVerity = AbVolumePairID(spec, "esp") + if isVerity || pairID != "esp" { + t.Errorf("non-verity: got (%q,%v), want (esp,false)", pairID, isVerity) + } +} diff --git a/tools/storm/e2e/validate/orchestrate.go b/tools/storm/e2e/validate/orchestrate.go new file mode 100644 index 0000000000..7ce1a6b963 --- /dev/null +++ b/tools/storm/e2e/validate/orchestrate.go @@ -0,0 +1,280 @@ +package validate + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/pkg/hostconfig" + "tridenttools/storm/utils/sshutils" + "tridenttools/storm/utils/sysinspect" + tridentutil "tridenttools/storm/utils/trident" +) + +// ValidateBase runs the full `base` marker validation (partitions, users, UEFI +// fallback) against the installed host, accumulating all sub-check failures in +// sa. It ports tests/e2e_tests/base_test.py. +// +// expectedState is the servicing state the host is expected to report (normally +// "provisioned"). abActive is the expected active A/B volume; it is only used +// when the host configuration declares an A/B update. +func ValidateBase( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + expectedState tridentutil.ServicingState, + abActive tridentutil.AbVolumeSelection, +) { + spec := hs.Spec() + + ValidatePartitions(sa, client, hs, expectedState, abActive) + ValidateUsers(sa, client, spec) + ValidateUefiFallback(sa, client, spec) +} + +// ValidatePartitions ports base_test.py::test_partitions. It confirms the +// servicing state, that every configured partition is present both in the Host +// Status and on the running system, and (for A/B configs on non-verity roots) +// that the active volume's device path matches the mounted root device. +func ValidatePartitions( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + expectedState tridentutil.ServicingState, + abActive tridentutil.AbVolumeSelection, +) { + spec := hs.Spec() + + // Gather system state. + blkid, err := sysinspect.Blkid(client) + if err != nil { + sa.Fail("partitions/blkid", err) + return + } + if _, err := sysinspect.Lsblk(client); err != nil { + sa.Fail("partitions/lsblk", err) + return + } + + // Set of PARTLABELs present on the system (partitions_system_info keys in + // base_test.py). + presentPartlabels := make(map[string]struct{}) + for _, entry := range blkid { + if label, ok := entry.Get("PARTLABEL"); ok { + presentPartlabels[label] = struct{}{} + } + } + + // Servicing state. + sa.Assert("partitions/servicing-state", + hs.ServicingState() == expectedState, + "expected servicingState %q, got %q", expectedState, hs.ServicingState()) + + // Every configured partition must appear in the Host Status partitionPaths + // and on the system (as a PARTLABEL). + partitionPaths := hs.PartitionPaths() + for _, part := range ExpectedPartitions(spec) { + if _, ok := partitionPaths[part.ID]; !ok { + sa.Failf("partitions/status-path", + "partition %q missing from Host Status partitionPaths", part.ID) + } + if _, ok := presentPartlabels[part.ID]; !ok { + sa.Failf("partitions/system-present", + "partition %q (PARTLABEL) not found on system", part.ID) + } + } + + // A/B active-volume device-path cross-check (non-verity root only; verity + // A/B is covered by verity validation). + if spec.HasABUpdate() { + validateActiveVolumePath(sa, client, hs, spec, blkid, abActive) + } +} + +// validateActiveVolumePath ports the A/B branch of base_test.py::test_partitions +// for non-verity roots. +func validateActiveVolumePath( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + spec hostconfig.HostConfig, + blkid map[string]sysinspect.BlkidEntry, + abActive tridentutil.AbVolumeSelection, +) { + rootDeviceID, ok := RootFilesystemDeviceID(spec) + if !ok { + sa.Failf("partitions/ab-root", "root mount point not found in Host Status spec") + return + } + + pairID, rootIsVerity := AbVolumePairID(spec, rootDeviceID) + if rootIsVerity { + // Verity root A/B validation lives in the verity validation, not base. + return + } + + activeVolumeID, ok := ActiveVolumeID(spec, pairID, abActive) + if !ok { + sa.Failf("partitions/ab-active", "no volume pair with id %q for %s", pairID, abActive) + return + } + + isPart := IsPartition(spec, activeVolumeID) + isRaid := IsRaid(spec, activeVolumeID) + if isPart == isRaid { + sa.Failf("partitions/ab-kind", + "active volume %q must be exactly one of partition/raid (partition=%v raid=%v)", + activeVolumeID, isPart, isRaid) + return + } + + // Resolve the device path we expect Host Status to report for the active + // volume, based on the actual mounted root device. + rootMountDevice, ok := getRootMountDevice(client) + if !ok { + sa.Failf("partitions/ab-mount", "could not determine device mounted at /") + return + } + rootBasename := rootMountDevice[strings.LastIndex(rootMountDevice, "/")+1:] + + var expectedPath string + switch { + case isPart: + entry, ok := blkid[rootBasename] + if !ok { + sa.Failf("partitions/ab-blkid", "no blkid entry for root device %q", rootBasename) + return + } + partuuid, ok := entry.Get("PARTUUID") + if !ok { + sa.Failf("partitions/ab-partuuid", "root device %q has no PARTUUID", rootBasename) + return + } + expectedPath = "/dev/disk/by-partuuid/" + partuuid + case isRaid: + name, found, err := sysinspect.RaidNameForDevice(client, rootMountDevice) + if err != nil { + sa.Fail("partitions/ab-raid", err) + return + } + if !found { + sa.Failf("partitions/ab-raid", "could not resolve RAID name for %q", rootMountDevice) + return + } + expectedPath = name + } + + if actual, ok := hs.PartitionPaths()[activeVolumeID]; ok { + sa.Assert("partitions/ab-path-match", + actual == expectedPath, + "active volume %q path mismatch: Host Status has %q, expected %q", + activeVolumeID, actual, expectedPath) + } else { + sa.Failf("partitions/ab-path", "active volume %q missing from partitionPaths", activeVolumeID) + } + + // Active volume selection must match expectation. + actualVol, present := hs.AbActiveVolume() + sa.Assert("partitions/ab-active-volume", + present && actualVol == abActive, + "expected abActiveVolume %q, got %q (present=%v)", abActive, actualVol, present) +} + +// getRootMountDevice returns the device mounted at "/" from `mount` output. +func getRootMountDevice(client *ssh.Client) (string, bool) { + entries, err := sysinspect.Mount(client) + if err != nil { + return "", false + } + return sysinspect.RootDevice(entries) +} + +// ValidateUsers ports base_test.py::test_users. It confirms that every user and +// group declared in the Host Configuration exists on the system. +func ValidateUsers(sa *SoftAsserter, client *ssh.Client, spec hostconfig.HostConfig) { + systemUsers, err := sysinspect.Users(client) + if err != nil { + sa.Fail("users/passwd", err) + return + } + systemGroups, err := sysinspect.Groups(client) + if err != nil { + sa.Fail("users/group", err) + return + } + + for _, user := range spec.S("os", "users").Children() { + name, ok := user.S("name").Data().(string) + if !ok { + continue + } + if _, present := systemUsers[name]; !present { + sa.Failf("users/present", "configured user %q not found in /etc/passwd", name) + } + + for _, group := range user.S("groups").Children() { + groupName, ok := group.Data().(string) + if !ok { + continue + } + members, present := systemGroups[groupName] + if !present { + sa.Failf("users/group-present", "configured group %q not found in /etc/group", groupName) + continue + } + if _, ok := members[name]; !ok { + sa.Failf("users/group-member", "user %q not a member of group %q", name, groupName) + } + } + } +} + +// ValidateUefiFallback ports base_test.py::test_uefi_fallback. It validates the +// UEFI fallback boot entries according to the configured mode (disabled, +// conservative, optimistic; defaulting to conservative). +func ValidateUefiFallback(sa *SoftAsserter, client *ssh.Client, spec hostconfig.HostConfig) { + mode := "conservative" + if m, ok := spec.S("os", "uefiFallback").Data().(string); ok { + mode = m + } + + switch mode { + case "disabled": + // /efi/boot/EFI/BOOT should be empty. + out, err := sshutils.RunCommand(client, "sudo find /efi/boot/EFI/BOOT/* && exit 1 || exit 0") + if err != nil { + sa.Fail("uefi/disabled", err) + return + } + sa.Assert("uefi/disabled", out.Status == 0, + "/efi/boot/EFI/BOOT is not empty for disabled uefiFallback") + return + case "conservative", "optimistic": + // handled below + default: + sa.Failf("uefi/mode", "unknown uefiFallback mode: %q", mode) + return + } + + info, err := sysinspect.EfiBootMgr(client) + if err != nil { + sa.Fail("uefi/efibootmgr", err) + return + } + currentName, ok := info.CurrentName() + if !ok { + sa.Failf("uefi/current", "could not determine current boot entry name (BootCurrent=%q)", info.BootCurrent) + return + } + + // Fallback boot files should match the current boot's files. + cmd := fmt.Sprintf("sudo diff /efi/boot/EFI/BOOT/* /efi/azl/EFI/%s/* && exit 1 || exit 0", currentName) + out, err := sshutils.RunCommand(client, cmd) + if err != nil { + sa.Fail("uefi/diff", err) + return + } + sa.Assert("uefi/diff", out.Status == 0, + "UEFI fallback files differ from current boot entry %q", currentName) +} diff --git a/tools/storm/e2e/validate/softassert.go b/tools/storm/e2e/validate/softassert.go new file mode 100644 index 0000000000..0b7157e03e --- /dev/null +++ b/tools/storm/e2e/validate/softassert.go @@ -0,0 +1,77 @@ +// Package validate holds the E2E host-state validation logic ported from the +// Python pytest suite (tests/e2e_tests/*.py). Validations run over SSH against +// the installed VM and assert that the real host state matches both the Host +// Configuration and what Trident reports in its Host Status. +package validate + +import ( + "errors" + "fmt" + + "github.com/sirupsen/logrus" +) + +// SoftAsserter accumulates sub-check failures within a single storm test case +// so that every sub-check runs (and is logged individually) even after an +// earlier one fails, instead of bailing on the first failure via +// runtime.Goexit(). At the end of the case, call Err() and pass the result to +// tc.FailFromError. +// +// This is the interim approach (stage 1) chosen while storm lacks native +// soft-assert / subtest support: all sub-checks run and each failure is logged, +// but they are reported as a single failed test case (one combined message) +// rather than per-sub-check JUnit rows. See the E2E storm-port plan for the +// deferred per-subtest reporting enhancement. +type SoftAsserter struct { + errs []error +} + +// Check runs fn and, if it returns an error, records and logs it prefixed with +// name. fn is always executed; a failure never stops subsequent checks. +func (s *SoftAsserter) Check(name string, fn func() error) { + if err := fn(); err != nil { + s.record(name, err) + } +} + +// Fail records and logs a failure for the named sub-check. +func (s *SoftAsserter) Fail(name string, err error) { + s.record(name, err) +} + +// Failf records and logs a formatted failure for the named sub-check. +func (s *SoftAsserter) Failf(name, format string, args ...any) { + s.record(name, fmt.Errorf(format, args...)) +} + +// Assert records a failure with the given message if cond is false. +func (s *SoftAsserter) Assert(name string, cond bool, msgFormat string, args ...any) { + if !cond { + s.record(name, fmt.Errorf(msgFormat, args...)) + } +} + +func (s *SoftAsserter) record(name string, err error) { + wrapped := fmt.Errorf("%s: %w", name, err) + logrus.Errorf("validation sub-check failed: %v", wrapped) + s.errs = append(s.errs, wrapped) +} + +// HasFailures reports whether any sub-check has failed. +func (s *SoftAsserter) HasFailures() bool { + return len(s.errs) > 0 +} + +// Failures returns the number of failed sub-checks. +func (s *SoftAsserter) Failures() int { + return len(s.errs) +} + +// Err returns the combined error of all failed sub-checks, or nil if none +// failed. The result is suitable to pass directly to tc.FailFromError. +func (s *SoftAsserter) Err() error { + if len(s.errs) == 0 { + return nil + } + return fmt.Errorf("%d validation sub-check(s) failed: %w", len(s.errs), errors.Join(s.errs...)) +} diff --git a/tools/storm/e2e/validate/softassert_test.go b/tools/storm/e2e/validate/softassert_test.go new file mode 100644 index 0000000000..41a71ea7d7 --- /dev/null +++ b/tools/storm/e2e/validate/softassert_test.go @@ -0,0 +1,55 @@ +package validate + +import ( + "errors" + "strings" + "testing" +) + +func TestSoftAsserter_AllPass(t *testing.T) { + var sa SoftAsserter + ran := 0 + sa.Check("a", func() error { ran++; return nil }) + sa.Check("b", func() error { ran++; return nil }) + sa.Assert("c", true, "should not fire") + + if ran != 2 { + t.Errorf("ran = %d, want 2", ran) + } + if sa.HasFailures() { + t.Error("HasFailures() = true, want false") + } + if sa.Err() != nil { + t.Errorf("Err() = %v, want nil", sa.Err()) + } +} + +func TestSoftAsserter_ContinuesAfterFailure(t *testing.T) { + var sa SoftAsserter + ran := 0 + sa.Check("first", func() error { ran++; return errors.New("boom") }) + sa.Check("second", func() error { ran++; return nil }) // must still run + sa.Failf("third", "value %d bad", 7) + sa.Assert("fourth", false, "cond false") + + if ran != 2 { + t.Errorf("ran = %d, want 2 (both Check fns must execute)", ran) + } + if !sa.HasFailures() { + t.Fatal("HasFailures() = false, want true") + } + if sa.Failures() != 3 { + t.Errorf("Failures() = %d, want 3", sa.Failures()) + } + + err := sa.Err() + if err == nil { + t.Fatal("Err() = nil, want combined error") + } + msg := err.Error() + for _, want := range []string{"first: boom", "third: value 7 bad", "fourth: cond false", "3 validation sub-check(s) failed"} { + if !strings.Contains(msg, want) { + t.Errorf("Err() = %q, missing %q", msg, want) + } + } +} diff --git a/tools/storm/utils/sysinspect/blkid.go b/tools/storm/utils/sysinspect/blkid.go new file mode 100644 index 0000000000..a07a9238ef --- /dev/null +++ b/tools/storm/utils/sysinspect/blkid.go @@ -0,0 +1,79 @@ +// Package sysinspect provides parsers that gather structured host state from a +// running system over SSH. Each helper runs a standard inspection tool (blkid, +// lsblk, mount, cryptsetup, veritysetup, ...) and returns typed data for E2E +// validations. These replace the ad-hoc string parsing previously done in the +// Python E2E suite (tests/e2e_tests/*.py). +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// BlkidEntry holds the tag=value fields blkid reports for a single block +// device, e.g. UUID, TYPE, PARTLABEL, PARTUUID, LABEL, BLOCK_SIZE. +type BlkidEntry struct { + // Device is the kernel device name (e.g. "sda1"), i.e. the basename of the + // device path reported by blkid. + Device string + // Fields holds the parsed tag=value pairs (quotes stripped). + Fields map[string]string +} + +// Get returns the value of a blkid field and whether it was present. +func (e BlkidEntry) Get(field string) (string, bool) { + v, ok := e.Fields[field] + return v, ok +} + +// Blkid runs `sudo blkid` on the host and returns the parsed entries keyed by +// kernel device name (basename of the device path). +// +// Example line: +// +// /dev/sda2: UUID="04267584-..." BLOCK_SIZE="4096" TYPE="ext4" PARTLABEL="root-a" PARTUUID="f1be3a27-..." +func Blkid(client *ssh.Client) (map[string]BlkidEntry, error) { + out, err := sshutils.CommandOutput(client, "sudo blkid") + if err != nil { + return nil, fmt.Errorf("failed to run blkid: %w", err) + } + return ParseBlkid(out), nil +} + +// ParseBlkid parses the stdout of `blkid` into entries keyed by kernel device +// name. It is separated from Blkid so it can be unit-tested without SSH. +func ParseBlkid(stdout string) map[string]BlkidEntry { + entries := make(map[string]BlkidEntry) + + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // Split "device: field=val field=val ..." into device and the rest. + devicePart, rest, found := strings.Cut(line, ": ") + if !found { + continue + } + + device := devicePart[strings.LastIndex(devicePart, "/")+1:] + entry := BlkidEntry{Device: device, Fields: make(map[string]string)} + + for _, field := range strings.Fields(rest) { + key, value, ok := strings.Cut(field, "=") + if !ok { + continue + } + entry.Fields[key] = strings.Trim(value, "\"") + } + + entries[device] = entry + } + + return entries +} diff --git a/tools/storm/utils/sysinspect/efi.go b/tools/storm/utils/sysinspect/efi.go new file mode 100644 index 0000000000..9dd5e1e137 --- /dev/null +++ b/tools/storm/utils/sysinspect/efi.go @@ -0,0 +1,77 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// EfiBootInfo is the parsed result of `efibootmgr`. +type EfiBootInfo struct { + // BootCurrent is the active boot entry number (e.g. "0001"). + BootCurrent string + // EntryNames maps a boot entry number (e.g. "0001") to its label (the token + // following "Boot####*" on the entry line). + EntryNames map[string]string +} + +// CurrentName returns the label of the currently-booted entry and whether both +// BootCurrent and its label were found. +func (e EfiBootInfo) CurrentName() (string, bool) { + if e.BootCurrent == "" { + return "", false + } + name, ok := e.EntryNames[e.BootCurrent] + return name, ok +} + +// EfiBootMgr runs `sudo efibootmgr` and returns the parsed boot info. +func EfiBootMgr(client *ssh.Client) (EfiBootInfo, error) { + out, err := sshutils.CommandOutput(client, "sudo efibootmgr") + if err != nil { + return EfiBootInfo{}, fmt.Errorf("failed to run efibootmgr: %w", err) + } + return ParseEfiBootMgr(out), nil +} + +// ParseEfiBootMgr parses `efibootmgr` output, extracting BootCurrent and the +// label of each Boot#### entry. Mirrors base_test.py::test_uefi_fallback. +// +// Example: +// +// BootCurrent: 0001 +// BootOrder: 0001,0000 +// Boot0000* UiApp +// Boot0001* azl HD(1,GPT,...) +func ParseEfiBootMgr(stdout string) EfiBootInfo { + info := EfiBootInfo{EntryNames: make(map[string]string)} + + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + line = strings.TrimSpace(line) + switch { + case strings.HasPrefix(line, "BootCurrent:"): + _, value, _ := strings.Cut(line, ":") + info.BootCurrent = strings.TrimSpace(value) + + case strings.HasPrefix(line, "Boot"): + // Entry lines look like "Boot0001* label ...". Skip non-entry + // "Boot*" lines such as BootOrder/BootNext. + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + // fields[0] = "Boot0001*" (or "Boot0001"); extract the 4-hex number. + head := strings.TrimPrefix(fields[0], "Boot") + head = strings.TrimSuffix(head, "*") + if len(head) != 4 { + continue + } + info.EntryNames[head] = fields[1] + } + } + + return info +} diff --git a/tools/storm/utils/sysinspect/lsblk.go b/tools/storm/utils/sysinspect/lsblk.go new file mode 100644 index 0000000000..4461f025cd --- /dev/null +++ b/tools/storm/utils/sysinspect/lsblk.go @@ -0,0 +1,63 @@ +package sysinspect + +import ( + "encoding/json" + "fmt" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// LsblkDevice is a single node in the `lsblk -J` tree. Sizes are in bytes when +// lsblk is invoked with -b. +type LsblkDevice struct { + Name string `json:"name"` + MajMin string `json:"maj:min"` + RM bool `json:"rm"` + Size int64 `json:"size"` + RO bool `json:"ro"` + Type string `json:"type"` + Mountpoints []*string `json:"mountpoints"` + Children []LsblkDevice `json:"children,omitempty"` +} + +// LsblkOutput is the top-level structure of `lsblk -J` output. +type LsblkOutput struct { + Blockdevices []LsblkDevice `json:"blockdevices"` +} + +// Partitions flattens the block-device tree into the set of leaf devices, +// treating any block device without children as a partition (mirrors the +// flattening done in base_test.py). +func (o LsblkOutput) Partitions() []LsblkDevice { + var partitions []LsblkDevice + for _, bd := range o.Blockdevices { + if len(bd.Children) == 0 { + partitions = append(partitions, bd) + continue + } + partitions = append(partitions, bd.Children...) + } + return partitions +} + +// Lsblk runs `lsblk -J -b` on the host and returns the parsed tree with sizes +// in bytes. +func Lsblk(client *ssh.Client) (LsblkOutput, error) { + out, err := sshutils.CommandOutput(client, "lsblk -J -b") + if err != nil { + return LsblkOutput{}, fmt.Errorf("failed to run lsblk: %w", err) + } + return ParseLsblk(out) +} + +// ParseLsblk parses `lsblk -J -b` JSON output. Separated from Lsblk for unit +// testing without SSH. +func ParseLsblk(stdout string) (LsblkOutput, error) { + var parsed LsblkOutput + if err := json.Unmarshal([]byte(stdout), &parsed); err != nil { + return LsblkOutput{}, fmt.Errorf("failed to parse lsblk JSON: %w", err) + } + return parsed, nil +} diff --git a/tools/storm/utils/sysinspect/mount.go b/tools/storm/utils/sysinspect/mount.go new file mode 100644 index 0000000000..81fff8d2a9 --- /dev/null +++ b/tools/storm/utils/sysinspect/mount.go @@ -0,0 +1,58 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// MountEntry describes one line of `mount` output. +type MountEntry struct { + Device string + MountPoint string + FsType string +} + +// Mount runs `mount` on the host and returns the parsed entries. +func Mount(client *ssh.Client) ([]MountEntry, error) { + out, err := sshutils.CommandOutput(client, "mount") + if err != nil { + return nil, fmt.Errorf("failed to run mount: %w", err) + } + return ParseMount(out), nil +} + +// ParseMount parses `mount` output lines of the form +// "device on mount_point type fs_type (options)". +func ParseMount(stdout string) []MountEntry { + var entries []MountEntry + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + entry := MountEntry{ + Device: fields[0], + MountPoint: fields[2], + FsType: "unknown", + } + if len(fields) > 4 { + entry.FsType = fields[4] + } + entries = append(entries, entry) + } + return entries +} + +// RootDevice returns the device mounted at "/" and whether one was found. +func RootDevice(entries []MountEntry) (string, bool) { + for _, e := range entries { + if e.MountPoint == "/" { + return e.Device, true + } + } + return "", false +} diff --git a/tools/storm/utils/sysinspect/parsers2_test.go b/tools/storm/utils/sysinspect/parsers2_test.go new file mode 100644 index 0000000000..ddd96a5ca9 --- /dev/null +++ b/tools/storm/utils/sysinspect/parsers2_test.go @@ -0,0 +1,89 @@ +package sysinspect + +import "testing" + +const mdSample = `total 0 +lrwxrwxrwx 1 root root 8 Apr 1 22:42 home -> ../md124 +lrwxrwxrwx 1 root root 8 Apr 1 22:42 root-a -> ../md127 +lrwxrwxrwx 1 root root 8 Apr 1 22:42 root-b -> ../md125 +lrwxrwxrwx 1 root root 8 Apr 1 22:42 trident -> ../md126` + +func TestParseRaidName(t *testing.T) { + name, ok := parseRaidName(mdSample, "/dev/md127") + if !ok { + t.Fatal("expected to resolve /dev/md127") + } + if name != "/dev/md/root-a" { + t.Errorf("got %q, want /dev/md/root-a", name) + } + + // Bare name form. + name, ok = parseRaidName(mdSample, "md125") + if !ok || name != "/dev/md/root-b" { + t.Errorf("got %q (ok=%v), want /dev/md/root-b", name, ok) + } + + // Unknown device. + if _, ok := parseRaidName(mdSample, "/dev/md999"); ok { + t.Error("md999 should not resolve") + } +} + +const passwdSample = `root:x:0:0:root:/root:/bin/bash +bin:x:1:1:bin:/dev/null:/bin/false +testing-user:x:1001:1001::/home/testing-user:/bin/bash` + +const groupSample = `root:x:0: +wheel:x:10:testing-user,admin +bin:x:1:daemon +empty:x:99:` + +func TestParsePasswd(t *testing.T) { + users := ParsePasswd(passwdSample) + if len(users) != 3 { + t.Fatalf("got %d users, want 3", len(users)) + } + if _, ok := users["testing-user"]; !ok { + t.Error("testing-user missing") + } +} + +func TestParseGroup(t *testing.T) { + groups := ParseGroup(groupSample) + wheel, ok := groups["wheel"] + if !ok { + t.Fatal("wheel group missing") + } + if _, ok := wheel["testing-user"]; !ok { + t.Error("testing-user not in wheel") + } + if _, ok := wheel["admin"]; !ok { + t.Error("admin not in wheel") + } + if len(groups["empty"]) != 0 { + t.Errorf("empty group should have no members, got %v", groups["empty"]) + } +} + +const efiSample = `BootCurrent: 0001 +Timeout: 0 seconds +BootOrder: 0001,0000 +Boot0000* UiApp FvVol(...) +Boot0001* azl HD(1,GPT,abc)/File(...)` + +func TestParseEfiBootMgr(t *testing.T) { + info := ParseEfiBootMgr(efiSample) + if info.BootCurrent != "0001" { + t.Errorf("BootCurrent = %q, want 0001", info.BootCurrent) + } + name, ok := info.CurrentName() + if !ok { + t.Fatal("CurrentName not found") + } + if name != "azl" { + t.Errorf("CurrentName = %q, want azl", name) + } + if info.EntryNames["0000"] != "UiApp" { + t.Errorf("entry 0000 = %q, want UiApp", info.EntryNames["0000"]) + } +} diff --git a/tools/storm/utils/sysinspect/parsers_test.go b/tools/storm/utils/sysinspect/parsers_test.go new file mode 100644 index 0000000000..5b21360b1a --- /dev/null +++ b/tools/storm/utils/sysinspect/parsers_test.go @@ -0,0 +1,108 @@ +package sysinspect + +import "testing" + +const blkidSample = `/dev/sr0: BLOCK_SIZE="2048" UUID="2023-12-16-00-55-13-99" LABEL="TRIDENT_CDROM" TYPE="iso9660" +/dev/sda4: LABEL="3e9cecef-5a01-4" UUID="37a7b4fa-87f0-4887-895b-393f46c345a0" TYPE="swap" PARTLABEL="swap" PARTUUID="3e9cecef-5a01-43d6-a1ae-58bf24f42521" +/dev/sda2: UUID="04267584-7e18-4612-a649-c71e1811bd82" BLOCK_SIZE="4096" TYPE="ext4" PARTLABEL="root-a" PARTUUID="f1be3a27-36e2-4d4b-b8ec-5b0b5909cbf9" +/dev/sda1: SEC_TYPE="msdos" UUID="D920-8BA4" BLOCK_SIZE="512" TYPE="vfat" PARTLABEL="esp" PARTUUID="6fcc7c57-b21c-46e5-bc79-041c7fc53f34" +/dev/sda3: PARTLABEL="root-b" PARTUUID="573fdf4c-9133-4a9f-8cf5-aff7b74d1aeb"` + +func TestParseBlkid(t *testing.T) { + entries := ParseBlkid(blkidSample) + if len(entries) != 5 { + t.Fatalf("got %d entries, want 5", len(entries)) + } + + sda2, ok := entries["sda2"] + if !ok { + t.Fatal("missing sda2 entry") + } + if v, _ := sda2.Get("TYPE"); v != "ext4" { + t.Errorf("sda2 TYPE = %q, want ext4", v) + } + if v, _ := sda2.Get("PARTLABEL"); v != "root-a" { + t.Errorf("sda2 PARTLABEL = %q, want root-a", v) + } + if v, _ := sda2.Get("PARTUUID"); v != "f1be3a27-36e2-4d4b-b8ec-5b0b5909cbf9" { + t.Errorf("sda2 PARTUUID = %q", v) + } + + // sda3 has no TYPE (unformatted B volume) - Get should report absent. + sda3 := entries["sda3"] + if _, ok := sda3.Get("TYPE"); ok { + t.Error("sda3 should not have TYPE") + } + if v, _ := sda3.Get("PARTLABEL"); v != "root-b" { + t.Errorf("sda3 PARTLABEL = %q, want root-b", v) + } +} + +const lsblkSample = `{ + "blockdevices": [ + {"name":"sda","maj:min":"8:0","rm":false,"size":34359738368,"ro":false,"type":"disk","mountpoints":[null], + "children":[ + {"name":"sda1","maj:min":"8:1","rm":false,"size":1073741824,"ro":false,"type":"part","mountpoints":["/boot/efi"]}, + {"name":"sda2","maj:min":"8:2","rm":false,"size":8589934592,"ro":false,"type":"part","mountpoints":["/"]} + ]}, + {"name":"sdb","maj:min":"8:16","rm":false,"size":34359738368,"ro":false,"type":"disk","mountpoints":[null], + "children":[ + {"name":"sdb1","maj:min":"8:17","rm":false,"size":10485760,"ro":false,"type":"part","mountpoints":[null]} + ]}, + {"name":"sr0","maj:min":"11:0","rm":true,"size":501121024,"ro":false,"type":"rom","mountpoints":[null]} + ] +}` + +func TestParseLsblk(t *testing.T) { + out, err := ParseLsblk(lsblkSample) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(out.Blockdevices) != 3 { + t.Fatalf("got %d block devices, want 3", len(out.Blockdevices)) + } + + parts := out.Partitions() + // sda1, sda2, sdb1 (children) + sr0 (no children) = 4 + if len(parts) != 4 { + t.Fatalf("got %d partitions, want 4", len(parts)) + } + + byName := map[string]LsblkDevice{} + for _, p := range parts { + byName[p.Name] = p + } + if byName["sda2"].Size != 8589934592 { + t.Errorf("sda2 size = %d, want 8589934592", byName["sda2"].Size) + } + if got := byName["sda2"].Mountpoints; len(got) != 1 || got[0] == nil || *got[0] != "/" { + t.Errorf("sda2 mountpoint unexpected: %v", got) + } + if _, ok := byName["sr0"]; !ok { + t.Error("sr0 should be treated as a leaf partition") + } +} + +const mountSample = `/dev/sda3 on / type ext4 (rw,relatime) +devtmpfs on /dev type devtmpfs (rw,nosuid) +/dev/sda5 on /home type ext4 (rw,relatime) +/dev/sda1 on /boot/efi type vfat (rw,relatime)` + +func TestParseMountAndRootDevice(t *testing.T) { + entries := ParseMount(mountSample) + if len(entries) != 4 { + t.Fatalf("got %d mount entries, want 4", len(entries)) + } + + root, ok := RootDevice(entries) + if !ok { + t.Fatal("root device not found") + } + if root != "/dev/sda3" { + t.Errorf("root device = %q, want /dev/sda3", root) + } + + if entries[0].FsType != "ext4" { + t.Errorf("first entry fstype = %q, want ext4", entries[0].FsType) + } +} diff --git a/tools/storm/utils/sysinspect/raid.go b/tools/storm/utils/sysinspect/raid.go new file mode 100644 index 0000000000..efc91db6e9 --- /dev/null +++ b/tools/storm/utils/sysinspect/raid.go @@ -0,0 +1,54 @@ +package sysinspect + +import ( + "fmt" + "regexp" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// mdSymlinkRe matches an `ls -l /dev/md` line, capturing the RAID array name +// and the md device it links to, e.g. "root-a -> ../md127". +var mdSymlinkRe = regexp.MustCompile(`(\S+)\s+->\s+\.\./(md\d+)`) + +// RaidNameForDevice resolves a kernel md device path (e.g. "/dev/md127") to its +// friendly RAID array path (e.g. "/dev/md/root-a") by inspecting `ls -l /dev/md`. +// Returns ("", false) if /dev/md does not exist or no matching array is found. +// +// Mirrors base_test.py::get_raid_name_from_device_name. +func RaidNameForDevice(client *ssh.Client, deviceName string) (string, bool, error) { + // Tolerate a missing /dev/md directory (non-RAID configs) without error. + out, err := sshutils.RunCommand(client, "ls -l /dev/md") + if err != nil { + return "", false, fmt.Errorf("failed to run ls -l /dev/md: %w", err) + } + if out.Status != 0 { + // Directory absent or empty: device is not a RAID array. + return "", false, nil + } + + name, found := parseRaidName(out.Stdout, deviceName) + return name, found, nil +} + +// parseRaidName extracts the friendly RAID path for the given md device from +// `ls -l /dev/md` output. deviceName may be a full path ("/dev/md127") or bare +// name ("md127"). +func parseRaidName(stdout, deviceName string) (string, bool) { + mdName := deviceName[strings.LastIndex(deviceName, "/")+1:] + + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + matches := mdSymlinkRe.FindStringSubmatch(line) + if matches == nil { + continue + } + // matches[1] = array name, matches[2] = md device (e.g. md127) + if matches[2] == mdName { + return "/dev/md/" + matches[1], true + } + } + return "", false +} diff --git a/tools/storm/utils/sysinspect/users.go b/tools/storm/utils/sysinspect/users.go new file mode 100644 index 0000000000..0d095f2eed --- /dev/null +++ b/tools/storm/utils/sysinspect/users.go @@ -0,0 +1,70 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// Users runs `cat /etc/passwd` and returns the set of usernames on the host. +func Users(client *ssh.Client) (map[string]struct{}, error) { + out, err := sshutils.CommandOutput(client, "cat /etc/passwd") + if err != nil { + return nil, fmt.Errorf("failed to read /etc/passwd: %w", err) + } + return ParsePasswd(out), nil +} + +// ParsePasswd parses /etc/passwd content into a set of usernames (the first +// colon-separated field of each line). +func ParsePasswd(stdout string) map[string]struct{} { + users := make(map[string]struct{}) + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + name, _, _ := strings.Cut(line, ":") + users[name] = struct{}{} + } + return users +} + +// Groups runs `cat /etc/group` and returns, for each group, the set of member +// usernames listed in the final field. +func Groups(client *ssh.Client) (map[string]map[string]struct{}, error) { + out, err := sshutils.CommandOutput(client, "cat /etc/group") + if err != nil { + return nil, fmt.Errorf("failed to read /etc/group: %w", err) + } + return ParseGroup(out), nil +} + +// ParseGroup parses /etc/group content into a map of group name -> member set. +// Lines look like "wheel:x:10:testing-user,other". +func ParseGroup(stdout string) map[string]map[string]struct{} { + groups := make(map[string]map[string]struct{}) + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.Split(line, ":") + if len(parts) < 1 { + continue + } + members := make(map[string]struct{}) + if len(parts) >= 4 && parts[len(parts)-1] != "" { + for _, m := range strings.Split(parts[len(parts)-1], ",") { + if m != "" { + members[m] = struct{}{} + } + } + } + groups[parts[0]] = members + } + return groups +} diff --git a/tools/storm/utils/trident/hoststatus.go b/tools/storm/utils/trident/hoststatus.go new file mode 100644 index 0000000000..439e089057 --- /dev/null +++ b/tools/storm/utils/trident/hoststatus.go @@ -0,0 +1,131 @@ +package trident + +import ( + "fmt" + + "github.com/Jeffail/gabs/v2" + "golang.org/x/crypto/ssh" + "gopkg.in/yaml.v3" + + "tridenttools/pkg/hostconfig" +) + +// ServicingState mirrors trident_api::status::ServicingState (kebab-case). It +// is the value reported under `servicingState` in `trident get` output. +type ServicingState string + +const ( + ServicingStateNotProvisioned ServicingState = "not-provisioned" + ServicingStateCleanInstallStaged ServicingState = "clean-install-staged" + ServicingStateAbUpdateStaged ServicingState = "ab-update-staged" + ServicingStateManualRollbackAbStaged ServicingState = "manual-rollback-ab-staged" + ServicingStateManualRollbackRuntimeStaged ServicingState = "manual-rollback-runtime-staged" + ServicingStateRuntimeUpdateStaged ServicingState = "runtime-update-staged" + ServicingStateCleanInstallFinalized ServicingState = "clean-install-finalized" + ServicingStateAbUpdateFinalized ServicingState = "ab-update-finalized" + ServicingStateManualRollbackAbFinalized ServicingState = "manual-rollback-ab-finalized" + ServicingStateProvisioned ServicingState = "provisioned" + ServicingStateAbUpdateHealthCheckFailed ServicingState = "ab-update-health-check-failed" +) + +// AbVolumeSelection mirrors trident_api::status::AbVolumeSelection (kebab-case). +// It is the value reported under `abActiveVolume` in `trident get` output. +type AbVolumeSelection string + +const ( + AbVolumeA AbVolumeSelection = "volume-a" + AbVolumeB AbVolumeSelection = "volume-b" +) + +// Other returns the opposite A/B volume selection. +func (v AbVolumeSelection) Other() AbVolumeSelection { + if v == AbVolumeA { + return AbVolumeB + } + return AbVolumeA +} + +// HostStatus is a hybrid view over the YAML emitted by `trident get`. It wraps a +// gabs.Container (escape hatch for rarely-touched corners) and provides typed +// accessors for the stable core fields that E2E validations depend on. +type HostStatus struct { + *gabs.Container +} + +// NewHostStatusFromYaml parses `trident get` YAML output into a HostStatus. +// +// The Host Status embeds a Host Configuration under `spec` whose `contents` +// nodes carry custom YAML tags (e.g. `!image`). yaml.v3 decodes these into +// plain maps when the target is `map[string]any`, so no custom tag constructor +// is required (unlike the Python suite's yaml.add_multi_constructor). +func NewHostStatusFromYaml(yamlData []byte) (HostStatus, error) { + var data map[string]any + if err := yaml.Unmarshal(yamlData, &data); err != nil { + return HostStatus{}, fmt.Errorf("failed to unmarshal Host Status YAML: %w", err) + } + + return HostStatus{Container: gabs.Wrap(data)}, nil +} + +// GetHostStatus runs `trident get` on the host over the provided SSH client and +// parses the result into a HostStatus. +func GetHostStatus(runtime RuntimeType, client *ssh.Client) (HostStatus, error) { + out, err := InvokeTrident(runtime, client, nil, "get") + if err != nil { + return HostStatus{}, fmt.Errorf("failed to invoke 'trident get': %w", err) + } + if err := out.Check(); err != nil { + return HostStatus{}, fmt.Errorf("'trident get' failed: %s", out.Report()) + } + + return NewHostStatusFromYaml([]byte(out.Stdout)) +} + +// ServicingState returns the current servicing state of the host. +func (hs *HostStatus) ServicingState() ServicingState { + s, _ := hs.S("servicingState").Data().(string) + return ServicingState(s) +} + +// AbActiveVolume returns the active A/B volume and whether it is present. It is +// absent on hosts that were never A/B updated (or failed before provisioning). +func (hs *HostStatus) AbActiveVolume() (AbVolumeSelection, bool) { + s, ok := hs.S("abActiveVolume").Data().(string) + if !ok { + return "", false + } + return AbVolumeSelection(s), true +} + +// PartitionPaths returns the device path of each block device, keyed by device +// ID (the `partitionPaths` map). +func (hs *HostStatus) PartitionPaths() map[string]string { + result := make(map[string]string) + for id, child := range hs.S("partitionPaths").ChildrenMap() { + if path, ok := child.Data().(string); ok { + result[id] = path + } + } + return result +} + +// Spec returns the embedded Host Configuration (`spec`) as a HostConfig, reusing +// the existing gabs-backed configuration handling for storage introspection. +func (hs *HostStatus) Spec() hostconfig.HostConfig { + return hostconfig.NewHostConfigFromContainer(hs.S("spec")) +} + +// LastError returns the serialized YAML of the `lastError` field and whether it +// is present. Validations match substrings against it (e.g. rollback checks). +func (hs *HostStatus) LastError() (string, bool) { + container := hs.S("lastError") + if container == nil || container.Data() == nil { + return "", false + } + + raw, err := yaml.Marshal(container.Data()) + if err != nil { + return "", false + } + return string(raw), true +} diff --git a/tools/storm/utils/trident/hoststatus_test.go b/tools/storm/utils/trident/hoststatus_test.go new file mode 100644 index 0000000000..4f7a13c16c --- /dev/null +++ b/tools/storm/utils/trident/hoststatus_test.go @@ -0,0 +1,133 @@ +package trident + +import ( + "strings" + "testing" +) + +const sampleHostStatusYaml = ` +abActiveVolume: volume-a +diskUuids: + disk-0: f4265b47-09cd-4d5e-aa92-684fb783f817 +installIndex: 0 +partitionPaths: + esp: /dev/sda1 + root-a: /dev/sda2 + root-b: /dev/sda3 +servicingState: provisioned +lastError: null +spec: + storage: + abUpdate: + volumePairs: + - id: root + volumeAId: root-a + volumeBId: root-b + disks: + - device: /dev/sda + id: disk-0 + partitions: + - id: esp + size: 8M + type: esp + - id: root-a + size: 4G + type: linux-generic + image: + url: http://blob/regular.cosi + contents: !image + sha256: abc123 + length: 705090048 +` + +func TestNewHostStatusFromYaml_CoreFields(t *testing.T) { + hs, err := NewHostStatusFromYaml([]byte(sampleHostStatusYaml)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if got := hs.ServicingState(); got != ServicingStateProvisioned { + t.Errorf("ServicingState() = %q, want %q", got, ServicingStateProvisioned) + } + + vol, present := hs.AbActiveVolume() + if !present { + t.Fatal("AbActiveVolume() reported absent, want present") + } + if vol != AbVolumeA { + t.Errorf("AbActiveVolume() = %q, want %q", vol, AbVolumeA) + } + + paths := hs.PartitionPaths() + if len(paths) != 3 { + t.Errorf("PartitionPaths() len = %d, want 3", len(paths)) + } + if paths["esp"] != "/dev/sda1" { + t.Errorf("PartitionPaths()[esp] = %q, want /dev/sda1", paths["esp"]) + } + if paths["root-a"] != "/dev/sda2" { + t.Errorf("PartitionPaths()[root-a] = %q, want /dev/sda2", paths["root-a"]) + } +} + +func TestNewHostStatusFromYaml_CustomImageTag(t *testing.T) { + // The `!image` tag on `spec.image.contents` must decode into a plain map, + // reachable via the gabs escape hatch through Spec(). + hs, err := NewHostStatusFromYaml([]byte(sampleHostStatusYaml)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + spec := hs.Spec() + if got := spec.S("image", "contents", "sha256").Data(); got != "abc123" { + t.Errorf("spec image contents sha256 = %v, want abc123", got) + } + if !spec.HasABUpdate() { + t.Error("Spec().HasABUpdate() = false, want true") + } +} + +func TestHostStatus_AbActiveVolumeAbsent(t *testing.T) { + hs, err := NewHostStatusFromYaml([]byte("servicingState: not-provisioned\n")) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if got := hs.ServicingState(); got != ServicingStateNotProvisioned { + t.Errorf("ServicingState() = %q, want %q", got, ServicingStateNotProvisioned) + } + if _, present := hs.AbActiveVolume(); present { + t.Error("AbActiveVolume() reported present, want absent") + } +} + +func TestHostStatus_LastError(t *testing.T) { + hs, err := NewHostStatusFromYaml([]byte(sampleHostStatusYaml)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, present := hs.LastError(); present { + t.Error("null lastError should be reported as absent") + } + + withErr, err := NewHostStatusFromYaml([]byte("lastError:\n message: Failed health check(s)\n")) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + msg, present := withErr.LastError() + if !present { + t.Fatal("LastError() reported absent, want present") + } + if !strings.Contains(msg, "Failed health check(s)") { + t.Errorf("LastError() = %q, want it to contain 'Failed health check(s)'", msg) + } +} + +func TestAbVolumeSelection_Other(t *testing.T) { + if got := AbVolumeA.Other(); got != AbVolumeB { + t.Errorf("AbVolumeA.Other() = %q, want %q", got, AbVolumeB) + } + if got := AbVolumeB.Other(); got != AbVolumeA { + t.Errorf("AbVolumeB.Other() = %q, want %q", got, AbVolumeA) + } +} From 8b8d3e2da0fe637dfd16ff7da29f5eb0f2a573fd Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 24 Jul 2026 16:55:51 -0700 Subject: [PATCH 02/41] storm/e2e: port extensions host-state validation Port extensions_test.py: for each configured sysext/confext, verify the extension path exists on the host and that the extension is active per 'systemd- status --json'. Self-selects when the Host Config declares sysexts/confexts. - storm/utils/sysinspect/systemd_ext.go: systemd-sysext/confext status JSON parser. - storm/e2e/validate/extensions.go: ValidateExtensions + HasExtensions. - scenario/validate.go: wire extensions into validate-* cases. Unit-tested; storm-trident builds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/storm/e2e/scenario/validate.go | 10 ++- tools/storm/e2e/validate/extensions.go | 87 +++++++++++++++++++ tools/storm/e2e/validate/extensions_test.go | 17 ++++ tools/storm/utils/sysinspect/systemd_ext.go | 50 +++++++++++ .../utils/sysinspect/systemd_ext_test.go | 24 +++++ 5 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 tools/storm/e2e/validate/extensions.go create mode 100644 tools/storm/e2e/validate/extensions_test.go create mode 100644 tools/storm/utils/sysinspect/systemd_ext.go create mode 100644 tools/storm/utils/sysinspect/systemd_ext_test.go diff --git a/tools/storm/e2e/scenario/validate.go b/tools/storm/e2e/scenario/validate.go index 3bd5010180..dd31bf074e 100644 --- a/tools/storm/e2e/scenario/validate.go +++ b/tools/storm/e2e/scenario/validate.go @@ -37,8 +37,14 @@ func (s *TridentE2EScenario) validateHostState(tc storm.TestCase) error { // `base` validation always applies. validate.ValidateBase(&sa, s.sshClient, hs, trident.ServicingStateProvisioned, s.expectedActiveVolume) - // Future markers (encryption, verity, extensions) will self-select here - // based on the Host Configuration. + // `extensions` validation self-selects when the Host Config declares + // sysexts/confexts. + if validate.HasExtensions(hs) { + validate.ValidateExtensions(&sa, s.sshClient, hs) + } + + // Future markers (encryption, verity) will self-select here based on the + // Host Configuration. if err := sa.Err(); err != nil { tc.FailFromError(err) diff --git a/tools/storm/e2e/validate/extensions.go b/tools/storm/e2e/validate/extensions.go new file mode 100644 index 0000000000..2c5d67f88e --- /dev/null +++ b/tools/storm/e2e/validate/extensions.go @@ -0,0 +1,87 @@ +package validate + +import ( + "fmt" + "path/filepath" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" + "tridenttools/storm/utils/sysinspect" + tridentutil "tridenttools/storm/utils/trident" +) + +// extensionKinds maps a Host Configuration `os` extension list key to the +// systemd extension type used on the command line. +var extensionKinds = map[string]string{ + "sysexts": "sysext", + "confexts": "confext", +} + +// HasExtensions reports whether the Host Status spec declares any system +// extensions (sysexts or confexts), used to self-select the extensions +// validation. +func HasExtensions(hs tridentutil.HostStatus) bool { + os := hs.Spec().S("os") + for key := range extensionKinds { + if os.Exists(key) { + return true + } + } + return false +} + +// ValidateExtensions ports extensions_test.py::test_extensions. For each +// configured sysext/confext it confirms the extension path exists on the host +// and that the extension is active per `systemd- status`. +func ValidateExtensions(sa *SoftAsserter, client *ssh.Client, hs tridentutil.HostStatus) { + os := hs.Spec().S("os") + + for listKey, extType := range extensionKinds { + configured := os.S(listKey).Children() + if len(configured) == 0 { + continue + } + + active, err := sysinspect.SystemdExtStatus(client, extType) + if err != nil { + sa.Fail(fmt.Sprintf("extensions/%s-status", extType), err) + continue + } + + for _, ext := range configured { + path, ok := ext.S("path").Data().(string) + if !ok { + sa.Failf(fmt.Sprintf("extensions/%s-path", extType), + "configured %s entry has no path", extType) + continue + } + + // Verify the extension path exists on the target OS. + out, err := sshutils.RunCommand(client, fmt.Sprintf("test -e %s", path)) + if err != nil { + sa.Fail(fmt.Sprintf("extensions/%s-exists", extType), err) + } else { + sa.Assert(fmt.Sprintf("extensions/%s-exists", extType), + out.Status == 0, "%s path does not exist: %s", extType, path) + } + + // The active extension name is the file stem (basename minus its + // final extension), matching Python's Path.stem. + name := extensionStem(path) + _, isActive := active[name] + sa.Assert(fmt.Sprintf("extensions/%s-active", extType), + isActive, "%s %q not found in 'systemd-%s status'", extType, name, extType) + } + } +} + +// extensionStem returns the basename of a path with its final extension +// removed, matching Python's pathlib.Path.stem (e.g. "/x/foo.raw" -> "foo", +// "/x/foo.bar.raw" -> "foo.bar"). +func extensionStem(path string) string { + base := filepath.Base(path) + ext := filepath.Ext(base) + return strings.TrimSuffix(base, ext) +} diff --git a/tools/storm/e2e/validate/extensions_test.go b/tools/storm/e2e/validate/extensions_test.go new file mode 100644 index 0000000000..eb31de952f --- /dev/null +++ b/tools/storm/e2e/validate/extensions_test.go @@ -0,0 +1,17 @@ +package validate + +import "testing" + +func TestExtensionStem(t *testing.T) { + cases := map[string]string{ + "/var/lib/ext/foo.raw": "foo", + "/var/lib/ext/foo.bar.raw": "foo.bar", + "foo": "foo", + "/a/b/c.sysext.raw": "c.sysext", + } + for in, want := range cases { + if got := extensionStem(in); got != want { + t.Errorf("extensionStem(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/tools/storm/utils/sysinspect/systemd_ext.go b/tools/storm/utils/sysinspect/systemd_ext.go new file mode 100644 index 0000000000..fc9b8538e4 --- /dev/null +++ b/tools/storm/utils/sysinspect/systemd_ext.go @@ -0,0 +1,50 @@ +package sysinspect + +import ( + "encoding/json" + "fmt" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// systemdExtHierarchy is one entry of `systemd-sysext status --json` / +// `systemd-confext status --json` output: a hierarchy (e.g. /usr, /opt, /etc) +// with the list of extensions currently merged into it. +type systemdExtHierarchy struct { + Hierarchy string `json:"hierarchy"` + Extensions []string `json:"extensions"` +} + +// SystemdExtStatus runs `systemd- status --json=pretty` on the host and +// returns the set of active extension names across all hierarchies. extType is +// "sysext" or "confext". +func SystemdExtStatus(client *ssh.Client, extType string) (map[string]struct{}, error) { + cmd := fmt.Sprintf("sudo systemd-%s status --json=pretty --no-pager", extType) + out, err := sshutils.RunCommand(client, cmd) + if err != nil { + return nil, fmt.Errorf("failed to run systemd-%s status: %w", extType, err) + } + if err := out.Check(); err != nil { + return nil, fmt.Errorf("systemd-%s status failed: %s", extType, out.Report()) + } + return ParseSystemdExtStatus(out.Stdout) +} + +// ParseSystemdExtStatus parses `systemd-sysext/confext status --json` output +// into the set of active extension names. Separated for unit testing. +func ParseSystemdExtStatus(stdout string) (map[string]struct{}, error) { + var hierarchies []systemdExtHierarchy + if err := json.Unmarshal([]byte(stdout), &hierarchies); err != nil { + return nil, fmt.Errorf("failed to parse systemd extension status JSON: %w", err) + } + + active := make(map[string]struct{}) + for _, h := range hierarchies { + for _, ext := range h.Extensions { + active[ext] = struct{}{} + } + } + return active, nil +} diff --git a/tools/storm/utils/sysinspect/systemd_ext_test.go b/tools/storm/utils/sysinspect/systemd_ext_test.go new file mode 100644 index 0000000000..a4ef91e866 --- /dev/null +++ b/tools/storm/utils/sysinspect/systemd_ext_test.go @@ -0,0 +1,24 @@ +package sysinspect + +import "testing" + +const sysextStatusSample = `[ + {"hierarchy":"/opt","extensions":["myext"]}, + {"hierarchy":"/usr","extensions":["myext","other"]}, + {"hierarchy":"/var","extensions":null} +]` + +func TestParseSystemdExtStatus(t *testing.T) { + active, err := ParseSystemdExtStatus(sysextStatusSample) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(active) != 2 { + t.Fatalf("got %d active exts, want 2 (deduped)", len(active)) + } + for _, want := range []string{"myext", "other"} { + if _, ok := active[want]; !ok { + t.Errorf("missing active ext %q", want) + } + } +} From 3f0d16c370ee64cda0b5609345d2aa8d2b59ef37 Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 24 Jul 2026 17:01:58 -0700 Subject: [PATCH 03/41] storm/e2e: port rollback and ab_update_staged validation - rollback (rollback_test.py): validate rolled-back servicing state, health-check lastError, active volume (absent when not-provisioned), and the health-check failure log messages. Self-selects via a top-level 'health' section in the Host Config (HasRollbackIntent), replacing base validation and expecting not-provisioned. - ab_update_staged (ab_update_staged_test.py): validate the staged servicing state and unchanged active volume. Wired inline into the split A/B flow between stage and finalize. Unit-tested; storm-trident builds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/storm/e2e/scenario/ab_update.go | 13 +++ tools/storm/e2e/scenario/validate.go | 28 +++--- tools/storm/e2e/validate/rollback.go | 102 ++++++++++++++++++++++ tools/storm/e2e/validate/rollback_test.go | 67 ++++++++++++++ tools/storm/e2e/validate/staged.go | 25 ++++++ 5 files changed, 225 insertions(+), 10 deletions(-) create mode 100644 tools/storm/e2e/validate/rollback.go create mode 100644 tools/storm/e2e/validate/rollback_test.go create mode 100644 tools/storm/e2e/validate/staged.go diff --git a/tools/storm/e2e/scenario/ab_update.go b/tools/storm/e2e/scenario/ab_update.go index 521b9d9c4c..f422626ec2 100644 --- a/tools/storm/e2e/scenario/ab_update.go +++ b/tools/storm/e2e/scenario/ab_update.go @@ -17,6 +17,7 @@ import ( "tridenttools/pkg/netlaunch" "tridenttools/pkg/netlisten" "tridenttools/storm/e2e/testrings" + "tridenttools/storm/e2e/validate" "tridenttools/storm/utils/ssh/sftp" "tridenttools/storm/utils/sshutils" "tridenttools/storm/utils/trident" @@ -269,6 +270,18 @@ func (s *TridentE2EScenario) abUpdateOs(tc storm.TestCase, split bool) error { return fmt.Errorf("failed to run Trident A/B update: %w", err) } + // Between stage and finalize, validate the staged state. The active + // volume must not have changed yet (it flips only after finalize+reboot). + stagedHs, err := trident.GetHostStatus(s.runtime, s.sshClient) + if err != nil { + return fmt.Errorf("failed to get Host Status after staging A/B update: %w", err) + } + var sa validate.SoftAsserter + validate.ValidateAbUpdateStaged(&sa, stagedHs, s.expectedActiveVolume) + if stagedErr := sa.Err(); stagedErr != nil { + tc.FailFromError(stagedErr) + } + logrus.Infof("Running split Trident A/B update (finalize)...") err = runTridentUpdate(tc, s.runtime, s.sshClient, args+" --allowed-operations finalize", false) if err != nil { diff --git a/tools/storm/e2e/scenario/validate.go b/tools/storm/e2e/scenario/validate.go index dd31bf074e..b619953986 100644 --- a/tools/storm/e2e/scenario/validate.go +++ b/tools/storm/e2e/scenario/validate.go @@ -34,18 +34,26 @@ func (s *TridentE2EScenario) validateHostState(tc storm.TestCase) error { var sa validate.SoftAsserter - // `base` validation always applies. - validate.ValidateBase(&sa, s.sshClient, hs, trident.ServicingStateProvisioned, s.expectedActiveVolume) - - // `extensions` validation self-selects when the Host Config declares - // sysexts/confexts. - if validate.HasExtensions(hs) { - validate.ValidateExtensions(&sa, s.sshClient, hs) + if validate.HasRollbackIntent(hs) { + // Health-check rollback scenarios (e.g. health-checks-install) replace + // base validation with rollback validation and expect the host to have + // rolled back rather than reached the provisioned state. + validate.ValidateRollback(&sa, s.sshClient, hs, + trident.ServicingStateNotProvisioned, s.expectedActiveVolume) + } else { + // `base` validation always applies. + validate.ValidateBase(&sa, s.sshClient, hs, trident.ServicingStateProvisioned, s.expectedActiveVolume) + + // `extensions` validation self-selects when the Host Config declares + // sysexts/confexts. + if validate.HasExtensions(hs) { + validate.ValidateExtensions(&sa, s.sshClient, hs) + } + + // Future markers (encryption, verity) will self-select here based on + // the Host Configuration. } - // Future markers (encryption, verity) will self-select here based on the - // Host Configuration. - if err := sa.Err(); err != nil { tc.FailFromError(err) } diff --git a/tools/storm/e2e/validate/rollback.go b/tools/storm/e2e/validate/rollback.go new file mode 100644 index 0000000000..e45e4fc33c --- /dev/null +++ b/tools/storm/e2e/validate/rollback.go @@ -0,0 +1,102 @@ +package validate + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" + tridentutil "tridenttools/storm/utils/trident" +) + +// Health-check rollback failure log location and the messages the +// health-checks-install scenario is expected to produce. These strings are +// properties of that scenario's Host Configuration health checks (scripts +// `invoke-rollback-from-script` referencing two non-existent services), mirrored +// from rollback_test.py. +const ( + healthCheckFailureLogGlob = "/var/lib/trident/trident-health-check-failure-*.log" + rollbackFailedHealthError = "Failed health check(s)" +) + +var expectedRollbackLogMessages = []string{ + "Script 'invoke-rollback-from-script' failed", + "Unit non-existent-service1.service could not be found", + "Unit non-existent-service2.service could not be found", +} + +// HasRollbackIntent reports whether the scenario is expected to trigger a +// health-check rollback, signalled by a top-level `health` section in the Host +// Configuration. Such scenarios replace `base` validation with rollback +// validation. +func HasRollbackIntent(hs tridentutil.HostStatus) bool { + return hs.Spec().Exists("health") +} + +// ValidateRollback ports rollback_test.py::test_rollback. It confirms the host +// reached the expected (rolled-back) servicing state, that the last error +// reflects a failed health check, that the active volume is unchanged (or +// absent when not provisioned), and that the health-check failure log records +// the expected script/service failures. +func ValidateRollback( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + expectedState tridentutil.ServicingState, + abActive tridentutil.AbVolumeSelection, +) { + sa.Assert("rollback/servicing-state", + hs.ServicingState() == expectedState, + "expected servicingState %q, got %q", expectedState, hs.ServicingState()) + + if lastErr, ok := hs.LastError(); ok { + sa.Assert("rollback/last-error", + strings.Contains(lastErr, rollbackFailedHealthError), + "lastError does not contain %q: %s", rollbackFailedHealthError, lastErr) + } else { + sa.Failf("rollback/last-error", "expected a lastError reflecting a failed health check") + } + + if expectedState == tridentutil.ServicingStateNotProvisioned { + if _, present := hs.AbActiveVolume(); present { + sa.Failf("rollback/active-volume", "abActiveVolume should be absent when not provisioned") + } + } else { + actual, present := hs.AbActiveVolume() + sa.Assert("rollback/active-volume", + present && actual == abActive, + "expected abActiveVolume %q, got %q (present=%v)", abActive, actual, present) + } + + validateRollbackLogs(sa, client) +} + +// validateRollbackLogs checks that exactly one health-check failure log exists +// and that it records the expected failure messages. +func validateRollbackLogs(sa *SoftAsserter, client *ssh.Client) { + listOut, err := sshutils.CommandOutput(client, "sudo ls "+healthCheckFailureLogGlob) + if err != nil { + sa.Fail("rollback/log-list", err) + return + } + + logFiles := strings.Fields(strings.TrimSpace(listOut)) + if len(logFiles) != 1 { + sa.Failf("rollback/log-count", "expected exactly 1 health-check failure log, found %d: %v", + len(logFiles), logFiles) + return + } + + content, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo cat %s", logFiles[0])) + if err != nil { + sa.Fail("rollback/log-read", err) + return + } + + for _, want := range expectedRollbackLogMessages { + sa.Assert("rollback/log-message", + strings.Contains(content, want), + "health-check failure log missing message %q", want) + } +} diff --git a/tools/storm/e2e/validate/rollback_test.go b/tools/storm/e2e/validate/rollback_test.go new file mode 100644 index 0000000000..dfa0f373a2 --- /dev/null +++ b/tools/storm/e2e/validate/rollback_test.go @@ -0,0 +1,67 @@ +package validate + +import ( + "strings" + "testing" + + tridentutil "tridenttools/storm/utils/trident" +) + +func TestHasRollbackIntent(t *testing.T) { + withHealth, _ := tridentutil.NewHostStatusFromYaml([]byte("spec:\n health:\n healthChecks: []\n")) + if !HasRollbackIntent(withHealth) { + t.Error("expected rollback intent when spec.health present") + } + withoutHealth, _ := tridentutil.NewHostStatusFromYaml([]byte("spec:\n storage: {}\n")) + if HasRollbackIntent(withoutHealth) { + t.Error("did not expect rollback intent without spec.health") + } +} + +// TestRollbackHostStatusChecks exercises the host-status portion of the rollback +// contract (state, absent active volume, health-check lastError) that +// ValidateRollback asserts; the log-file portion requires SSH and is covered by +// integration runs. +func TestRollbackHostStatusChecks(t *testing.T) { + hs, _ := tridentutil.NewHostStatusFromYaml([]byte( + "servicingState: not-provisioned\nlastError:\n message: Failed health check(s)\nspec:\n health: {}\n")) + + if hs.ServicingState() != tridentutil.ServicingStateNotProvisioned { + t.Errorf("state = %q, want not-provisioned", hs.ServicingState()) + } + if _, present := hs.AbActiveVolume(); present { + t.Error("abActiveVolume should be absent when not provisioned") + } + le, ok := hs.LastError() + if !ok || !strings.Contains(le, rollbackFailedHealthError) { + t.Errorf("lastError = %q, want it to contain %q", le, rollbackFailedHealthError) + } +} + +func TestValidateAbUpdateStaged(t *testing.T) { + hs, _ := tridentutil.NewHostStatusFromYaml([]byte( + "servicingState: ab-update-staged\nabActiveVolume: volume-a\n")) + var sa SoftAsserter + ValidateAbUpdateStaged(&sa, hs, tridentutil.AbVolumeA) + if sa.HasFailures() { + t.Errorf("expected no failures, got: %v", sa.Err()) + } + + // Wrong state -> failure. + bad, _ := tridentutil.NewHostStatusFromYaml([]byte( + "servicingState: provisioned\nabActiveVolume: volume-a\n")) + var sa2 SoftAsserter + ValidateAbUpdateStaged(&sa2, bad, tridentutil.AbVolumeA) + if !sa2.HasFailures() { + t.Error("expected failure for non-staged state") + } + + // Volume already flipped -> failure. + flipped, _ := tridentutil.NewHostStatusFromYaml([]byte( + "servicingState: ab-update-staged\nabActiveVolume: volume-b\n")) + var sa3 SoftAsserter + ValidateAbUpdateStaged(&sa3, flipped, tridentutil.AbVolumeA) + if !sa3.HasFailures() { + t.Error("expected failure when active volume already changed") + } +} diff --git a/tools/storm/e2e/validate/staged.go b/tools/storm/e2e/validate/staged.go new file mode 100644 index 0000000000..89c37c8f8a --- /dev/null +++ b/tools/storm/e2e/validate/staged.go @@ -0,0 +1,25 @@ +package validate + +import ( + tridentutil "tridenttools/storm/utils/trident" +) + +// ValidateAbUpdateStaged ports ab_update_staged_test.py::test_ab_update_staged. +// It confirms that, after staging an A/B update but before finalizing, the host +// reports the staged servicing state and that the active volume has not yet +// changed. abActive is the volume expected to still be active before finalize. +func ValidateAbUpdateStaged( + sa *SoftAsserter, + hs tridentutil.HostStatus, + abActive tridentutil.AbVolumeSelection, +) { + sa.Assert("ab-staged/servicing-state", + hs.ServicingState() == tridentutil.ServicingStateAbUpdateStaged, + "expected servicingState %q, got %q", + tridentutil.ServicingStateAbUpdateStaged, hs.ServicingState()) + + actual, present := hs.AbActiveVolume() + sa.Assert("ab-staged/active-volume", + present && actual == abActive, + "expected abActiveVolume %q (unchanged), got %q (present=%v)", abActive, actual, present) +} From 756b0be9af88208cd273df2ca8e34d77fe77a9fc Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 24 Jul 2026 17:06:25 -0700 Subject: [PATCH 04/41] storm/e2e: port verity host-state validation Port verity_test.py::test_verity_root: confirm /dev/mapper/root exists and veritysetup reports it active/verified/read-only, then validate the data and hash devices correspond to the expected block devices, handling both A/B (active volume of data/hash pairs) and non-A/B configs, and both partition and RAID backing devices. Self-selects when the Host Config declares a verity device. - storm/utils/sysinspect/veritysetup.go: veritysetup status parser. - storm/utils/sysinspect/blkid.go: add full device Path to BlkidEntry. - storm/e2e/validate/verity.go: ValidateVerity + HasVerity. - scenario/validate.go: wire verity into validate-* cases. Unit-tested; storm-trident builds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/storm/e2e/scenario/validate.go | 10 +- tools/storm/e2e/validate/verity.go | 218 ++++++++++++++++++ tools/storm/e2e/validate/verity_test.go | 55 +++++ tools/storm/utils/sysinspect/blkid.go | 5 +- tools/storm/utils/sysinspect/veritysetup.go | 65 ++++++ .../utils/sysinspect/veritysetup_test.go | 47 ++++ 6 files changed, 397 insertions(+), 3 deletions(-) create mode 100644 tools/storm/e2e/validate/verity.go create mode 100644 tools/storm/e2e/validate/verity_test.go create mode 100644 tools/storm/utils/sysinspect/veritysetup.go create mode 100644 tools/storm/utils/sysinspect/veritysetup_test.go diff --git a/tools/storm/e2e/scenario/validate.go b/tools/storm/e2e/scenario/validate.go index b619953986..fdd008e493 100644 --- a/tools/storm/e2e/scenario/validate.go +++ b/tools/storm/e2e/scenario/validate.go @@ -50,8 +50,14 @@ func (s *TridentE2EScenario) validateHostState(tc storm.TestCase) error { validate.ValidateExtensions(&sa, s.sshClient, hs) } - // Future markers (encryption, verity) will self-select here based on - // the Host Configuration. + // `verity` validation self-selects when the Host Config declares a + // verity device. + if validate.HasVerity(hs) { + validate.ValidateVerity(&sa, s.sshClient, hs, s.expectedActiveVolume) + } + + // Future markers (encryption) will self-select here based on the Host + // Configuration. } if err := sa.Err(); err != nil { diff --git a/tools/storm/e2e/validate/verity.go b/tools/storm/e2e/validate/verity.go new file mode 100644 index 0000000000..44240d622a --- /dev/null +++ b/tools/storm/e2e/validate/verity.go @@ -0,0 +1,218 @@ +package validate + +import ( + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/pkg/hostconfig" + "tridenttools/storm/utils/sysinspect" + tridentutil "tridenttools/storm/utils/trident" +) + +// VerityDevice describes a verity device declared in the Host Status spec. +type VerityDevice struct { + ID string + Name string + DataDeviceID string + HashDeviceID string +} + +// HasVerity reports whether the spec declares any verity device, used to +// self-select verity validation. +func HasVerity(hs tridentutil.HostStatus) bool { + return hs.Spec().Exists("storage", "verity") +} + +// verityForRoot returns the verity device whose id matches the root +// filesystem's device ID, and whether it was found. +func verityForRoot(spec hostconfig.HostConfig, rootDeviceID string) (VerityDevice, bool) { + for _, v := range spec.S("storage", "verity").Children() { + id, _ := v.S("id").Data().(string) + if id != rootDeviceID { + continue + } + dev := VerityDevice{ID: id} + dev.Name, _ = v.S("name").Data().(string) + dev.DataDeviceID, _ = v.S("dataDeviceId").Data().(string) + dev.HashDeviceID, _ = v.S("hashDeviceId").Data().(string) + return dev, true + } + return VerityDevice{}, false +} + +// ValidateVerity ports verity_test.py::test_verity_root. It confirms the root +// verity device mapper exists and is active/verified/read-only, then validates +// that its data and hash devices correspond to the expected block devices +// (accounting for A/B updates and RAID arrays). +func ValidateVerity( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + abActive tridentutil.AbVolumeSelection, +) { + spec := hs.Spec() + + blkid, err := sysinspect.Blkid(client) + if err != nil { + sa.Fail("verity/blkid", err) + return + } + if !blkidHasPath(blkid, "/dev/mapper/root") { + sa.Failf("verity/mapper", "/dev/mapper/root not present in blkid output") + } + + // Locate the root verity device from the Host Status. + rootDeviceID, ok := RootFilesystemDeviceID(spec) + if !ok { + sa.Failf("verity/root-mount", "root mount point not found in Host Status spec") + return + } + verity, ok := verityForRoot(spec, rootDeviceID) + if !ok || verity.HashDeviceID == "" { + sa.Failf("verity/config", "no verity configuration found for root device %q", rootDeviceID) + return + } + + name := verity.Name + if name == "" { + name = "root" + } + + // veritysetup status must show an active, verified, read-only device. + status, err := sysinspect.VeritySetup(client, name) + if err != nil { + sa.Fail("verity/status", err) + return + } + sa.Assert("verity/active", status.Active, "verity device %q is not active/in-use", name) + assertVerityField(sa, status, "type", "VERITY") + assertVerityField(sa, status, "status", "verified") + assertVerityField(sa, status, "mode", "readonly") + + dataDevice, dataOk := status.Get("data device") + hashDevice, hashOk := status.Get("hash device") + if !dataOk || !hashOk { + sa.Failf("verity/devices", "veritysetup status missing data/hash device fields") + return + } + + if spec.HasABUpdate() { + validateVerityAbDevices(sa, client, spec, blkid, verity, dataDevice, hashDevice, abActive) + } else { + validateVerityNonAbDevices(sa, client, blkid, verity, dataDevice, hashDevice) + } +} + +// validateVerityAbDevices validates the A/B branch: the veritysetup data/hash +// devices must correspond to the active volume of the data/hash A/B pairs. +func validateVerityAbDevices( + sa *SoftAsserter, + client *ssh.Client, + spec hostconfig.HostConfig, + blkid map[string]sysinspect.BlkidEntry, + verity VerityDevice, + dataDevice, hashDevice string, + abActive tridentutil.AbVolumeSelection, +) { + activeDataID, dataFound := ActiveVolumeID(spec, verity.DataDeviceID, abActive) + activeHashID, hashFound := ActiveVolumeID(spec, verity.HashDeviceID, abActive) + if !dataFound || !hashFound { + sa.Failf("verity/ab-pair", "could not resolve active data/hash volume for %s", abActive) + return + } + + dataRaid, _, _ := sysinspect.RaidNameForDevice(client, dataDevice) + hashRaid, _, _ := sysinspect.RaidNameForDevice(client, hashDevice) + dataIsRaid := dataRaid != "" + hashIsRaid := hashRaid != "" + if dataIsRaid != hashIsRaid { + sa.Failf("verity/ab-raid-parity", + "data/hash RAID mismatch: data raid=%v hash raid=%v", dataIsRaid, hashIsRaid) + return + } + + if dataIsRaid { + sa.Assert("verity/ab-data-raid", basename(dataRaid) == activeDataID, + "active data volume %q != raid %q", activeDataID, basename(dataRaid)) + sa.Assert("verity/ab-hash-raid", basename(hashRaid) == activeHashID, + "active hash volume %q != raid %q", activeHashID, basename(hashRaid)) + return + } + + // Partition case: PARTLABEL of the block device must equal the active ID. + assertPartlabelEquals(sa, blkid, dataDevice, activeDataID, "verity/ab-data-partlabel") + assertPartlabelEquals(sa, blkid, hashDevice, activeHashID, "verity/ab-hash-partlabel") +} + +// validateVerityNonAbDevices validates the non-A/B branch: the veritysetup +// data/hash devices must correspond to the configured data/hash device IDs. +func validateVerityNonAbDevices( + sa *SoftAsserter, + client *ssh.Client, + blkid map[string]sysinspect.BlkidEntry, + verity VerityDevice, + dataDevice, hashDevice string, +) { + dataRaid, _, _ := sysinspect.RaidNameForDevice(client, dataDevice) + hashRaid, _, _ := sysinspect.RaidNameForDevice(client, hashDevice) + dataIsRaid := dataRaid != "" + hashIsRaid := hashRaid != "" + if dataIsRaid != hashIsRaid { + sa.Failf("verity/raid-parity", + "data/hash RAID mismatch: data raid=%v hash raid=%v", dataIsRaid, hashIsRaid) + return + } + + if dataIsRaid { + sa.Assert("verity/data-raid", basename(dataRaid) == verity.DataDeviceID, + "data device id %q != raid %q", verity.DataDeviceID, basename(dataRaid)) + sa.Assert("verity/hash-raid", basename(hashRaid) == verity.HashDeviceID, + "hash device id %q != raid %q", verity.HashDeviceID, basename(hashRaid)) + return + } + + // Partition case: base_test's non-A/B branch only asserts presence in blkid. + if !blkidHasDevice(blkid, basename(dataDevice)) { + sa.Failf("verity/data-present", "verity data device %q not present in blkid", dataDevice) + } + if !blkidHasDevice(blkid, basename(hashDevice)) { + sa.Failf("verity/hash-present", "verity hash device %q not present in blkid", hashDevice) + } +} + +func assertVerityField(sa *SoftAsserter, status sysinspect.VeritySetupStatus, field, want string) { + got, ok := status.Get(field) + sa.Assert("verity/"+field, ok && got == want, + "veritysetup %s = %q, want %q", field, got, want) +} + +func assertPartlabelEquals(sa *SoftAsserter, blkid map[string]sysinspect.BlkidEntry, devicePath, wantLabel, checkName string) { + entry, ok := blkid[basename(devicePath)] + if !ok { + sa.Failf(checkName, "device %q not present in blkid", devicePath) + return + } + label, ok := entry.Get("PARTLABEL") + sa.Assert(checkName, ok && label == wantLabel, + "device %q PARTLABEL = %q, want %q", devicePath, label, wantLabel) +} + +func blkidHasDevice(blkid map[string]sysinspect.BlkidEntry, deviceBasename string) bool { + _, ok := blkid[deviceBasename] + return ok +} + +// blkidHasPath reports whether any blkid entry has the given full device path. +func blkidHasPath(blkid map[string]sysinspect.BlkidEntry, path string) bool { + for _, entry := range blkid { + if entry.Path == path { + return true + } + } + return false +} + +func basename(path string) string { + return path[strings.LastIndex(path, "/")+1:] +} diff --git a/tools/storm/e2e/validate/verity_test.go b/tools/storm/e2e/validate/verity_test.go new file mode 100644 index 0000000000..45d89e1412 --- /dev/null +++ b/tools/storm/e2e/validate/verity_test.go @@ -0,0 +1,55 @@ +package validate + +import ( + "testing" + + tridentutil "tridenttools/storm/utils/trident" +) + +const verityStatusYaml = ` +servicingState: provisioned +spec: + storage: + verity: + - id: root + name: root + dataDeviceId: root-data + hashDeviceId: root-hash + filesystems: + - deviceId: root + mountPoint: / +` + +func TestHasVerity(t *testing.T) { + hs, _ := tridentutil.NewHostStatusFromYaml([]byte(verityStatusYaml)) + if !HasVerity(hs) { + t.Error("expected HasVerity=true") + } + plain, _ := tridentutil.NewHostStatusFromYaml([]byte("spec:\n storage:\n disks: []\n")) + if HasVerity(plain) { + t.Error("expected HasVerity=false without verity") + } +} + +func TestVerityForRoot(t *testing.T) { + hs, _ := tridentutil.NewHostStatusFromYaml([]byte(verityStatusYaml)) + dev, ok := verityForRoot(hs.Spec(), "root") + if !ok { + t.Fatal("verityForRoot not found") + } + if dev.Name != "root" || dev.DataDeviceID != "root-data" || dev.HashDeviceID != "root-hash" { + t.Errorf("verity device = %+v", dev) + } + if _, ok := verityForRoot(hs.Spec(), "nonexistent"); ok { + t.Error("expected not found for nonexistent root") + } +} + +func TestBasename(t *testing.T) { + if basename("/dev/md/root-a") != "root-a" { + t.Error("basename /dev/md/root-a") + } + if basename("sda1") != "sda1" { + t.Error("basename sda1") + } +} diff --git a/tools/storm/utils/sysinspect/blkid.go b/tools/storm/utils/sysinspect/blkid.go index a07a9238ef..c507e7f247 100644 --- a/tools/storm/utils/sysinspect/blkid.go +++ b/tools/storm/utils/sysinspect/blkid.go @@ -20,6 +20,9 @@ type BlkidEntry struct { // Device is the kernel device name (e.g. "sda1"), i.e. the basename of the // device path reported by blkid. Device string + // Path is the full device path reported by blkid (e.g. "/dev/sda1", + // "/dev/mapper/root"). + Path string // Fields holds the parsed tag=value pairs (quotes stripped). Fields map[string]string } @@ -62,7 +65,7 @@ func ParseBlkid(stdout string) map[string]BlkidEntry { } device := devicePart[strings.LastIndex(devicePart, "/")+1:] - entry := BlkidEntry{Device: device, Fields: make(map[string]string)} + entry := BlkidEntry{Device: device, Path: devicePart, Fields: make(map[string]string)} for _, field := range strings.Fields(rest) { key, value, ok := strings.Cut(field, "=") diff --git a/tools/storm/utils/sysinspect/veritysetup.go b/tools/storm/utils/sysinspect/veritysetup.go new file mode 100644 index 0000000000..046236c062 --- /dev/null +++ b/tools/storm/utils/sysinspect/veritysetup.go @@ -0,0 +1,65 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// VeritySetupStatus is the parsed output of `veritysetup status `. +type VeritySetupStatus struct { + // Active is true when the first status line reports the device is active + // and in use. + Active bool + // Fields holds the "key: value" lines below the header (e.g. "type", + // "status", "mode", "data device", "hash device"). + Fields map[string]string +} + +// Get returns a status field value and whether it was present. +func (s VeritySetupStatus) Get(field string) (string, bool) { + v, ok := s.Fields[field] + return v, ok +} + +// VeritySetup runs `sudo veritysetup status ` on the host and returns the +// parsed status. +func VeritySetup(client *ssh.Client, name string) (VeritySetupStatus, error) { + out, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo veritysetup status %s", name)) + if err != nil { + return VeritySetupStatus{}, fmt.Errorf("failed to run veritysetup status %s: %w", name, err) + } + return ParseVeritySetupStatus(out, name), nil +} + +// ParseVeritySetupStatus parses `veritysetup status ` output. The first +// line is the " is active and is in use." header; subsequent lines are +// "key: value" pairs (keys may contain spaces, e.g. "data device"). +func ParseVeritySetupStatus(stdout, name string) VeritySetupStatus { + status := VeritySetupStatus{Fields: make(map[string]string)} + + lines := strings.Split(strings.TrimSpace(stdout), "\n") + if len(lines) == 0 { + return status + } + + header := strings.TrimSpace(lines[0]) + status.Active = header == fmt.Sprintf("/dev/mapper/%s is active and is in use.", name) + + for _, line := range lines[1:] { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + if key != "" && value != "" { + status.Fields[key] = value + } + } + + return status +} diff --git a/tools/storm/utils/sysinspect/veritysetup_test.go b/tools/storm/utils/sysinspect/veritysetup_test.go new file mode 100644 index 0000000000..3f23a6ad91 --- /dev/null +++ b/tools/storm/utils/sysinspect/veritysetup_test.go @@ -0,0 +1,47 @@ +package sysinspect + +import "testing" + +const veritysetupSample = `/dev/mapper/root is active and is in use. + type: VERITY + status: verified + hash type: 1 + data block: 4096 + hash block: 4096 + hash name: sha256 + data device: /dev/sda3 + size: 1377128 sectors + mode: readonly + hash device: /dev/sda4 + hash offset: 8 sectors + root hash: a8c34ed685f365352231db21aa36ff23bf8b658e001afa8e498f57d1755e9a19 + flags: panic_on_corruption` + +func TestParseVeritySetupStatus(t *testing.T) { + s := ParseVeritySetupStatus(veritysetupSample, "root") + if !s.Active { + t.Error("expected Active=true") + } + if v, _ := s.Get("type"); v != "VERITY" { + t.Errorf("type = %q, want VERITY", v) + } + if v, _ := s.Get("status"); v != "verified" { + t.Errorf("status = %q, want verified", v) + } + if v, _ := s.Get("mode"); v != "readonly" { + t.Errorf("mode = %q, want readonly", v) + } + if v, _ := s.Get("data device"); v != "/dev/sda3" { + t.Errorf("data device = %q, want /dev/sda3", v) + } + if v, _ := s.Get("hash device"); v != "/dev/sda4" { + t.Errorf("hash device = %q, want /dev/sda4", v) + } +} + +func TestParseVeritySetupStatus_Inactive(t *testing.T) { + s := ParseVeritySetupStatus("/dev/mapper/root is inactive.", "root") + if s.Active { + t.Error("expected Active=false for inactive device") + } +} From a81d44eb165d7479c3e8838e11023e6b5ad56941 Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 24 Jul 2026 17:10:33 -0700 Subject: [PATCH 05/41] storm/utils/sysinspect: add encryption inspection parsers Add SSH parsers needed for the encryption validation port: cryptsetup status + luksDump (JSON), dmsetup info, findmnt, active swaps, readlink -f, getenforce/setenforce, and blkid --output export. All unit-tested. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/storm/utils/sysinspect/blkid.go | 39 ++++++ tools/storm/utils/sysinspect/cryptsetup.go | 125 ++++++++++++++++++ tools/storm/utils/sysinspect/dmsetup.go | 38 ++++++ .../sysinspect/encryption_parsers_test.go | 101 ++++++++++++++ tools/storm/utils/sysinspect/findmnt.go | 54 ++++++++ tools/storm/utils/sysinspect/swap.go | 61 +++++++++ 6 files changed, 418 insertions(+) create mode 100644 tools/storm/utils/sysinspect/cryptsetup.go create mode 100644 tools/storm/utils/sysinspect/dmsetup.go create mode 100644 tools/storm/utils/sysinspect/encryption_parsers_test.go create mode 100644 tools/storm/utils/sysinspect/findmnt.go create mode 100644 tools/storm/utils/sysinspect/swap.go diff --git a/tools/storm/utils/sysinspect/blkid.go b/tools/storm/utils/sysinspect/blkid.go index c507e7f247..a38185bf95 100644 --- a/tools/storm/utils/sysinspect/blkid.go +++ b/tools/storm/utils/sysinspect/blkid.go @@ -80,3 +80,42 @@ func ParseBlkid(stdout string) map[string]BlkidEntry { return entries } + +// BlkidExport runs `sudo blkid --output export` and returns a map from full +// device path (DEVNAME) to its properties. This export format groups each +// device's fields as `KEY=value` lines separated by blank lines, and is used by +// the encryption validation (mirrors encryption_test.py::get_blkid_output). +func BlkidExport(client *ssh.Client) (map[string]map[string]string, error) { + out, err := sshutils.CommandOutput(client, "sudo blkid --output export") + if err != nil { + return nil, fmt.Errorf("failed to run blkid --output export: %w", err) + } + return ParseBlkidExport(out), nil +} + +// ParseBlkidExport parses `blkid --output export` output into a map keyed by +// DEVNAME (full device path). +func ParseBlkidExport(stdout string) map[string]map[string]string { + devices := make(map[string]map[string]string) + var current string + + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if line == "" { + current = "" + continue + } + key, value, ok := strings.Cut(line, "=") + if !ok { + continue + } + if key == "DEVNAME" { + current = value + devices[current] = make(map[string]string) + } else if current != "" { + devices[current][key] = value + } + } + + return devices +} diff --git a/tools/storm/utils/sysinspect/cryptsetup.go b/tools/storm/utils/sysinspect/cryptsetup.go new file mode 100644 index 0000000000..1473441090 --- /dev/null +++ b/tools/storm/utils/sysinspect/cryptsetup.go @@ -0,0 +1,125 @@ +package sysinspect + +import ( + "encoding/json" + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// CryptsetupStatus is the parsed output of `cryptsetup status `. +type CryptsetupStatus struct { + // Active is true when the device is active (LUKS2 volumes are always open + // and thus active). + Active bool + // InUse is true when the first line reports the device is "active and is in + // use" (mounted); false when it is merely "active". + InUse bool + Fields map[string]string +} + +// Get returns a status field value and whether it was present. +func (s CryptsetupStatus) Get(field string) (string, bool) { + v, ok := s.Fields[field] + return v, ok +} + +// Cryptsetup runs `sudo cryptsetup status ` and returns the parsed status. +func Cryptsetup(client *ssh.Client, name string) (CryptsetupStatus, error) { + out, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo cryptsetup status %s", name)) + if err != nil { + return CryptsetupStatus{}, fmt.Errorf("failed to run cryptsetup status %s: %w", name, err) + } + return ParseCryptsetupStatus(out, name), nil +} + +// ParseCryptsetupStatus parses `cryptsetup status ` output. The first line +// is the header (" is active[ and is in use]."); subsequent lines are +// "key: value" pairs. +func ParseCryptsetupStatus(stdout, name string) CryptsetupStatus { + status := CryptsetupStatus{Fields: make(map[string]string)} + + lines := strings.Split(strings.TrimSpace(stdout), "\n") + if len(lines) == 0 { + return status + } + + header := strings.TrimSpace(lines[0]) + inUseHeader := fmt.Sprintf("/dev/mapper/%s is active and is in use.", name) + activeHeader := fmt.Sprintf("/dev/mapper/%s is active.", name) + switch header { + case inUseHeader: + status.Active = true + status.InUse = true + case activeHeader: + status.Active = true + } + + for _, line := range lines[1:] { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + if key != "" { + status.Fields[key] = value + } + } + + return status +} + +// LuksDump is the subset of `cryptsetup luksDump --dump-json-metadata` output +// that the encryption validation inspects. +type LuksDump struct { + Keyslots map[string]LuksKeyslot `json:"keyslots"` + Tokens map[string]LuksToken `json:"tokens"` + Digests map[string]LuksDigest `json:"digests"` +} + +type LuksKeyslot struct { + Type string `json:"type"` + Kdf struct { + Type string `json:"type"` + Hash string `json:"hash"` + } `json:"kdf"` + Area struct { + Encryption string `json:"encryption"` + } `json:"area"` +} + +type LuksToken struct { + Type string `json:"type"` + Keyslots []string `json:"keyslots"` + Tpm2Pcrlock bool `json:"tpm2_pcrlock"` + Tpm2Pcrs []int `json:"tpm2-pcrs"` +} + +type LuksDigest struct { + Type string `json:"type"` + Hash string `json:"hash"` +} + +// CryptsetupLuksDump runs `cryptsetup luksDump --dump-json-metadata ` and +// returns the parsed metadata. +func CryptsetupLuksDump(client *ssh.Client, devicePath string) (LuksDump, error) { + out, err := sshutils.CommandOutput(client, + fmt.Sprintf("sudo cryptsetup luksDump --dump-json-metadata %s", devicePath)) + if err != nil { + return LuksDump{}, fmt.Errorf("failed to run cryptsetup luksDump %s: %w", devicePath, err) + } + return ParseLuksDump(out) +} + +// ParseLuksDump parses cryptsetup luksDump JSON metadata. +func ParseLuksDump(stdout string) (LuksDump, error) { + var dump LuksDump + if err := json.Unmarshal([]byte(stdout), &dump); err != nil { + return LuksDump{}, fmt.Errorf("failed to parse luksDump JSON: %w", err) + } + return dump, nil +} diff --git a/tools/storm/utils/sysinspect/dmsetup.go b/tools/storm/utils/sysinspect/dmsetup.go new file mode 100644 index 0000000000..16f288d444 --- /dev/null +++ b/tools/storm/utils/sysinspect/dmsetup.go @@ -0,0 +1,38 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// DmsetupInfo runs `sudo dmsetup info ` and returns the parsed key:value +// fields (Name, State, Tables present, UUID, ...). +func DmsetupInfo(client *ssh.Client, name string) (map[string]string, error) { + out, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo dmsetup info %s", name)) + if err != nil { + return nil, fmt.Errorf("failed to run dmsetup info %s: %w", name, err) + } + return ParseKeyValueLines(out), nil +} + +// ParseKeyValueLines parses lines of the form "key: value" into a map. Keys may +// contain spaces (e.g. "Tables present"); the split is on the first colon. +func ParseKeyValueLines(stdout string) map[string]string { + result := make(map[string]string) + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + if key != "" { + result[key] = value + } + } + return result +} diff --git a/tools/storm/utils/sysinspect/encryption_parsers_test.go b/tools/storm/utils/sysinspect/encryption_parsers_test.go new file mode 100644 index 0000000000..b5454df986 --- /dev/null +++ b/tools/storm/utils/sysinspect/encryption_parsers_test.go @@ -0,0 +1,101 @@ +package sysinspect + +import "testing" + +func TestParseCryptsetupStatus_InUse(t *testing.T) { + sample := `/dev/mapper/web is active and is in use. + type: n/a + cipher: aes-xts-plain64 + keysize: 512 bits + device: /dev/md127 + mode: read/write` + s := ParseCryptsetupStatus(sample, "web") + if !s.Active || !s.InUse { + t.Errorf("Active=%v InUse=%v, want both true", s.Active, s.InUse) + } + if v, _ := s.Get("cipher"); v != "aes-xts-plain64" { + t.Errorf("cipher = %q", v) + } + if v, _ := s.Get("keysize"); v != "512 bits" { + t.Errorf("keysize = %q", v) + } +} + +func TestParseCryptsetupStatus_ActiveNotInUse(t *testing.T) { + s := ParseCryptsetupStatus("/dev/mapper/web is active.\n cipher: aes-xts-plain64", "web") + if !s.Active || s.InUse { + t.Errorf("Active=%v InUse=%v, want active-not-inuse", s.Active, s.InUse) + } +} + +func TestParseLuksDump(t *testing.T) { + sample := `{ + "keyslots": {"1": {"type":"luks2","kdf":{"type":"pbkdf2","hash":"sha512"},"area":{"encryption":"aes-xts-plain64"}}}, + "tokens": {"0": {"type":"systemd-tpm2","keyslots":["1"],"tpm2_pcrlock":false,"tpm2-pcrs":[7]}}, + "digests": {"0": {"type":"pbkdf2","hash":"sha512"}} + }` + d, err := ParseLuksDump(sample) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if d.Keyslots["1"].Type != "luks2" || d.Keyslots["1"].Kdf.Hash != "sha512" { + t.Errorf("keyslot 1 = %+v", d.Keyslots["1"]) + } + if d.Keyslots["1"].Area.Encryption != "aes-xts-plain64" { + t.Errorf("area encryption = %q", d.Keyslots["1"].Area.Encryption) + } + tok := d.Tokens["0"] + if tok.Type != "systemd-tpm2" || tok.Tpm2Pcrlock != false || len(tok.Tpm2Pcrs) != 1 || tok.Tpm2Pcrs[0] != 7 { + t.Errorf("token 0 = %+v", tok) + } + if d.Digests["0"].Type != "pbkdf2" || d.Digests["0"].Hash != "sha512" { + t.Errorf("digest 0 = %+v", d.Digests["0"]) + } +} + +func TestParseKeyValueLines(t *testing.T) { + sample := `Name: web +State: ACTIVE +Tables present: LIVE +UUID: CRYPT-LUKS2-475f03514bb749bbb9af1f53f94b91cb-web` + m := ParseKeyValueLines(sample) + if m["Name"] != "web" || m["State"] != "ACTIVE" || m["Tables present"] != "LIVE" { + t.Errorf("parsed = %v", m) + } + if m["UUID"] != "CRYPT-LUKS2-475f03514bb749bbb9af1f53f94b91cb-web" { + t.Errorf("UUID = %q", m["UUID"]) + } +} + +func TestParseFindmnt(t *testing.T) { + sample := `TARGET SOURCE FSTYPE OPTIONS +/mnt/web /dev/mapper/web ext4 rw,relatime` + rows := ParseFindmnt(sample) + if len(rows) != 1 { + t.Fatalf("got %d rows, want 1", len(rows)) + } + r := rows[0] + if r.Target != "/mnt/web" || r.Source != "/dev/mapper/web" || r.FsType != "ext4" { + t.Errorf("row = %+v", r) + } +} + +func TestParseBlkidExport(t *testing.T) { + sample := `DEVNAME=/dev/md127 +UUID=475f0351-4bb7-49bb-b9af-1f53f94b91cb +TYPE=crypto_LUKS +PARTLABEL=web + +DEVNAME=/dev/sr0 +TYPE=iso9660` + devs := ParseBlkidExport(sample) + if len(devs) != 2 { + t.Fatalf("got %d devices, want 2", len(devs)) + } + if devs["/dev/md127"]["TYPE"] != "crypto_LUKS" { + t.Errorf("md127 TYPE = %q", devs["/dev/md127"]["TYPE"]) + } + if devs["/dev/md127"]["PARTLABEL"] != "web" { + t.Errorf("md127 PARTLABEL = %q", devs["/dev/md127"]["PARTLABEL"]) + } +} diff --git a/tools/storm/utils/sysinspect/findmnt.go b/tools/storm/utils/sysinspect/findmnt.go new file mode 100644 index 0000000000..22ccb84c29 --- /dev/null +++ b/tools/storm/utils/sysinspect/findmnt.go @@ -0,0 +1,54 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// FindmntRow is one row of `findmnt ` output. +type FindmntRow struct { + Target string + Source string + FsType string + Options string +} + +// Findmnt runs `sudo findmnt ` and returns the parsed rows. +func Findmnt(client *ssh.Client, target string) ([]FindmntRow, error) { + out, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo findmnt %s", target)) + if err != nil { + return nil, fmt.Errorf("failed to run findmnt %s: %w", target, err) + } + return ParseFindmnt(out), nil +} + +// ParseFindmnt parses `findmnt ` table output. The first line is the +// header (TARGET SOURCE FSTYPE OPTIONS); columns are whitespace-separated. +func ParseFindmnt(stdout string) []FindmntRow { + lines := strings.Split(strings.TrimSpace(stdout), "\n") + if len(lines) < 2 { + return nil + } + + var rows []FindmntRow + for _, line := range lines[1:] { + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + row := FindmntRow{ + Target: fields[0], + Source: fields[1], + FsType: fields[2], + } + if len(fields) > 3 { + row.Options = fields[3] + } + rows = append(rows, row) + } + return rows +} diff --git a/tools/storm/utils/sysinspect/swap.go b/tools/storm/utils/sysinspect/swap.go new file mode 100644 index 0000000000..5dfbf2f7d9 --- /dev/null +++ b/tools/storm/utils/sysinspect/swap.go @@ -0,0 +1,61 @@ +package sysinspect + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/storm/utils/sshutils" +) + +// ActiveSwaps returns the set of active swap device paths, canonicalized via +// `readlink -f`. Mirrors encryption_test.py::get_active_swaps. +func ActiveSwaps(client *ssh.Client) (map[string]struct{}, error) { + cmd := "swapon --show=NAME --raw --bytes --noheadings | xargs -r -I @ readlink -f @" + out, err := sshutils.CommandOutput(client, cmd) + if err != nil { + return nil, fmt.Errorf("failed to list active swaps: %w", err) + } + + swaps := make(map[string]struct{}) + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + line = strings.TrimSpace(line) + if line != "" { + swaps[line] = struct{}{} + } + } + return swaps, nil +} + +// ReadlinkF resolves a path to its canonical absolute form via `readlink -f`. +func ReadlinkF(client *ssh.Client, path string) (string, error) { + out, err := sshutils.CommandOutput(client, fmt.Sprintf("sudo readlink -f %s", path)) + if err != nil { + return "", fmt.Errorf("failed to readlink -f %s: %w", path, err) + } + return strings.TrimSpace(out), nil +} + +// Getenforce returns the current SELinux enforcement mode ("Enforcing", +// "Permissive", or "Disabled"). +func Getenforce(client *ssh.Client) (string, error) { + out, err := sshutils.CommandOutput(client, "sudo getenforce") + if err != nil { + return "", fmt.Errorf("failed to run getenforce: %w", err) + } + return strings.TrimSpace(out), nil +} + +// Setenforce sets the SELinux enforcement mode (0 = permissive, 1 = enforcing). +func Setenforce(client *ssh.Client, enforcing bool) error { + mode := "0" + if enforcing { + mode = "1" + } + _, err := sshutils.CommandOutput(client, "sudo setenforce "+mode) + if err != nil { + return fmt.Errorf("failed to run setenforce %s: %w", mode, err) + } + return nil +} From ed904b77d2b935a3677cc75e6264d1423b073907 Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 24 Jul 2026 17:14:28 -0700 Subject: [PATCH 06/41] storm/e2e: port encryption host-state validation Port encryption_test.py::test_encryption (the largest marker). For each configured encryption volume, validate: - the backing device (partition or RAID array) is crypto_LUKS; - LUKS2 metadata via cryptsetup luksDump (digest/keyslot/token, TPM2 policy differing for UKI vs grub images) with the SELinux permissive workaround; - device-mapper state via dmsetup info (LUKS2 vs PLAIN for swap); - cryptsetup status (cipher/keysize, in-use vs active); - mount/swap/active status via findmnt/swapon, accounting for A/B pair membership and the active volume. Self-selects when the Host Config declares encryption volumes; uses the scenario's isUki param for the expected TPM2 policy. Unit-tested (parsers + HC helpers); storm-trident builds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/storm/e2e/scenario/validate.go | 7 +- tools/storm/e2e/validate/encryption.go | 395 ++++++++++++++++++++ tools/storm/e2e/validate/encryption_test.go | 156 ++++++++ 3 files changed, 556 insertions(+), 2 deletions(-) create mode 100644 tools/storm/e2e/validate/encryption.go create mode 100644 tools/storm/e2e/validate/encryption_test.go diff --git a/tools/storm/e2e/scenario/validate.go b/tools/storm/e2e/scenario/validate.go index fdd008e493..42884f2e94 100644 --- a/tools/storm/e2e/scenario/validate.go +++ b/tools/storm/e2e/scenario/validate.go @@ -56,8 +56,11 @@ func (s *TridentE2EScenario) validateHostState(tc storm.TestCase) error { validate.ValidateVerity(&sa, s.sshClient, hs, s.expectedActiveVolume) } - // Future markers (encryption) will self-select here based on the Host - // Configuration. + // `encryption` validation self-selects when the Host Config declares + // encryption volumes. + if validate.HasEncryption(hs) { + validate.ValidateEncryption(&sa, s.sshClient, hs, s.configParams.IsUki, s.expectedActiveVolume) + } } if err := sa.Err(); err != nil { diff --git a/tools/storm/e2e/validate/encryption.go b/tools/storm/e2e/validate/encryption.go new file mode 100644 index 0000000000..ece6be6c68 --- /dev/null +++ b/tools/storm/e2e/validate/encryption.go @@ -0,0 +1,395 @@ +package validate + +import ( + "fmt" + "strings" + + "golang.org/x/crypto/ssh" + + "tridenttools/pkg/hostconfig" + "tridenttools/storm/utils/sshutils" + "tridenttools/storm/utils/sysinspect" + tridentutil "tridenttools/storm/utils/trident" +) + +const ( + expectedCipher = "aes-xts-plain64" + expectedKeysize = "512 bits" + expectedFsType = "ext4" + expectedLuksType = "crypto_LUKS" + expectedDigestType = "pbkdf2" + expectedDigestHash = "sha512" +) + +// HasEncryption reports whether the spec declares encryption volumes, used to +// self-select the encryption validation. +func HasEncryption(hs tridentutil.HostStatus) bool { + return hs.Spec().Exists("storage", "encryption") +} + +// ValidateEncryption ports encryption_test.py::test_encryption. For each +// configured encryption volume it validates the backing device is LUKS, the +// LUKS metadata, the device-mapper state, and the mount/swap/active status +// (accounting for A/B update pairs). isUki selects the expected TPM2 policy. +func ValidateEncryption( + sa *SoftAsserter, + client *ssh.Client, + hs tridentutil.HostStatus, + isUki bool, + abActive tridentutil.AbVolumeSelection, +) { + spec := hs.Spec() + + blockDevs, err := sysinspect.BlkidExport(client) + if err != nil { + sa.Fail("encryption/blkid-export", err) + return + } + + for _, crypt := range spec.S("storage", "encryption", "volumes").Children() { + cryptID, _ := crypt.S("id").Data().(string) + cryptDevName, _ := crypt.S("deviceName").Data().(string) + cryptDevID, _ := crypt.S("deviceId").Data().(string) + checkCryptDevice(sa, client, spec, isUki, abActive, blockDevs, cryptID, cryptDevName, cryptDevID) + } +} + +func checkCryptDevice( + sa *SoftAsserter, + client *ssh.Client, + spec hostconfig.HostConfig, + isUki bool, + abActive tridentutil.AbVolumeSelection, + blockDevs map[string]map[string]string, + cryptID, cryptDevName, cryptDevID string, +) { + cryptDevicePath := "/dev/mapper/" + cryptDevName + + checkParentDevices(sa, client, spec, isUki, blockDevs, cryptDevID) + + swap := false + isInUse := true + + if pair, isVolumeA, ok := childAbUpdateVolumePair(spec, cryptID); ok { + // Encryption volume is an A/B pair member: it is in use only when its + // side matches the active volume. + isInUse = (abActive == tridentutil.AbVolumeA && isVolumeA) || + (abActive == tridentutil.AbVolumeB && !isVolumeA) + + pairID, _ := pair.S("id").Data().(string) + fs, ok := filesystemByDeviceID(spec, pairID) + if !ok { + sa.Failf("encryption/ab-fs", "no filesystem for A/B volume pair %q", pairID) + } else if mp, ok := mountPointPath(&fs); ok { + checkExists(sa, client, mp) + checkFindmnt(sa, client, mp, cryptDevicePath, isInUse) + } else { + sa.Failf("encryption/ab-mount", "no mount point for A/B volume pair %q", pairID) + } + } else if isSwapDevice(spec, cryptID) { + swap = true + swaps, err := sysinspect.ActiveSwaps(client) + if err != nil { + sa.Fail("encryption/swaps", err) + } else { + realPath, err := sysinspect.ReadlinkF(client, cryptDevicePath) + if err != nil { + sa.Fail("encryption/swap-readlink", err) + } else if _, active := swaps[realPath]; !active { + sa.Failf("encryption/swap-active", "swap %q not in active swaps %v", realPath, swaps) + } + } + } else { + fs, ok := filesystemByDeviceID(spec, cryptID) + if !ok { + sa.Failf("encryption/fs", "no filesystem for encryption volume %q", cryptID) + } else if mp, ok := mountPointPath(&fs); ok { + checkExists(sa, client, mp) + checkFindmnt(sa, client, mp, cryptDevicePath, isInUse) + } else { + sa.Failf("encryption/mount", "encryption volume %q filesystem is not mounted", cryptID) + } + } + + checkExists(sa, client, cryptDevicePath) + checkCryptsetupStatus(sa, client, cryptDevName, isInUse) + checkDmsetupInfo(sa, client, cryptDevName, swap) +} + +// checkParentDevices confirms the crypt device's backing block device is LUKS +// and validates its LUKS metadata. +func checkParentDevices( + sa *SoftAsserter, + client *ssh.Client, + spec hostconfig.HostConfig, + isUki bool, + blockDevs map[string]map[string]string, + cryptDevID string, +) { + var cryptDevPath string + if IsPartition(spec, cryptDevID) { + path, ok := blockDevPathByPartlabel(blockDevs, cryptDevID) + if !ok { + sa.Failf("encryption/parent-partition", "no device with PARTLABEL %q", cryptDevID) + return + } + cryptDevPath = path + } else { + raidName, ok := raidSoftwareArrayName(spec, cryptDevID) + if !ok { + sa.Failf("encryption/parent-kind", "%q is neither a partition nor a RAID array", cryptDevID) + return + } + path, err := sysinspect.ReadlinkF(client, "/dev/md/"+raidName) + if err != nil { + sa.Fail("encryption/parent-raid", err) + return + } + cryptDevPath = path + } + + dev, ok := blockDevs[cryptDevPath] + if !ok { + sa.Failf("encryption/parent-blkid", "no blkid entry for %q", cryptDevPath) + return + } + sa.Assert("encryption/parent-type", dev["TYPE"] == expectedLuksType, + "device %q TYPE = %q, want %q", cryptDevPath, dev["TYPE"], expectedLuksType) + + checkLuksDump(sa, client, cryptDevPath, isUki) +} + +// checkLuksDump validates the LUKS2 metadata from cryptsetup luksDump. +func checkLuksDump(sa *SoftAsserter, client *ssh.Client, cryptDevPath string, isUki bool) { + // luksDump needs an SELinux permission the Trident policy intentionally + // omits; temporarily drop to permissive (a testing-infra quirk), matching + // encryption_test.py. + enforcing := false + if mode, err := sysinspect.Getenforce(client); err == nil && mode == "Enforcing" { + enforcing = true + if err := sysinspect.Setenforce(client, false); err != nil { + sa.Fail("encryption/selinux", err) + } + } + + dump, err := sysinspect.CryptsetupLuksDump(client, cryptDevPath) + + if enforcing { + // Best-effort restore of enforcing mode. + if restoreErr := sysinspect.Setenforce(client, true); restoreErr != nil { + sa.Fail("encryption/selinux-restore", restoreErr) + } + } + + if err != nil { + sa.Fail("encryption/luks-dump", err) + return + } + + digest, ok := dump.Digests["0"] + if !ok { + sa.Failf("encryption/luks-digest", "luksDump missing digest 0") + } else { + sa.Assert("encryption/luks-digest-type", digest.Type == expectedDigestType, + "digest type = %q, want %q", digest.Type, expectedDigestType) + sa.Assert("encryption/luks-digest-hash", digest.Hash == expectedDigestHash, + "digest hash = %q, want %q", digest.Hash, expectedDigestHash) + } + + token, ok := dump.Tokens["0"] + if !ok { + sa.Failf("encryption/luks-token", "luksDump missing token 0") + } else { + sa.Assert("encryption/luks-token-count", len(dump.Tokens) == 1, + "expected 1 token, got %d", len(dump.Tokens)) + sa.Assert("encryption/luks-token-keyslots", len(token.Keyslots) == 1 && contains(token.Keyslots, "1"), + "expected token keyslot [1], got %v", token.Keyslots) + sa.Assert("encryption/luks-token-type", token.Type == "systemd-tpm2", + "token type = %q, want systemd-tpm2", token.Type) + if isUki { + sa.Assert("encryption/luks-pcrlock", token.Tpm2Pcrlock, + "expected tpm2_pcrlock=true for UKI image") + sa.Assert("encryption/luks-pcrs", len(token.Tpm2Pcrs) == 0, + "expected empty tpm2-pcrs for UKI image, got %v", token.Tpm2Pcrs) + } else { + sa.Assert("encryption/luks-pcrlock", !token.Tpm2Pcrlock, + "expected tpm2_pcrlock=false for non-UKI image") + sa.Assert("encryption/luks-pcrs", len(token.Tpm2Pcrs) == 1 && token.Tpm2Pcrs[0] == 7, + "expected tpm2-pcrs=[7] for non-UKI image, got %v", token.Tpm2Pcrs) + } + } + + keyslot, ok := dump.Keyslots["1"] + if !ok { + sa.Failf("encryption/luks-keyslot", "luksDump missing keyslot 1") + } else { + sa.Assert("encryption/luks-keyslot-count", len(dump.Keyslots) == 1, + "expected 1 keyslot, got %d", len(dump.Keyslots)) + sa.Assert("encryption/luks-keyslot-type", keyslot.Type == "luks2", + "keyslot type = %q, want luks2", keyslot.Type) + sa.Assert("encryption/luks-kdf-type", keyslot.Kdf.Type == "pbkdf2", + "keyslot kdf type = %q, want pbkdf2", keyslot.Kdf.Type) + sa.Assert("encryption/luks-kdf-hash", keyslot.Kdf.Hash == "sha512", + "keyslot kdf hash = %q, want sha512", keyslot.Kdf.Hash) + sa.Assert("encryption/luks-area-enc", keyslot.Area.Encryption == expectedCipher, + "keyslot area encryption = %q, want %q", keyslot.Area.Encryption, expectedCipher) + } +} + +func checkCryptsetupStatus(sa *SoftAsserter, client *ssh.Client, name string, isInUse bool) { + status, err := sysinspect.Cryptsetup(client, name) + if err != nil { + sa.Fail("encryption/cryptsetup-status", err) + return + } + if isInUse { + sa.Assert("encryption/cryptsetup-inuse", status.InUse, + "expected %q to be active and in use", name) + } else { + sa.Assert("encryption/cryptsetup-active", status.Active && !status.InUse, + "expected %q to be active but not in use", name) + } + if cipher, _ := status.Get("cipher"); cipher != expectedCipher { + sa.Failf("encryption/cryptsetup-cipher", "cipher = %q, want %q", cipher, expectedCipher) + } + if keysize, _ := status.Get("keysize"); keysize != expectedKeysize { + sa.Failf("encryption/cryptsetup-keysize", "keysize = %q, want %q", keysize, expectedKeysize) + } +} + +func checkDmsetupInfo(sa *SoftAsserter, client *ssh.Client, name string, swap bool) { + info, err := sysinspect.DmsetupInfo(client, name) + if err != nil { + sa.Fail("encryption/dmsetup", err) + return + } + sa.Assert("encryption/dmsetup-name", info["Name"] == name, + "dmsetup Name = %q, want %q", info["Name"], name) + sa.Assert("encryption/dmsetup-state", info["State"] == "ACTIVE", + "dmsetup State = %q, want ACTIVE", info["State"]) + sa.Assert("encryption/dmsetup-tables", info["Tables present"] == "LIVE", + "dmsetup Tables present = %q, want LIVE", info["Tables present"]) + + cryptKind := "LUKS2" + if swap { + cryptKind = "PLAIN" + } + prefix := fmt.Sprintf("CRYPT-%s-", cryptKind) + suffix := "-" + name + uuid := info["UUID"] + sa.Assert("encryption/dmsetup-uuid-prefix", strings.HasPrefix(uuid, prefix), + "dmsetup UUID %q does not start with %q", uuid, prefix) + sa.Assert("encryption/dmsetup-uuid-suffix", strings.HasSuffix(uuid, suffix), + "dmsetup UUID %q does not end with %q", uuid, suffix) +} + +// checkExists runs `sudo ls ` and records a failure if it does not exist. +func checkExists(sa *SoftAsserter, client *ssh.Client, path string) { + out, err := sshutils.RunCommand(client, fmt.Sprintf("sudo ls %s", path)) + if err != nil { + sa.Fail("encryption/exists", err) + return + } + sa.Assert("encryption/exists", out.Status == 0, "path does not exist: %s", path) +} + +// checkFindmnt validates the findmnt row for target: when active, SOURCE must be +// the crypt device; when inactive, SOURCE must differ. FSTYPE is always ext4. +func checkFindmnt(sa *SoftAsserter, client *ssh.Client, target, source string, isActive bool) { + rows, err := sysinspect.Findmnt(client, target) + if err != nil { + sa.Fail("encryption/findmnt", err) + return + } + if len(rows) != 1 { + sa.Failf("encryption/findmnt-rows", "expected 1 findmnt row for %q, got %d", target, len(rows)) + return + } + row := rows[0] + sa.Assert("encryption/findmnt-target", row.Target == target, + "findmnt TARGET = %q, want %q", row.Target, target) + sa.Assert("encryption/findmnt-fstype", row.FsType == expectedFsType, + "findmnt FSTYPE = %q, want %q", row.FsType, expectedFsType) + if isActive { + sa.Assert("encryption/findmnt-source", row.Source == source, + "findmnt SOURCE = %q, want %q (active)", row.Source, source) + } else { + sa.Assert("encryption/findmnt-source", row.Source != source, + "findmnt SOURCE = %q, expected different from %q (inactive)", row.Source, source) + } +} + +// --- Host Configuration helpers (operate on the Host Status spec) --- + +// childAbUpdateVolumePair returns the A/B volume pair that has cryptID as one of +// its volumes, whether cryptID is the A side, and whether such a pair exists. +func childAbUpdateVolumePair(spec hostconfig.HostConfig, cryptID string) (*hostconfig.HostConfig, bool, bool) { + for _, pair := range spec.S("storage", "abUpdate", "volumePairs").Children() { + if a, _ := pair.S("volumeAId").Data().(string); a == cryptID { + hc := hostconfig.NewHostConfigFromContainer(pair) + return &hc, true, true + } + if b, _ := pair.S("volumeBId").Data().(string); b == cryptID { + hc := hostconfig.NewHostConfigFromContainer(pair) + return &hc, false, true + } + } + return nil, false, false +} + +// filesystemByDeviceID returns the filesystem with the given deviceId. +func filesystemByDeviceID(spec hostconfig.HostConfig, deviceID string) (hostconfig.HostConfig, bool) { + for _, fs := range spec.S("storage", "filesystems").Children() { + if id, _ := fs.S("deviceId").Data().(string); id == deviceID { + return hostconfig.NewHostConfigFromContainer(fs), true + } + } + return hostconfig.HostConfig{}, false +} + +// isSwapDevice reports whether devID is configured as swap. storage.swap entries +// may be plain device-id strings or objects with a deviceId field. +func isSwapDevice(spec hostconfig.HostConfig, devID string) bool { + for _, swap := range spec.S("storage", "swap").Children() { + if s, ok := swap.Data().(string); ok && s == devID { + return true + } + if id, ok := swap.S("deviceId").Data().(string); ok && id == devID { + return true + } + } + return false +} + +// raidSoftwareArrayName returns the name of the software RAID array with the +// given id. +func raidSoftwareArrayName(spec hostconfig.HostConfig, id string) (string, bool) { + for _, raid := range spec.S("storage", "raid", "software").Children() { + if rid, _ := raid.S("id").Data().(string); rid == id { + if name, ok := raid.S("name").Data().(string); ok { + return name, true + } + } + } + return "", false +} + +// blockDevPathByPartlabel returns the device path whose PARTLABEL matches label. +func blockDevPathByPartlabel(blockDevs map[string]map[string]string, label string) (string, bool) { + for path, dev := range blockDevs { + if dev["PARTLABEL"] == label { + return path, true + } + } + return "", false +} + +// contains reports whether s contains v. +func contains(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} diff --git a/tools/storm/e2e/validate/encryption_test.go b/tools/storm/e2e/validate/encryption_test.go new file mode 100644 index 0000000000..4373a284fd --- /dev/null +++ b/tools/storm/e2e/validate/encryption_test.go @@ -0,0 +1,156 @@ +package validate + +import ( + "testing" + + "tridenttools/pkg/hostconfig" + tridentutil "tridenttools/storm/utils/trident" +) + +const encSpecYaml = ` +storage: + encryption: + volumes: + - id: enc-root + deviceName: root + deviceId: root-luks + - id: enc-swap + deviceName: swapdev + deviceId: swap-part + disks: + - id: os + partitions: + - id: swap-part + size: 2G + raid: + software: + - id: root-luks + name: rootarray + abUpdate: + volumePairs: + - id: root + volumeAId: enc-root + volumeBId: enc-root-b + filesystems: + - deviceId: root + mountPoint: / + swap: + - enc-swap +` + +func encSpec(t *testing.T) hostconfig.HostConfig { + t.Helper() + hc, err := hostconfig.NewHostConfigFromYaml([]byte(encSpecYaml)) + if err != nil { + t.Fatalf("parse: %v", err) + } + return hc +} + +func TestHasEncryption(t *testing.T) { + hs, _ := tridentutil.NewHostStatusFromYaml([]byte("spec:\n" + indent(encSpecYaml))) + if !HasEncryption(hs) { + t.Error("expected HasEncryption=true") + } + plain, _ := tridentutil.NewHostStatusFromYaml([]byte("spec:\n storage:\n disks: []\n")) + if HasEncryption(plain) { + t.Error("expected HasEncryption=false") + } +} + +func TestChildAbUpdateVolumePair(t *testing.T) { + spec := encSpec(t) + pair, isA, ok := childAbUpdateVolumePair(spec, "enc-root") + if !ok || !isA { + t.Fatalf("enc-root: ok=%v isA=%v, want true,true", ok, isA) + } + if id, _ := pair.S("id").Data().(string); id != "root" { + t.Errorf("pair id = %q, want root", id) + } + if _, isA, ok := childAbUpdateVolumePair(spec, "enc-root-b"); !ok || isA { + t.Errorf("enc-root-b: ok=%v isA=%v, want true,false", ok, isA) + } + if _, _, ok := childAbUpdateVolumePair(spec, "nope"); ok { + t.Error("nope should not be found") + } +} + +func TestFilesystemByDeviceID(t *testing.T) { + spec := encSpec(t) + fs, ok := filesystemByDeviceID(spec, "root") + if !ok { + t.Fatal("root fs not found") + } + if mp, ok := mountPointPath(&fs); !ok || mp != "/" { + t.Errorf("mount = %q ok=%v, want /", mp, ok) + } + if _, ok := filesystemByDeviceID(spec, "missing"); ok { + t.Error("missing fs should not be found") + } +} + +func TestIsSwapDevice(t *testing.T) { + spec := encSpec(t) + if !isSwapDevice(spec, "enc-swap") { + t.Error("enc-swap should be swap") + } + if isSwapDevice(spec, "enc-root") { + t.Error("enc-root should not be swap") + } +} + +func TestRaidSoftwareArrayName(t *testing.T) { + spec := encSpec(t) + name, ok := raidSoftwareArrayName(spec, "root-luks") + if !ok || name != "rootarray" { + t.Errorf("got (%q,%v), want (rootarray,true)", name, ok) + } + if _, ok := raidSoftwareArrayName(spec, "swap-part"); ok { + t.Error("swap-part is a partition, not raid") + } +} + +func TestBlockDevPathByPartlabel(t *testing.T) { + devs := map[string]map[string]string{ + "/dev/sda4": {"PARTLABEL": "swap-part", "TYPE": "crypto_LUKS"}, + "/dev/sda1": {"PARTLABEL": "esp"}, + } + path, ok := blockDevPathByPartlabel(devs, "swap-part") + if !ok || path != "/dev/sda4" { + t.Errorf("got (%q,%v), want (/dev/sda4,true)", path, ok) + } + if _, ok := blockDevPathByPartlabel(devs, "nope"); ok { + t.Error("nope should not match") + } +} + +// indent prefixes every non-empty line with two spaces (to nest encSpecYaml +// under a `spec:` key). +func indent(s string) string { + out := "" + for _, line := range splitLines(s) { + if line == "" { + out += "\n" + } else { + out += " " + line + "\n" + } + } + return out +} + +func splitLines(s string) []string { + var lines []string + cur := "" + for _, r := range s { + if r == '\n' { + lines = append(lines, cur) + cur = "" + } else { + cur += string(r) + } + } + if cur != "" { + lines = append(lines, cur) + } + return lines +} From c1b2d7eee956938daf1ce1feb53411c7ab852ba3 Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 24 Jul 2026 18:55:03 -0700 Subject: [PATCH 07/41] storm/e2e: fix verity self-selection to root-verity only Code review found HasVerity self-selected on any storage.verity entry, so usr-verity configs (verity on /usr, plain root) would run root-verity validation and false-fail with 'no verity configuration found for root device'. The ported verity_test.py::test_verity_root only validates root verity and only ran on root-verity configs. Gate HasVerity on whether the root filesystem is itself a verity device, matching that scope. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/storm/e2e/validate/verity.go | 15 ++++++++++++--- tools/storm/e2e/validate/verity_test.go | 23 ++++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/tools/storm/e2e/validate/verity.go b/tools/storm/e2e/validate/verity.go index 44240d622a..321c32483d 100644 --- a/tools/storm/e2e/validate/verity.go +++ b/tools/storm/e2e/validate/verity.go @@ -18,10 +18,19 @@ type VerityDevice struct { HashDeviceID string } -// HasVerity reports whether the spec declares any verity device, used to -// self-select verity validation. +// HasVerity reports whether the ROOT filesystem is backed by a verity device, +// used to self-select verity validation. This matches the scope of the ported +// verity_test.py::test_verity_root, which only validates root verity: configs +// where verity protects a non-root path (e.g. usr-verity) are intentionally not +// selected (they receive only base validation, as under pytest). func HasVerity(hs tridentutil.HostStatus) bool { - return hs.Spec().Exists("storage", "verity") + spec := hs.Spec() + rootID, ok := RootFilesystemDeviceID(spec) + if !ok { + return false + } + _, ok = verityForRoot(spec, rootID) + return ok } // verityForRoot returns the verity device whose id matches the root diff --git a/tools/storm/e2e/validate/verity_test.go b/tools/storm/e2e/validate/verity_test.go index 45d89e1412..525ef954b6 100644 --- a/tools/storm/e2e/validate/verity_test.go +++ b/tools/storm/e2e/validate/verity_test.go @@ -21,14 +21,35 @@ spec: ` func TestHasVerity(t *testing.T) { + // Root is a verity device -> selected. hs, _ := tridentutil.NewHostStatusFromYaml([]byte(verityStatusYaml)) if !HasVerity(hs) { - t.Error("expected HasVerity=true") + t.Error("expected HasVerity=true when root is a verity device") } + // No verity at all -> not selected. plain, _ := tridentutil.NewHostStatusFromYaml([]byte("spec:\n storage:\n disks: []\n")) if HasVerity(plain) { t.Error("expected HasVerity=false without verity") } + // usr-verity: verity exists but protects /usr, not root -> NOT selected + // (root-verity validation must not run and false-fail). + usrVerity, _ := tridentutil.NewHostStatusFromYaml([]byte(` +spec: + storage: + verity: + - id: usr + name: usr + dataDeviceId: usr-data + hashDeviceId: usr-hash + filesystems: + - deviceId: root + mountPoint: / + - deviceId: usr + mountPoint: /usr +`)) + if HasVerity(usrVerity) { + t.Error("expected HasVerity=false for usr-verity (root is not a verity device)") + } } func TestVerityForRoot(t *testing.T) { From c01585ad78ce5f55ac66195dca9be8037b955abd Mon Sep 17 00:00:00 2001 From: Paco Date: Mon, 27 Jul 2026 14:29:47 -0700 Subject: [PATCH 08/41] storm/e2e: expect install failure for rollback-intent scenarios checkTridentViaSshAfterInstall hardcoded expectSuccessfulCommit=true, so health-check rollback scenarios (health-checks-install), whose install deliberately fails and rolls back, would false-fail before rollback validation runs. Gate expectSuccessfulCommit on a new hasRollbackIntent() (top-level 'health' in the Host Config), matching the pipeline's --expect-failed-commit handling. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- junit.xml | 5683 ++++++++++++++++++++++++++ metrics-ab-update-1-ab-update.jsonl | 45 + tools/storm/e2e/scenario/install.go | 8 +- tools/storm/e2e/scenario/validate.go | 8 + trident-clean-install-metrics.jsonl | 49 + 5 files changed, 5792 insertions(+), 1 deletion(-) create mode 100644 junit.xml create mode 100644 metrics-ab-update-1-ab-update.jsonl create mode 100644 trident-clean-install-metrics.jsonl diff --git a/junit.xml b/junit.xml new file mode 100644 index 0000000000..54fe9ddadf --- /dev/null +++ b/junit.xml @@ -0,0 +1,5683 @@ + + + + + + + + + + + trident-e2e-base_vm-host-network + + + + + + + + + +INFO[0000] Created and started network 'trident-e2e-base_vm-host-network' +TRAC[0000] Storage pool trident-e2e-base_vm-host-pool does not exist, skipping deletion +TRAC[0000] Defining storage pool with XML: + + trident-e2e-base_vm-host-pool + + /var/lib/libvirt/trident-e2e-base_vm-host + + -1 + -1 + 0755 + + + +INFO[0000] Created and started storage pool 'trident-e2e-base_vm-host-pool' +TRAC[0000] Storage pool trident-e2e-base_vm-host-nvram-pool does not exist, skipping deletion +TRAC[0000] Defining storage pool with XML: + + trident-e2e-base_vm-host-nvram-pool + + /var/lib/trident-e2e-base_vm-host-nvram + + -1 + -1 + 0777 + + + +INFO[0000] Created and started storage pool 'trident-e2e-base_vm-host-nvram-pool' +TRAC[0000] Domain trident-e2e-base_vm-host-vm-0 does not exist, skipping deletion +DEBU[0000] Setting up volume 'trident-e2e-base_vm-host-vm-0_VARS.fd' at path '/var/lib/trident-e2e-base_vm-host-nvram/trident-e2e-base_vm-host-vm-0_VARS.fd' +TRAC[0000] Volume trident-e2e-base_vm-host-vm-0_VARS.fd does not exist, skipping deletion +TRAC[0000] Defining volume with XML: + + trident-e2e-base_vm-host-vm-0_VARS.fd + 0 + + /var/lib/trident-e2e-base_vm-host-nvram/trident-e2e-base_vm-host-vm-0_VARS.fd + + + 0666 + + + +DEBU[0000] No OS disk specified for volume 'trident-e2e-base_vm-host-vm-0_VARS.fd', creating blank disk +INFO[0000] Created volume 'trident-e2e-base_vm-host-vm-0_VARS.fd' +DEBU[0000] Uploading NVRAM template from /usr/share/OVMF/OVMF_VARS_4M.ms.fd to /var/lib/trident-e2e-base_vm-host-nvram/trident-e2e-base_vm-host-vm-0_VARS.fd +DEBU[0000] Uploading 540672 bytes from file '/usr/share/OVMF/OVMF_VARS_4M.ms.fd' to volume 'trident-e2e-base_vm-host-vm-0_VARS.fd' +INFO[0000] Uploaded file '/usr/share/OVMF/OVMF_VARS_4M.ms.fd' to volume 'trident-e2e-base_vm-host-vm-0_VARS.fd' +DEBU[0000] Setting up volume 'trident-e2e-base_vm-host-vm-0-volume-0.qcow2' at path '/var/lib/libvirt/trident-e2e-base_vm-host/trident-e2e-base_vm-host-vm-0-volume-0.qcow2' +TRAC[0000] Volume trident-e2e-base_vm-host-vm-0-volume-0.qcow2 does not exist, skipping deletion +TRAC[0000] Defining volume with XML: + + trident-e2e-base_vm-host-vm-0-volume-0.qcow2 + 32 + + /var/lib/libvirt/trident-e2e-base_vm-host/trident-e2e-base_vm-host-vm-0-volume-0.qcow2 + + + 0644 + + + +DEBU[0000] No OS disk specified for volume 'trident-e2e-base_vm-host-vm-0-volume-0.qcow2', creating blank disk +INFO[0000] Created volume 'trident-e2e-base_vm-host-vm-0-volume-0.qcow2' +DEBU[0000] Setting up volume 'trident-e2e-base_vm-host-vm-0-volume-1.qcow2' at path '/var/lib/libvirt/trident-e2e-base_vm-host/trident-e2e-base_vm-host-vm-0-volume-1.qcow2' +TRAC[0000] Volume trident-e2e-base_vm-host-vm-0-volume-1.qcow2 does not exist, skipping deletion +TRAC[0000] Defining volume with XML: + + trident-e2e-base_vm-host-vm-0-volume-1.qcow2 + 32 + + /var/lib/libvirt/trident-e2e-base_vm-host/trident-e2e-base_vm-host-vm-0-volume-1.qcow2 + + + 0644 + + + +DEBU[0000] No OS disk specified for volume 'trident-e2e-base_vm-host-vm-0-volume-1.qcow2', creating blank disk +INFO[0000] Created volume 'trident-e2e-base_vm-host-vm-0-volume-1.qcow2' +TRAC[0000] Defining domain with XML: + + trident-e2e-base_vm-host-vm-0 + 12 + 4 + + + virtdeploy:1 + + + + hvm + /usr/share/OVMF/OVMF_CODE_4M.ms.fd + /var/lib/trident-e2e-base_vm-host-nvram/trident-e2e-base_vm-host-vm-0_VARS.fd + + + + + + + + + Broadwell-IBRS + + + + + + + + + + + + + /usr/bin/qemu-system-x86_64 + + + + +
+ + + + + +
+ + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +INFO[0000] Created domain 'trident-e2e-base_vm-host-vm-0' ]]> + + + /etc/sudoers.d/testing-user + name: testing-privilege + runOn: + - clean-install + - ab-update +storage: + abUpdate: + volumePairs: + - id: root + volumeAId: root-a + volumeBId: root-b + disks: + - device: /dev/disk/by-path/pci-0000:00:1f.2-ata-2 + id: os + partitionTableType: gpt + partitions: + - id: root-a + size: 8G + type: root + - id: root-b + size: 8G + type: root + - id: esp + size: 1G + type: esp + - id: swap + size: 2G + type: swap + - id: home + size: 1G + type: home + - id: trident + size: 1G + type: linux-generic + - device: /dev/disk/by-path/pci-0000:00:1f.2-ata-3 + id: disk2 + partitionTableType: gpt + partitions: [] + filesystems: + - deviceId: trident + mountPoint: /var/lib/trident + source: new + - deviceId: home + mountPoint: /home + source: new + - deviceId: esp + mountPoint: + options: umask=0077 + path: /boot/efi + - deviceId: root + mountPoint: / + swap: + - swap +WARN[0000] No announce IP specified. Attempting to find default outbound IP to announce. +TRAC[0000] Checking Route: {Ifindex: 2 Dst: 0.0.0.0/0 Src: 10.91.162.60 Gw: 10.91.160.1 Flags: [] Table: 254 Realm: 0} +INFO[0000] Announcing address address="10.91.162.60:4000" +INFO[0000] Using Trident config file: /tmp/hc-tmp-960555467 +INFO[0000] Listening... address="0.0.0.0:4000" +INFO[0000] Using local VM +INFO[0000] Initializing VM with UUID '808030b3-4a24-47d2-95aa-e14b97faf97d' +TRAC[0000] Domain XML: + + trident-e2e-base_vm-host-vm-0 + 808030b3-4a24-47d2-95aa-e14b97faf97d + 12582912 + 12582912 + 4 + + + virtdeploy:1 + + + + hvm + + + + + /usr/share/OVMF/OVMF_CODE_4M.ms.fd + /var/lib/trident-e2e-base_vm-host-nvram/trident-e2e-base_vm-host-vm-0_VARS.fd + + + + + + + + + + + Broadwell-IBRS + + + + + + + + destroy + restart + destroy + + + + + + /usr/bin/qemu-system-x86_64 + + + + +
+ + + + + +
+ + + + + +
+ + +
+ + + +
+ + + +
+ + + +
+ + +
+ + +
+ + + + + +
+ + + + +
+ + + +
+ + + + +
+ + + + +
+ + + + +
+ + +
+ + + + + +
+ + + + + + + + + + + + + +
+ + +
+ + + + + + + + + + + +
+ +