infra: Full migration of emulated baremetal e2e tests to storm-based runner

.gitignore

@@ -366,4 +366,11 @@ vendor/
 
 # Virtdeploy files
 /tools/vm-netlaunch.yaml
-/tools/virt-deploy-metadata.json
+/tools/virt-deploy-metadata.json
+# Local storm-trident e2e run outputs
+/junit*.xml
+/logs*/
+/out*/
+/logstream*.log
+/*-metrics*.jsonl
+/metrics-*.jsonl

.pipelines/templates/e2e-template.yml

@@ -119,6 +119,7 @@ stages:
   - template: stages/testing_e2e/storm_e2e.yml
     parameters:
       stageType: ${{ parameters.stageType }}
+      acrServiceConnectionName: ${{ parameters.acrServiceConnectionName }}
 
   # Makefile validation, only for CI
   - ${{ if eq(parameters.stageType, 'ci') }}:

.pipelines/templates/stages/common_tasks/build-storm-trident.yml

@@ -0,0 +1,38 @@
+# Builds bin/storm-trident from the checked-out source. Used by the shortcut
+# (testing-trident.yml) storm E2E path so the test matrix and validation logic
+# come from the branch under test rather than a downloaded, possibly-stale
+# go-tools artifact. Mirrors the storm-trident build in building-tools.yml.
+#
+# Assumes the trident sources are already checked out (checkout_trident.yml) and
+# runs on a build-capable amd64 pool (e.g. trident-ubuntu-1es-pool-eastus2).
+steps:
+  - template: avoid-pypi-usage.yml
+  - template: update-protoc.yml
+    parameters:
+      protocArch: x86_64
+  - bash: |
+      set -eux
+      go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+      go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+      echo "Adding '$(go env GOPATH)/bin' to PATH for subsequent tasks"
+      echo "##vso[task.prependpath]$(go env GOPATH)/bin"
+    displayName: "Install protoc-gen-go"
+  - bash: |
+      set -eux
+      # `make bin/storm-trident` only regenerates the embedded configurations;
+      # the protobuf (pkg/tridentgrpc) and TLS cert (pkg/rcp/tlscerts) sources
+      # are generated by other targets (e.g. bin/netlaunch) in a full build.
+      # Generate them here so the standalone storm-trident build succeeds.
+      #
+      # Remove any pre-existing binary first: the test jobs download a go-tools
+      # artifact (a possibly-stale storm-trident) before this step, and make's
+      # timestamp check would otherwise treat that binary as up-to-date and skip
+      # the rebuild, silently using the stale binary.
+      rm -f bin/storm-trident
+      cd tools
+      go generate pkg/rcp/tlscerts/certs.go
+      go generate pkg/tridentgrpc/grpc.go
+      cd ..
+      make bin/storm-trident
+    displayName: "Build storm-trident from source"
+    workingDirectory: $(TRIDENT_SOURCE_DIR)

.pipelines/templates/stages/testing_e2e/build-storm-trident-stage.yml

@@ -0,0 +1,38 @@
+# Builds storm-trident once from the checked-out source and publishes it as a
+# pipeline artifact, so the E2E stages (matrix definition + test execution) can
+# consume the branch-under-test binary instead of a downloaded, possibly-stale
+# go-tools artifact. Used by the testing-trident.yml shortcut (testingRun).
+parameters:
+  - name: stageName
+    type: string
+    default: BuildStormTridentE2E
+
+  # Name of the published artifact containing the storm-trident binary.
+  - name: artifactName
+    type: string
+    default: storm-trident-e2e
+
+stages:
+  - stage: ${{ parameters.stageName }}
+    displayName: Build storm-trident for E2E
+    dependsOn: []
+    jobs:
+      - job: BuildStormTrident
+        displayName: Build storm-trident from source
+        timeoutInMinutes: 15
+        pool:
+          type: linux
+          name: trident-ubuntu-1es-pool-eastus2
+          hostArchitecture: amd64
+        variables:
+          ob_outputDirectory: /tmp/${{ parameters.artifactName }}
+          ob_artifactBaseName: ${{ parameters.artifactName }}
+        steps:
+          - template: ../common_tasks/checkout_trident.yml
+          - template: ../common_tasks/build-storm-trident.yml
+          - bash: |
+              set -eux
+              mkdir -p $(ob_outputDirectory)
+              cp bin/storm-trident $(ob_outputDirectory)/storm-trident
+            workingDirectory: $(TRIDENT_SOURCE_DIR)
+            displayName: "Stage storm-trident artifact"

.pipelines/templates/stages/testing_e2e/storm_e2e.yml

@@ -3,15 +3,47 @@ parameters:
     displayName: "Pipeline configuration type"
     type: string
 
+  - name: acrServiceConnectionName
+    displayName: "Service connection used to push extension/COSI images to ACR"
+    type: string
+    default: trident-dev-acr-write-umi-ECF
+
+  # When true, the E2E stages consume pre-built artifacts from the
+  # DownloadTestingElements stage (shortcut / testing-trident.yml) instead of
+  # depending on the in-pipeline build stages.
+  - name: testingRun
+    type: boolean
+    default: false
+
+  # Which VM runtimes to run. Both default to true so the full pipeline is
+  # unchanged; the shortcut pipeline toggles them individually.
+  - name: runVMHost
+    type: boolean
+    default: true
+
+  - name: runVMContainer
+    type: boolean
+    default: true
+
 stages:
+  # In a testingRun (shortcut), build storm-trident once from source and publish
+  # it so the matrix-definition and test-execution stages consume the
+  # branch-under-test binary instead of a downloaded, possibly-stale go-tools
+  # artifact.
+  - ${{ if eq(parameters.testingRun, true) }}:
+      - template: build-storm-trident-stage.yml
+
   - stage: DefineTests_E2E
     displayName: Define all E2E test matrices
     dependsOn:
-      - BuildingTools
+      - ${{ if eq(parameters.testingRun, true) }}:
+          - BuildStormTridentE2E
+      - ${{ else }}:
+          - BuildingTools
     jobs:
       - job: DefineTests
         displayName: Get List of tests to run
-        timeoutInMinutes: 10
+        timeoutInMinutes: 15
         pool:
           type: linux
 
@@ -21,32 +53,49 @@ stages:
 
         steps:
           - template: ../common_tasks/checkout_trident.yml
-          - task: DownloadPipelineArtifact@2
-            displayName: "Download go-tools"
-            inputs:
-              buildType: current
-              artifactName: "go-tools"
-              patterns: |
-                storm-trident
-              targetPath: "$(TRIDENT_SOURCE_DIR)/bin"
+          # In a testingRun (shortcut) consume the storm-trident built from the
+          # branch under test (BuildStormTridentE2E); otherwise download it from
+          # the in-pipeline build's go-tools artifact.
+          - ${{ if eq(parameters.testingRun, true) }}:
+              - task: DownloadPipelineArtifact@2
+                displayName: "Download storm-trident (built from source)"
+                inputs:
+                  buildType: current
+                  artifactName: "storm-trident-e2e"
+                  targetPath: "$(TRIDENT_SOURCE_DIR)/bin"
+          - ${{ else }}:
+              - task: DownloadPipelineArtifact@2
+                displayName: "Download go-tools"
+                inputs:
+                  buildType: current
+                  artifactName: "go-tools"
+                  patterns: |
+                    storm-trident
+                  targetPath: "$(TRIDENT_SOURCE_DIR)/bin"
           - bash: |
               chmod +x ./bin/storm-trident
               ./bin/storm-trident script e2e-matrix "${{ parameters.stageType }}"
             name: matrices
             workingDirectory: $(TRIDENT_SOURCE_DIR)
             displayName: Matrix of Trident configurations for E2E Tests
 
-  # TODO: enable once storm tests are ready
-  - template: test_execution_template.yml
-    parameters:
-      hardwareType: "VM"
-      runtimeType: "HOST"
+  - ${{ if parameters.runVMHost }}:
+      - template: test_execution_template.yml
+        parameters:
+          hardwareType: "VM"
+          runtimeType: "HOST"
+          testingRun: ${{ parameters.testingRun }}
+          acrServiceConnectionName: ${{ parameters.acrServiceConnectionName }}
 
-  # - template: test_execution_template.yml
-  #   parameters:
-  #     hardwareType: "VM"
-  #     runtimeType: "CONTAINER"
+  - ${{ if parameters.runVMContainer }}:
+      - template: test_execution_template.yml
+        parameters:
+          hardwareType: "VM"
+          runtimeType: "CONTAINER"
+          testingRun: ${{ parameters.testingRun }}
+          acrServiceConnectionName: ${{ parameters.acrServiceConnectionName }}
 
+  # TODO: enable once bare-metal host setup is supported in storm
   # - template: test_execution_template.yml
   #   parameters:
   #     hardwareType: "BM"