From 3f4d7076bb156f74a3dba20b4116f27fbd70b977 Mon Sep 17 00:00:00 2001 From: Alejandro Gullon Date: Thu, 3 Sep 2026 10:10:06 +0200 Subject: [PATCH 1/2] fix: increase greenboot and robot timeouts for release scenarios Release scenarios running upgrade paths with LVMS workloads followed by full standard suites were hitting timeout limits under I/O contention on x86 (c5.metal, 4750 Mbps EBS) when many VMs boot and pull images in parallel. Increase greenboot healthcheck timeout from 600s to 1200s and robot framework timeout from 30m (CI-overridden to 45m) to 60m for release scenarios only. Co-Authored-By: Claude Opus 4.6 (1M context) pre-commit.check-secrets: ENABLED --- test/bin/ci_phase_boot_and_test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index 56aef1cb30..ce8e77d56e 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -116,6 +116,12 @@ elif [[ "${SCENARIO_SOURCES}" =~ .*releases.* ]]; then # hypervisor only ever holds the still-running scenarios' VMs. jobs_arg="-j 20" scenario_action="create-run-shutdown" + # Release scenarios run upgrade paths with LVMS workloads followed by + # full standard suites, which need more time than the default 30m robot + # timeout and 600s greenboot healthcheck — especially under I/O + # contention when many VMs boot and pull images in parallel. + export GREENBOOT_TIMEOUT=1200 + export TEST_EXECUTION_TIMEOUT=60m fi TEST_OK=true From a2b7743e4516ee6102d830a2836a9249de2f8598 Mon Sep 17 00:00:00 2001 From: Alejandro Gullon Date: Thu, 3 Sep 2026 10:17:30 +0200 Subject: [PATCH 2/2] chore: remove per-scenario timeout overrides now centralized GREENBOOT_TIMEOUT=1200 and TEST_EXECUTION_TIMEOUT=60m are now set centrally in ci_phase_boot_and_test.sh for all release scenarios. Remove the redundant per-scenario overrides. Co-Authored-By: Claude Opus 4.6 (1M context) pre-commit.check-secrets: ENABLED --- .../el10/releases/el102-lrel@optional-sigstore.sh | 9 --------- .../scenarios-bootc/el10/releases/el102-lrel@optional.sh | 9 --------- test/scenarios-bootc/el10/releases/el102@rpm-standard.sh | 3 --- .../el9/releases/el98-lrel@optional-sigstore.sh | 9 --------- test/scenarios-bootc/el9/releases/el98-lrel@optional.sh | 9 --------- 5 files changed, 39 deletions(-) diff --git a/test/scenarios-bootc/el10/releases/el102-lrel@optional-sigstore.sh b/test/scenarios-bootc/el10/releases/el102-lrel@optional-sigstore.sh index e657720bb7..fd25080fd0 100644 --- a/test/scenarios-bootc/el10/releases/el102-lrel@optional-sigstore.sh +++ b/test/scenarios-bootc/el10/releases/el102-lrel@optional-sigstore.sh @@ -2,15 +2,6 @@ # Sourced from scenario.sh and uses functions defined there. -# Each optional suite restarts MicroShift with its own kustomizePaths config, -# adding ~10 minutes of restart overhead to the total execution time. -# shellcheck disable=SC2034 # used elsewhere -TEST_EXECUTION_TIMEOUT=60m - -# shellcheck disable=SC2034 # used elsewhere -# Increase greenboot timeout for optional packages (more services to start) -GREENBOOT_TIMEOUT=1200 - # Enable container signature verification for current release images, # including the optional components. # These are ec / rc / z-stream, thus must all to be signed. diff --git a/test/scenarios-bootc/el10/releases/el102-lrel@optional.sh b/test/scenarios-bootc/el10/releases/el102-lrel@optional.sh index 9a62cfccff..a17d90753d 100644 --- a/test/scenarios-bootc/el10/releases/el102-lrel@optional.sh +++ b/test/scenarios-bootc/el10/releases/el102-lrel@optional.sh @@ -2,15 +2,6 @@ # Sourced from scenario.sh and uses functions defined there. -# Each optional suite restarts MicroShift with its own kustomizePaths config, -# adding ~10 minutes of restart overhead to the total execution time. -# shellcheck disable=SC2034 # used elsewhere -TEST_EXECUTION_TIMEOUT=60m - -# shellcheck disable=SC2034 # used elsewhere -# Increase greenboot timeout for optional packages (more services to start) -GREENBOOT_TIMEOUT=1200 - # Redefine network-related settings to use the dedicated network bridge VM_BRIDGE_IP="$(get_vm_bridge_ip "${VM_MULTUS_NETWORK}")" # shellcheck disable=SC2034 # used elsewhere diff --git a/test/scenarios-bootc/el10/releases/el102@rpm-standard.sh b/test/scenarios-bootc/el10/releases/el102@rpm-standard.sh index 816814dbfb..aaa0b7da23 100644 --- a/test/scenarios-bootc/el10/releases/el102@rpm-standard.sh +++ b/test/scenarios-bootc/el10/releases/el102@rpm-standard.sh @@ -13,9 +13,6 @@ export SKIP_GREENBOOT=true # did not want to spend the resources on a new VM. export TEST_RANDOMIZATION=none -# Add extra timeout because it run both standard1 and standard2 suites. -export TEST_EXECUTION_TIMEOUT=60m - scenario_create_vms() { exit_if_brew_rpms_not_found diff --git a/test/scenarios-bootc/el9/releases/el98-lrel@optional-sigstore.sh b/test/scenarios-bootc/el9/releases/el98-lrel@optional-sigstore.sh index fc078e23c9..e13d318a43 100644 --- a/test/scenarios-bootc/el9/releases/el98-lrel@optional-sigstore.sh +++ b/test/scenarios-bootc/el9/releases/el98-lrel@optional-sigstore.sh @@ -2,15 +2,6 @@ # Sourced from scenario.sh and uses functions defined there. -# Each optional suite restarts MicroShift with its own kustomizePaths config, -# adding ~10 minutes of restart overhead to the total execution time. -# shellcheck disable=SC2034 # used elsewhere -TEST_EXECUTION_TIMEOUT=60m - -# shellcheck disable=SC2034 # used elsewhere -# Increase greenboot timeout for optional packages (more services to start) -GREENBOOT_TIMEOUT=1200 - # Enable container signature verification for current release images, # including the optional components. # These are ec / rc / z-stream, thus must all to be signed. diff --git a/test/scenarios-bootc/el9/releases/el98-lrel@optional.sh b/test/scenarios-bootc/el9/releases/el98-lrel@optional.sh index a4d3782b8e..2d271f649d 100644 --- a/test/scenarios-bootc/el9/releases/el98-lrel@optional.sh +++ b/test/scenarios-bootc/el9/releases/el98-lrel@optional.sh @@ -2,15 +2,6 @@ # Sourced from scenario.sh and uses functions defined there. -# Each optional suite restarts MicroShift with its own kustomizePaths config, -# adding ~10 minutes of restart overhead to the total execution time. -# shellcheck disable=SC2034 # used elsewhere -TEST_EXECUTION_TIMEOUT=60m - -# shellcheck disable=SC2034 # used elsewhere -# Increase greenboot timeout for optional packages (more services to start) -GREENBOOT_TIMEOUT=1200 - # Redefine network-related settings to use the dedicated network bridge VM_BRIDGE_IP="$(get_vm_bridge_ip "${VM_MULTUS_NETWORK}")" # shellcheck disable=SC2034 # used elsewhere