From 7e4127e7657e763bbafcba67a0032f6716e6ac56 Mon Sep 17 00:00:00 2001 From: Alejandro Gullon Date: Thu, 3 Sep 2026 10:10:38 +0200 Subject: [PATCH 1/2] fix: increase greenboot and robot timeouts for release scenarios Release scenarios running upgrade paths with LVMS workloads followed by full standard suites were hitting timeout limits under I/O contention on x86 (c5.metal, 4750 Mbps EBS) when many VMs boot and pull images in parallel. Increase greenboot healthcheck timeout from 600s to 1200s and robot framework timeout from 30m (CI-overridden to 45m) to 60m for release scenarios only. Co-Authored-By: Claude Opus 4.6 (1M context) pre-commit.check-secrets: ENABLED --- test/bin/ci_phase_boot_and_test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index 56aef1cb30..ce8e77d56e 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -116,6 +116,12 @@ elif [[ "${SCENARIO_SOURCES}" =~ .*releases.* ]]; then # hypervisor only ever holds the still-running scenarios' VMs. jobs_arg="-j 20" scenario_action="create-run-shutdown" + # Release scenarios run upgrade paths with LVMS workloads followed by + # full standard suites, which need more time than the default 30m robot + # timeout and 600s greenboot healthcheck — especially under I/O + # contention when many VMs boot and pull images in parallel. + export GREENBOOT_TIMEOUT=1200 + export TEST_EXECUTION_TIMEOUT=60m fi TEST_OK=true From 722d7fcc7abaf97a6540003343250e4471cb822f Mon Sep 17 00:00:00 2001 From: Alejandro Gullon Date: Thu, 3 Sep 2026 10:17:33 +0200 Subject: [PATCH 2/2] chore: remove per-scenario timeout overrides now centralized GREENBOOT_TIMEOUT=1200 and TEST_EXECUTION_TIMEOUT=60m are now set centrally in ci_phase_boot_and_test.sh for all release scenarios. Remove the redundant per-scenario overrides. Co-Authored-By: Claude Opus 4.6 (1M context) pre-commit.check-secrets: ENABLED --- .../el10/releases/el102-lrel@optional-sigstore.sh | 9 --------- .../scenarios-bootc/el10/releases/el102-lrel@optional.sh | 9 --------- test/scenarios-bootc/el10/releases/el102@rpm-standard.sh | 3 --- .../el9/releases/el98-lrel@optional-sigstore.sh | 9 --------- test/scenarios-bootc/el9/releases/el98-lrel@optional.sh | 9 --------- 5 files changed, 39 deletions(-) diff --git a/test/scenarios-bootc/el10/releases/el102-lrel@optional-sigstore.sh b/test/scenarios-bootc/el10/releases/el102-lrel@optional-sigstore.sh index e657720bb7..fd25080fd0 100644 --- a/test/scenarios-bootc/el10/releases/el102-lrel@optional-sigstore.sh +++ b/test/scenarios-bootc/el10/releases/el102-lrel@optional-sigstore.sh @@ -2,15 +2,6 @@ # Sourced from scenario.sh and uses functions defined there. -# Each optional suite restarts MicroShift with its own kustomizePaths config, -# adding ~10 minutes of restart overhead to the total execution time. -# shellcheck disable=SC2034 # used elsewhere -TEST_EXECUTION_TIMEOUT=60m - -# shellcheck disable=SC2034 # used elsewhere -# Increase greenboot timeout for optional packages (more services to start) -GREENBOOT_TIMEOUT=1200 - # Enable container signature verification for current release images, # including the optional components. # These are ec / rc / z-stream, thus must all to be signed. diff --git a/test/scenarios-bootc/el10/releases/el102-lrel@optional.sh b/test/scenarios-bootc/el10/releases/el102-lrel@optional.sh index 9a62cfccff..a17d90753d 100644 --- a/test/scenarios-bootc/el10/releases/el102-lrel@optional.sh +++ b/test/scenarios-bootc/el10/releases/el102-lrel@optional.sh @@ -2,15 +2,6 @@ # Sourced from scenario.sh and uses functions defined there. -# Each optional suite restarts MicroShift with its own kustomizePaths config, -# adding ~10 minutes of restart overhead to the total execution time. -# shellcheck disable=SC2034 # used elsewhere -TEST_EXECUTION_TIMEOUT=60m - -# shellcheck disable=SC2034 # used elsewhere -# Increase greenboot timeout for optional packages (more services to start) -GREENBOOT_TIMEOUT=1200 - # Redefine network-related settings to use the dedicated network bridge VM_BRIDGE_IP="$(get_vm_bridge_ip "${VM_MULTUS_NETWORK}")" # shellcheck disable=SC2034 # used elsewhere diff --git a/test/scenarios-bootc/el10/releases/el102@rpm-standard.sh b/test/scenarios-bootc/el10/releases/el102@rpm-standard.sh index 816814dbfb..aaa0b7da23 100644 --- a/test/scenarios-bootc/el10/releases/el102@rpm-standard.sh +++ b/test/scenarios-bootc/el10/releases/el102@rpm-standard.sh @@ -13,9 +13,6 @@ export SKIP_GREENBOOT=true # did not want to spend the resources on a new VM. export TEST_RANDOMIZATION=none -# Add extra timeout because it run both standard1 and standard2 suites. -export TEST_EXECUTION_TIMEOUT=60m - scenario_create_vms() { exit_if_brew_rpms_not_found diff --git a/test/scenarios-bootc/el9/releases/el98-lrel@optional-sigstore.sh b/test/scenarios-bootc/el9/releases/el98-lrel@optional-sigstore.sh index fc078e23c9..e13d318a43 100644 --- a/test/scenarios-bootc/el9/releases/el98-lrel@optional-sigstore.sh +++ b/test/scenarios-bootc/el9/releases/el98-lrel@optional-sigstore.sh @@ -2,15 +2,6 @@ # Sourced from scenario.sh and uses functions defined there. -# Each optional suite restarts MicroShift with its own kustomizePaths config, -# adding ~10 minutes of restart overhead to the total execution time. -# shellcheck disable=SC2034 # used elsewhere -TEST_EXECUTION_TIMEOUT=60m - -# shellcheck disable=SC2034 # used elsewhere -# Increase greenboot timeout for optional packages (more services to start) -GREENBOOT_TIMEOUT=1200 - # Enable container signature verification for current release images, # including the optional components. # These are ec / rc / z-stream, thus must all to be signed. diff --git a/test/scenarios-bootc/el9/releases/el98-lrel@optional.sh b/test/scenarios-bootc/el9/releases/el98-lrel@optional.sh index a4d3782b8e..2d271f649d 100644 --- a/test/scenarios-bootc/el9/releases/el98-lrel@optional.sh +++ b/test/scenarios-bootc/el9/releases/el98-lrel@optional.sh @@ -2,15 +2,6 @@ # Sourced from scenario.sh and uses functions defined there. -# Each optional suite restarts MicroShift with its own kustomizePaths config, -# adding ~10 minutes of restart overhead to the total execution time. -# shellcheck disable=SC2034 # used elsewhere -TEST_EXECUTION_TIMEOUT=60m - -# shellcheck disable=SC2034 # used elsewhere -# Increase greenboot timeout for optional packages (more services to start) -GREENBOOT_TIMEOUT=1200 - # Redefine network-related settings to use the dedicated network bridge VM_BRIDGE_IP="$(get_vm_bridge_ip "${VM_MULTUS_NETWORK}")" # shellcheck disable=SC2034 # used elsewhere