From 8185cc0c8bf46e80d4060e22afdd4a57f180c720 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Wed, 29 Jul 2026 13:05:51 -0500
Subject: [PATCH 01/10] INTEROP-9265: Add ACM operator upgrade step for OPP
 product upgrades

Add step registry entry at interop/opp/product-upgrade/acm/ that
upgrades ACM via OLM subscription channel change and validates the
operator reaches Succeeded phase. Includes MCE co-upgrade validation
and hub health checks (MCH phase, policy propagator, managed clusters).

This step gates downstream product upgrades (ACS, ODF, Quay) in the
OPP coordinated product upgrade workflow (INTEROP-8941).
---
 .../interop/opp/product-upgrade/OWNERS        |   3 +
 .../interop/opp/product-upgrade/acm/OWNERS    |   3 +
 ...nterop-opp-product-upgrade-acm-commands.sh | 370 ++++++++++++++++++
 ...-opp-product-upgrade-acm-ref.metadata.json |   4 +
 .../interop-opp-product-upgrade-acm-ref.yaml  |  49 +++
 5 files changed, 429 insertions(+)
 create mode 100644 ci-operator/step-registry/interop/opp/product-upgrade/OWNERS
 create mode 100644 ci-operator/step-registry/interop/opp/product-upgrade/acm/OWNERS
 create mode 100755 ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
 create mode 100644 ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.metadata.json
 create mode 100644 ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.yaml

diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/OWNERS b/ci-operator/step-registry/interop/opp/product-upgrade/OWNERS
new file mode 100644
index 0000000000000..41d144d3728a2
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/OWNERS
@@ -0,0 +1,3 @@
+approvers: &owners
+- cspi-qe-ocp-lp
+reviewers: *owners
diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/OWNERS b/ci-operator/step-registry/interop/opp/product-upgrade/acm/OWNERS
new file mode 100644
index 0000000000000..41d144d3728a2
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/OWNERS
@@ -0,0 +1,3 @@
+approvers: &owners
+- cspi-qe-ocp-lp
+reviewers: *owners
diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
new file mode 100755
index 0000000000000..c277d9578617f
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
@@ -0,0 +1,370 @@
+#!/bin/bash
+set -euo pipefail
+shopt -s inherit_errexit
+
+ACM_TARGET_CHANNEL="${ACM_TARGET_CHANNEL:-}"
+ACM_UPGRADE_TIMEOUT="${ACM_UPGRADE_TIMEOUT:-30m}"
+ACM_SUBSCRIPTION_NAME="${ACM_SUBSCRIPTION_NAME:-advanced-cluster-management}"
+ACM_SUBSCRIPTION_NAMESPACE="${ACM_SUBSCRIPTION_NAMESPACE:-open-cluster-management}"
+
+ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/artifacts}"
+mkdir -p "${ARTIFACT_DIR}"
+
+# shellcheck disable=SC2034
+typeset -i exit_code=0
+
+collect_diagnostics() {
+    local artifact_file="${ARTIFACT_DIR}/acm-upgrade-diagnostics.txt"
+    {
+        printf '=== ACM Operator Upgrade Diagnostics ===\n\n'
+        printf '=== Subscription ===\n'
+        oc get subscription "${ACM_SUBSCRIPTION_NAME}" -n "${ACM_SUBSCRIPTION_NAMESPACE}" -o yaml 2>&1 || true
+        printf '\n=== CSVs in %s ===\n' "${ACM_SUBSCRIPTION_NAMESPACE}"
+        oc get csv -n "${ACM_SUBSCRIPTION_NAMESPACE}" 2>&1 || true
+        printf '\n=== InstallPlan ===\n'
+        oc get installplan -n "${ACM_SUBSCRIPTION_NAMESPACE}" 2>&1 || true
+        printf '\n=== MCE CSVs ===\n'
+        oc get csv -n multicluster-engine 2>&1 || true
+        printf '\n=== Pods not Ready ===\n'
+        oc get pods -n "${ACM_SUBSCRIPTION_NAMESPACE}" --field-selector=status.phase!=Running,status.phase!=Succeeded 2>&1 || true
+        oc get pods -n multicluster-engine --field-selector=status.phase!=Running,status.phase!=Succeeded 2>&1 || true
+    } > "${artifact_file}"
+}
+
+trap 'exit_code=$?; if (( exit_code != 0 )); then collect_diagnostics; fi' EXIT
+
+get_current_csv() {
+    oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -o jsonpath='{.status.currentCSV}' 2>/dev/null
+}
+
+get_csv_phase() {
+    local csv_name="$1"
+    oc get csv "${csv_name}" \
+        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -o jsonpath='{.status.phase}' 2>/dev/null
+}
+
+get_installed_version() {
+    local csv_name
+    csv_name="$(get_current_csv)"
+    if [[ -z "${csv_name}" ]]; then
+        return 1
+    fi
+    oc get csv "${csv_name}" \
+        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -o jsonpath='{.spec.version}' 2>/dev/null
+}
+
+get_current_channel() {
+    oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -o jsonpath='{.spec.channel}' 2>/dev/null
+}
+
+resolve_target_channel() {
+    if [[ -n "${ACM_TARGET_CHANNEL}" ]]; then
+        echo "${ACM_TARGET_CHANNEL}"
+        return 0
+    fi
+
+    local current_channel
+    current_channel="$(get_current_channel)"
+    if [[ -z "${current_channel}" ]]; then
+        echo >&2 "ERROR: Cannot determine current subscription channel"
+        return 3
+    fi
+
+    local catalog_namespace
+    catalog_namespace="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -o jsonpath='{.spec.sourceNamespace}' 2>/dev/null)"
+
+    local package_name
+    package_name="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -o jsonpath='{.spec.name}' 2>/dev/null)"
+
+    local channels
+    channels="$(oc get packagemanifest "${package_name}" \
+        -n "${catalog_namespace}" \
+        -o jsonpath='{.status.channels[*].name}' 2>/dev/null)"
+
+    if [[ -z "${channels}" ]]; then
+        echo >&2 "ERROR: No channels found in packagemanifest for ${package_name}"
+        return 3
+    fi
+
+    local current_version next_channel=""
+    current_version="$(echo "${current_channel}" | grep -oE '[0-9]+\.[0-9]+' || true)"
+
+    for ch in ${channels}; do
+        local ch_version
+        ch_version="$(echo "${ch}" | grep -oE '[0-9]+\.[0-9]+' || true)"
+        if [[ -z "${ch_version}" ]]; then
+            continue
+        fi
+        if [[ -z "${current_version}" ]]; then
+            next_channel="${ch}"
+            break
+        fi
+        local current_major current_minor ch_major ch_minor
+        current_major="${current_version%%.*}"
+        current_minor="${current_version##*.}"
+        ch_major="${ch_version%%.*}"
+        ch_minor="${ch_version##*.}"
+
+        if (( ch_major > current_major )) || \
+           (( ch_major == current_major && ch_minor > current_minor )); then
+            if [[ -z "${next_channel}" ]]; then
+                next_channel="${ch}"
+            else
+                local next_version next_major next_minor
+                next_version="$(echo "${next_channel}" | grep -oE '[0-9]+\.[0-9]+' || true)"
+                next_major="${next_version%%.*}"
+                next_minor="${next_version##*.}"
+                if (( ch_major < next_major )) || \
+                   (( ch_major == next_major && ch_minor < next_minor )); then
+                    next_channel="${ch}"
+                fi
+            fi
+        fi
+    done
+
+    if [[ -z "${next_channel}" ]]; then
+        echo >&2 "ERROR: No upgrade channel found newer than ${current_channel}"
+        return 3
+    fi
+
+    echo "${next_channel}"
+}
+
+wait_for_csv_succeeded() {
+    local timeout_seconds
+    timeout_seconds="$(parse_timeout "${ACM_UPGRADE_TIMEOUT}")"
+    local start_time elapsed new_csv phase
+    start_time="$(date +%s)"
+
+    while true; do
+        elapsed="$(( $(date +%s) - start_time ))"
+        if (( elapsed > timeout_seconds )); then
+            echo >&2 "ERROR: Timeout (${ACM_UPGRADE_TIMEOUT}) waiting for CSV upgrade"
+            return 2
+        fi
+
+        new_csv="$(get_current_csv)"
+        if [[ -z "${new_csv}" ]]; then
+            sleep 10
+            continue
+        fi
+
+        phase="$(get_csv_phase "${new_csv}")"
+        echo "  CSV: ${new_csv}  Phase: ${phase}  (${elapsed}s elapsed)"
+
+        case "${phase}" in
+            Succeeded)
+                return 0
+                ;;
+            Failed)
+                echo >&2 "ERROR: CSV ${new_csv} entered Failed phase"
+                return 1
+                ;;
+            *)
+                sleep 15
+                ;;
+        esac
+    done
+}
+
+parse_timeout() {
+    local input="$1"
+    local minutes=0 seconds=0
+    if [[ "${input}" =~ ^([0-9]+)m$ ]]; then
+        minutes="${BASH_REMATCH[1]}"
+    elif [[ "${input}" =~ ^([0-9]+)s$ ]]; then
+        seconds="${BASH_REMATCH[1]}"
+    elif [[ "${input}" =~ ^([0-9]+)h$ ]]; then
+        minutes="$(( BASH_REMATCH[1] * 60 ))"
+    elif [[ "${input}" =~ ^([0-9]+)$ ]]; then
+        minutes="${input}"
+    fi
+    echo "$(( minutes * 60 + seconds ))"
+}
+
+validate_mce_upgrade() {
+    echo "Validating MCE (MultiCluster Engine) upgrade..."
+    local mce_csv
+    mce_csv="$(oc get csv -n multicluster-engine \
+        -o jsonpath='{.items[?(@.spec.displayName=="multicluster engine for Kubernetes")].metadata.name}' \
+        2>/dev/null || true)"
+
+    if [[ -z "${mce_csv}" ]]; then
+        mce_csv="$(oc get csv -n multicluster-engine \
+            -l operators.coreos.com/multicluster-engine.multicluster-engine= \
+            -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
+    fi
+
+    if [[ -z "${mce_csv}" ]]; then
+        echo "WARNING: MCE CSV not found; skipping MCE validation"
+        return 0
+    fi
+
+    local mce_phase
+    mce_phase="$(oc get csv "${mce_csv}" -n multicluster-engine \
+        -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+
+    echo "  MCE CSV: ${mce_csv}  Phase: ${mce_phase}"
+    if [[ "${mce_phase}" != "Succeeded" ]]; then
+        echo "WARNING: MCE CSV phase is ${mce_phase}, not Succeeded"
+        local timeout_end
+        timeout_end="$(( $(date +%s) + 300 ))"
+        while (( $(date +%s) < timeout_end )); do
+            mce_phase="$(oc get csv "${mce_csv}" -n multicluster-engine \
+                -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+            if [[ "${mce_phase}" == "Succeeded" ]]; then
+                echo "  MCE CSV reached Succeeded phase"
+                return 0
+            fi
+            sleep 15
+        done
+        echo >&2 "ERROR: MCE CSV did not reach Succeeded within 5 minutes"
+        return 1
+    fi
+    return 0
+}
+
+validate_hub_health() {
+    echo "Validating ACM hub health post-upgrade..."
+
+    local mch_status
+    mch_status="$(oc get multiclusterhub -A \
+        -o jsonpath='{.items[0].status.phase}' 2>/dev/null || true)"
+    echo "  MultiClusterHub phase: ${mch_status}"
+
+    if [[ "${mch_status}" != "Running" ]]; then
+        echo "  Waiting for MCH to reach Running phase (timeout: 5m)..."
+        local timeout_end
+        timeout_end="$(( $(date +%s) + 300 ))"
+        while (( $(date +%s) < timeout_end )); do
+            mch_status="$(oc get multiclusterhub -A \
+                -o jsonpath='{.items[0].status.phase}' 2>/dev/null || true)"
+            if [[ "${mch_status}" == "Running" ]]; then
+                break
+            fi
+            sleep 15
+        done
+        if [[ "${mch_status}" != "Running" ]]; then
+            echo >&2 "ERROR: MultiClusterHub did not reach Running phase"
+            return 1
+        fi
+    fi
+
+    echo "  Checking policy propagator..."
+    local propagator_ready
+    propagator_ready="$(oc get pods -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -l name=governance-policy-propagator \
+        -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' \
+        2>/dev/null || true)"
+    echo "  Policy propagator ready: ${propagator_ready}"
+
+    echo "  Checking managed clusters..."
+    local cluster_count available_count
+    cluster_count="$(oc get managedclusters --no-headers 2>/dev/null | wc -l || echo 0)"
+    available_count="$(oc get managedclusters \
+        -o jsonpath='{.items[?(@.status.conditions[?(@.type=="ManagedClusterConditionAvailable")].status=="True")].metadata.name}' \
+        2>/dev/null | wc -w || echo 0)"
+    echo "  Managed clusters: ${available_count}/${cluster_count} available"
+
+    echo "ACM hub health validation complete"
+    return 0
+}
+
+# === Main ===
+
+echo "=== ACM Operator Upgrade Step ==="
+echo "Namespace: ${ACM_SUBSCRIPTION_NAMESPACE}"
+echo "Subscription: ${ACM_SUBSCRIPTION_NAME}"
+echo "Timeout: ${ACM_UPGRADE_TIMEOUT}"
+
+current_csv="$(get_current_csv)"
+if [[ -z "${current_csv}" ]]; then
+    echo >&2 "ERROR: No ACM subscription found or no currentCSV set"
+    exit 3
+fi
+
+current_version="$(get_installed_version)"
+current_channel="$(get_current_channel)"
+echo "Current: CSV=${current_csv} Version=${current_version} Channel=${current_channel}"
+
+target_channel="$(resolve_target_channel)"
+echo "Target channel: ${target_channel}"
+
+if [[ "${target_channel}" == "${current_channel}" ]]; then
+    echo "Already on target channel ${target_channel}; checking if upgrade is available..."
+    install_plan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -o jsonpath='{.status.installPlanRef.name}' 2>/dev/null || true)"
+    if [[ -z "${install_plan}" ]]; then
+        echo "No pending upgrade on current channel; nothing to do"
+        exit 0
+    fi
+fi
+
+echo "Patching subscription channel: ${current_channel} -> ${target_channel}"
+oc patch subscription "${ACM_SUBSCRIPTION_NAME}" \
+    -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+    --type merge \
+    -p "{\"spec\":{\"channel\":\"${target_channel}\"}}"
+
+echo "Waiting for InstallPlan to be created..."
+sleep 10
+
+install_plan=""
+for _ in $(seq 1 12); do
+    install_plan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -o jsonpath='{.status.installplan.name}' 2>/dev/null || true)"
+    if [[ -z "${install_plan}" ]]; then
+        install_plan="$(oc get installplan -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+            --sort-by=.metadata.creationTimestamp \
+            -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)"
+    fi
+    if [[ -n "${install_plan}" ]]; then
+        break
+    fi
+    sleep 10
+done
+
+if [[ -n "${install_plan}" ]]; then
+    echo "InstallPlan: ${install_plan}"
+    local_approval="$(oc get installplan "${install_plan}" \
+        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -o jsonpath='{.spec.approval}' 2>/dev/null || true)"
+    if [[ "${local_approval}" == "Manual" ]]; then
+        echo "Approving manual InstallPlan..."
+        oc patch installplan "${install_plan}" \
+            -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+            --type merge \
+            -p '{"spec":{"approved":true}}'
+    fi
+fi
+
+echo "Waiting for ACM CSV to reach Succeeded phase..."
+wait_for_csv_succeeded
+new_csv="$(get_current_csv)"
+new_version="$(get_installed_version)"
+echo "Upgrade complete: ${current_version} -> ${new_version} (CSV: ${new_csv})"
+
+validate_mce_upgrade
+validate_hub_health
+
+{
+    printf '=== ACM Operator Upgrade Summary ===\n'
+    printf 'Previous: %s (%s)\n' "${current_version}" "${current_channel}"
+    printf 'Current:  %s (%s)\n' "${new_version}" "${target_channel}"
+    printf 'CSV:      %s\n' "${new_csv}"
+    printf 'Status:   SUCCESS\n'
+} > "${ARTIFACT_DIR}/acm-upgrade-summary.txt"
+
+echo "=== ACM Operator Upgrade: SUCCESS ==="
diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.metadata.json b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.metadata.json
new file mode 100644
index 0000000000000..7c62f23974cc1
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.metadata.json
@@ -0,0 +1,4 @@
+{
+    "display-name": "interop-opp-product-upgrade-acm",
+    "description": "Upgrade ACM operator via OLM subscription channel change and validate completion"
+}
diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.yaml b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.yaml
new file mode 100644
index 0000000000000..53bc8d83a2808
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.yaml
@@ -0,0 +1,49 @@
+ref:
+  as: interop-opp-product-upgrade-acm
+  from: cli
+  grace_period: 5m
+  commands: interop-opp-product-upgrade-acm-commands.sh
+  timeout: 45m
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+  env:
+    - name: ACM_TARGET_CHANNEL
+      default: ""
+      documentation: |-
+        Target subscription channel for ACM operator upgrade (e.g. release-2.13).
+        When empty, the step determines the next available channel automatically
+        by querying the catalog for channels newer than the currently installed version.
+    - name: ACM_UPGRADE_TIMEOUT
+      default: "30m"
+      documentation: |-
+        Maximum time to wait for the ACM operator upgrade to complete.
+        Covers CSV transition from Replacing to Succeeded phase.
+    - name: ACM_SUBSCRIPTION_NAME
+      default: "advanced-cluster-management"
+      documentation: |-
+        Name of the ACM Subscription resource in the target namespace.
+    - name: ACM_SUBSCRIPTION_NAMESPACE
+      default: "open-cluster-management"
+      documentation: |-
+        Namespace containing the ACM Subscription and CSV resources.
+  documentation: |-
+    Upgrades the ACM (Advanced Cluster Management) operator via OLM subscription
+    channel change and validates the upgrade completes successfully.
+
+    The step performs:
+    1. Identifies the currently installed ACM version via CSV
+    2. Patches the Subscription to the target channel (or next available)
+    3. Waits for the new CSV to reach Succeeded phase
+    4. Validates MCE (MultiCluster Engine) co-upgrade completion
+    5. Confirms hub connectivity and policy engine health
+
+    This step gates downstream product upgrades (ACS, ODF, Quay) in the
+    OPP coordinated product upgrade workflow.
+
+    Exit codes:
+      0 - upgrade successful, all health checks passed
+      1 - upgrade failed (CSV did not reach Succeeded)
+      2 - timeout waiting for upgrade
+      3 - precondition failure (no subscription found, no upgrade path)

From a616e00612d5bb71bec50c36e564dd00a3d63a6a Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Wed, 29 Jul 2026 13:35:43 -0500
Subject: [PATCH 02/10] Fix bugs from Chai Bot review: errexit safety,
 installPlanRef, timeout default

- Add || true to all oc get helpers to prevent inherit_errexit from
  killing callers before empty-string checks can run
- Use consistent .status.installPlanRef.name (not deprecated .installplan)
- Add fallback default (30m) for unrecognized timeout formats in parse_timeout
- Write acm-upgraded-version and acm-upgraded-channel to SHARED_DIR for
  downstream step consumption
---
 ...nterop-opp-product-upgrade-acm-commands.sh | 24 ++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
index c277d9578617f..45cfb87f98258 100755
--- a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
@@ -36,14 +36,14 @@ trap 'exit_code=$?; if (( exit_code != 0 )); then collect_diagnostics; fi' EXIT
 get_current_csv() {
     oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.status.currentCSV}' 2>/dev/null
+        -o jsonpath='{.status.currentCSV}' 2>/dev/null || true
 }
 
 get_csv_phase() {
     local csv_name="$1"
     oc get csv "${csv_name}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.status.phase}' 2>/dev/null
+        -o jsonpath='{.status.phase}' 2>/dev/null || true
 }
 
 get_installed_version() {
@@ -54,13 +54,13 @@ get_installed_version() {
     fi
     oc get csv "${csv_name}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.spec.version}' 2>/dev/null
+        -o jsonpath='{.spec.version}' 2>/dev/null || true
 }
 
 get_current_channel() {
     oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.spec.channel}' 2>/dev/null
+        -o jsonpath='{.spec.channel}' 2>/dev/null || true
 }
 
 resolve_target_channel() {
@@ -79,17 +79,17 @@ resolve_target_channel() {
     local catalog_namespace
     catalog_namespace="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.spec.sourceNamespace}' 2>/dev/null)"
+        -o jsonpath='{.spec.sourceNamespace}' 2>/dev/null || true)"
 
     local package_name
     package_name="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.spec.name}' 2>/dev/null)"
+        -o jsonpath='{.spec.name}' 2>/dev/null || true)"
 
     local channels
     channels="$(oc get packagemanifest "${package_name}" \
         -n "${catalog_namespace}" \
-        -o jsonpath='{.status.channels[*].name}' 2>/dev/null)"
+        -o jsonpath='{.status.channels[*].name}' 2>/dev/null || true)"
 
     if [[ -z "${channels}" ]]; then
         echo >&2 "ERROR: No channels found in packagemanifest for ${package_name}"
@@ -188,6 +188,9 @@ parse_timeout() {
         minutes="$(( BASH_REMATCH[1] * 60 ))"
     elif [[ "${input}" =~ ^([0-9]+)$ ]]; then
         minutes="${input}"
+    else
+        echo >&2 "WARNING: Unrecognized timeout format '${input}'; defaulting to 30m"
+        minutes=30
     fi
     echo "$(( minutes * 60 + seconds ))"
 }
@@ -324,7 +327,7 @@ install_plan=""
 for _ in $(seq 1 12); do
     install_plan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.status.installplan.name}' 2>/dev/null || true)"
+        -o jsonpath='{.status.installPlanRef.name}' 2>/dev/null || true)"
     if [[ -z "${install_plan}" ]]; then
         install_plan="$(oc get installplan -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
             --sort-by=.metadata.creationTimestamp \
@@ -367,4 +370,9 @@ validate_hub_health
     printf 'Status:   SUCCESS\n'
 } > "${ARTIFACT_DIR}/acm-upgrade-summary.txt"
 
+if [[ -n "${SHARED_DIR:-}" ]]; then
+    echo "${new_version}" > "${SHARED_DIR}/acm-upgraded-version"
+    echo "${target_channel}" > "${SHARED_DIR}/acm-upgraded-channel"
+fi
+
 echo "=== ACM Operator Upgrade: SUCCESS ==="

From df6065f3a5cbe2df371ea3a80b6117cea6cc5cd2 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Thu, 30 Jul 2026 14:46:48 -0500
Subject: [PATCH 03/10] Regenerate step registry metadata for ACM upgrade ref

The step-registry-metadata CI check requires auto-generated metadata
with path and owners fields. Replace the manually-written display-name
and description with the expected generated format.
---
 ...nterop-opp-product-upgrade-acm-ref.metadata.json | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.metadata.json b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.metadata.json
index 7c62f23974cc1..2581637b16588 100644
--- a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.metadata.json
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.metadata.json
@@ -1,4 +1,11 @@
 {
-    "display-name": "interop-opp-product-upgrade-acm",
-    "description": "Upgrade ACM operator via OLM subscription channel change and validate completion"
-}
+	"path": "interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-ref.yaml",
+	"owners": {
+		"approvers": [
+			"cspi-qe-ocp-lp"
+		],
+		"reviewers": [
+			"cspi-qe-ocp-lp"
+		]
+	}
+}
\ No newline at end of file

From 34337d4987c1423713f808cd15f36913d1055786 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Fri, 31 Jul 2026 13:18:54 -0500
Subject: [PATCH 04/10] Fix CSV upgrade race and stale InstallPlan detection

wait_for_csv_succeeded now skips iterations where currentCSV still
matches the pre-upgrade CSV, preventing false-positive success when
OLM has not yet processed the channel change.

Same-channel upgrade path now checks InstallPlan phase; a Complete
plan means no pending upgrade rather than relying on the presence of
installPlanRef.name (which persists after apply).
---
 .../acm/interop-opp-product-upgrade-acm-commands.sh  | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
index 45cfb87f98258..ec9867c648f01 100755
--- a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
@@ -141,6 +141,7 @@ resolve_target_channel() {
 }
 
 wait_for_csv_succeeded() {
+    local previous_csv="$1"
     local timeout_seconds
     timeout_seconds="$(parse_timeout "${ACM_UPGRADE_TIMEOUT}")"
     local start_time elapsed new_csv phase
@@ -154,7 +155,7 @@ wait_for_csv_succeeded() {
         fi
 
         new_csv="$(get_current_csv)"
-        if [[ -z "${new_csv}" ]]; then
+        if [[ -z "${new_csv}" || "${new_csv}" == "${previous_csv}" ]]; then
             sleep 10
             continue
         fi
@@ -312,6 +313,13 @@ if [[ "${target_channel}" == "${current_channel}" ]]; then
         echo "No pending upgrade on current channel; nothing to do"
         exit 0
     fi
+    plan_phase="$(oc get installplan "${install_plan}" \
+        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+    if [[ "${plan_phase}" == "Complete" ]]; then
+        echo "InstallPlan ${install_plan} already complete; no pending upgrade"
+        exit 0
+    fi
 fi
 
 echo "Patching subscription channel: ${current_channel} -> ${target_channel}"
@@ -354,7 +362,7 @@ if [[ -n "${install_plan}" ]]; then
 fi
 
 echo "Waiting for ACM CSV to reach Succeeded phase..."
-wait_for_csv_succeeded
+wait_for_csv_succeeded "${current_csv}"
 new_csv="$(get_current_csv)"
 new_version="$(get_installed_version)"
 echo "Upgrade complete: ${current_version} -> ${new_version} (CSV: ${new_csv})"

From f9e44993fec002be5723d0534f3ad48e1d4890c0 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Tue, 4 Aug 2026 12:58:39 -0500
Subject: [PATCH 05/10] fix: Apply mpitt best practices to ACM operator upgrade
 step

- set -euxo pipefail (add -x for xtrace)
- PascalCase functions with `function` keyword
- camelCase local/script variables via typeset
- Remove 2>/dev/null (xtrace needs visible output)
- Add terminal `true` for clean exit
---
 ...nterop-opp-product-upgrade-acm-commands.sh | 321 +++++++++---------
 1 file changed, 161 insertions(+), 160 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
index ec9867c648f01..a1fe763828489 100755
--- a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -euo pipefail
+set -euxo pipefail
 shopt -s inherit_errexit
 
 ACM_TARGET_CHANNEL="${ACM_TARGET_CHANNEL:-}"
@@ -11,10 +11,10 @@ ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/artifacts}"
 mkdir -p "${ARTIFACT_DIR}"
 
 # shellcheck disable=SC2034
-typeset -i exit_code=0
+typeset -i exitCode=0
 
-collect_diagnostics() {
-    local artifact_file="${ARTIFACT_DIR}/acm-upgrade-diagnostics.txt"
+function CollectDiagnostics () {
+    local artifactFile="${ARTIFACT_DIR}/acm-upgrade-diagnostics.txt"
     {
         printf '=== ACM Operator Upgrade Diagnostics ===\n\n'
         printf '=== Subscription ===\n'
@@ -28,147 +28,147 @@ collect_diagnostics() {
         printf '\n=== Pods not Ready ===\n'
         oc get pods -n "${ACM_SUBSCRIPTION_NAMESPACE}" --field-selector=status.phase!=Running,status.phase!=Succeeded 2>&1 || true
         oc get pods -n multicluster-engine --field-selector=status.phase!=Running,status.phase!=Succeeded 2>&1 || true
-    } > "${artifact_file}"
+    } > "${artifactFile}"
 }
 
-trap 'exit_code=$?; if (( exit_code != 0 )); then collect_diagnostics; fi' EXIT
+trap 'exitCode=$?; if (( exitCode != 0 )); then CollectDiagnostics; fi' EXIT
 
-get_current_csv() {
+function GetCurrentCsv () {
     oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.status.currentCSV}' 2>/dev/null || true
+        -o jsonpath='{.status.currentCSV}' || true
 }
 
-get_csv_phase() {
-    local csv_name="$1"
-    oc get csv "${csv_name}" \
+function GetCsvPhase () {
+    local csvName="$1"
+    oc get csv "${csvName}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.status.phase}' 2>/dev/null || true
+        -o jsonpath='{.status.phase}' || true
 }
 
-get_installed_version() {
-    local csv_name
-    csv_name="$(get_current_csv)"
-    if [[ -z "${csv_name}" ]]; then
+function GetInstalledVersion () {
+    local csvName
+    csvName="$(GetCurrentCsv)"
+    if [[ -z "${csvName}" ]]; then
         return 1
     fi
-    oc get csv "${csv_name}" \
+    oc get csv "${csvName}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.spec.version}' 2>/dev/null || true
+        -o jsonpath='{.spec.version}' || true
 }
 
-get_current_channel() {
+function GetCurrentChannel () {
     oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.spec.channel}' 2>/dev/null || true
+        -o jsonpath='{.spec.channel}' || true
 }
 
-resolve_target_channel() {
+function ResolveTargetChannel () {
     if [[ -n "${ACM_TARGET_CHANNEL}" ]]; then
         echo "${ACM_TARGET_CHANNEL}"
         return 0
     fi
 
-    local current_channel
-    current_channel="$(get_current_channel)"
-    if [[ -z "${current_channel}" ]]; then
+    local currentChannel
+    currentChannel="$(GetCurrentChannel)"
+    if [[ -z "${currentChannel}" ]]; then
         echo >&2 "ERROR: Cannot determine current subscription channel"
         return 3
     fi
 
-    local catalog_namespace
-    catalog_namespace="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+    local catalogNamespace
+    catalogNamespace="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.spec.sourceNamespace}' 2>/dev/null || true)"
+        -o jsonpath='{.spec.sourceNamespace}' || true)"
 
-    local package_name
-    package_name="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+    local packageName
+    packageName="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.spec.name}' 2>/dev/null || true)"
+        -o jsonpath='{.spec.name}' || true)"
 
     local channels
-    channels="$(oc get packagemanifest "${package_name}" \
-        -n "${catalog_namespace}" \
-        -o jsonpath='{.status.channels[*].name}' 2>/dev/null || true)"
+    channels="$(oc get packagemanifest "${packageName}" \
+        -n "${catalogNamespace}" \
+        -o jsonpath='{.status.channels[*].name}' || true)"
 
     if [[ -z "${channels}" ]]; then
-        echo >&2 "ERROR: No channels found in packagemanifest for ${package_name}"
+        echo >&2 "ERROR: No channels found in packagemanifest for ${packageName}"
         return 3
     fi
 
-    local current_version next_channel=""
-    current_version="$(echo "${current_channel}" | grep -oE '[0-9]+\.[0-9]+' || true)"
+    local currentVersion nextChannel=""
+    currentVersion="$(echo "${currentChannel}" | grep -oE '[0-9]+\.[0-9]+' || true)"
 
     for ch in ${channels}; do
-        local ch_version
-        ch_version="$(echo "${ch}" | grep -oE '[0-9]+\.[0-9]+' || true)"
-        if [[ -z "${ch_version}" ]]; then
+        local chVersion
+        chVersion="$(echo "${ch}" | grep -oE '[0-9]+\.[0-9]+' || true)"
+        if [[ -z "${chVersion}" ]]; then
             continue
         fi
-        if [[ -z "${current_version}" ]]; then
-            next_channel="${ch}"
+        if [[ -z "${currentVersion}" ]]; then
+            nextChannel="${ch}"
             break
         fi
-        local current_major current_minor ch_major ch_minor
-        current_major="${current_version%%.*}"
-        current_minor="${current_version##*.}"
-        ch_major="${ch_version%%.*}"
-        ch_minor="${ch_version##*.}"
-
-        if (( ch_major > current_major )) || \
-           (( ch_major == current_major && ch_minor > current_minor )); then
-            if [[ -z "${next_channel}" ]]; then
-                next_channel="${ch}"
+        local currentMajor currentMinor chMajor chMinor
+        currentMajor="${currentVersion%%.*}"
+        currentMinor="${currentVersion##*.}"
+        chMajor="${chVersion%%.*}"
+        chMinor="${chVersion##*.}"
+
+        if (( chMajor > currentMajor )) || \
+           (( chMajor == currentMajor && chMinor > currentMinor )); then
+            if [[ -z "${nextChannel}" ]]; then
+                nextChannel="${ch}"
             else
-                local next_version next_major next_minor
-                next_version="$(echo "${next_channel}" | grep -oE '[0-9]+\.[0-9]+' || true)"
-                next_major="${next_version%%.*}"
-                next_minor="${next_version##*.}"
-                if (( ch_major < next_major )) || \
-                   (( ch_major == next_major && ch_minor < next_minor )); then
-                    next_channel="${ch}"
+                local nextVersion nextMajor nextMinor
+                nextVersion="$(echo "${nextChannel}" | grep -oE '[0-9]+\.[0-9]+' || true)"
+                nextMajor="${nextVersion%%.*}"
+                nextMinor="${nextVersion##*.}"
+                if (( chMajor < nextMajor )) || \
+                   (( chMajor == nextMajor && chMinor < nextMinor )); then
+                    nextChannel="${ch}"
                 fi
             fi
         fi
     done
 
-    if [[ -z "${next_channel}" ]]; then
-        echo >&2 "ERROR: No upgrade channel found newer than ${current_channel}"
+    if [[ -z "${nextChannel}" ]]; then
+        echo >&2 "ERROR: No upgrade channel found newer than ${currentChannel}"
         return 3
     fi
 
-    echo "${next_channel}"
+    echo "${nextChannel}"
 }
 
-wait_for_csv_succeeded() {
-    local previous_csv="$1"
-    local timeout_seconds
-    timeout_seconds="$(parse_timeout "${ACM_UPGRADE_TIMEOUT}")"
-    local start_time elapsed new_csv phase
-    start_time="$(date +%s)"
+function WaitForCsvSucceeded () {
+    local previousCsv="$1"
+    local timeoutSeconds
+    timeoutSeconds="$(ParseTimeout "${ACM_UPGRADE_TIMEOUT}")"
+    local startTime elapsed newCsv phase
+    startTime="$(date +%s)"
 
     while true; do
-        elapsed="$(( $(date +%s) - start_time ))"
-        if (( elapsed > timeout_seconds )); then
+        elapsed="$(( $(date +%s) - startTime ))"
+        if (( elapsed > timeoutSeconds )); then
             echo >&2 "ERROR: Timeout (${ACM_UPGRADE_TIMEOUT}) waiting for CSV upgrade"
             return 2
         fi
 
-        new_csv="$(get_current_csv)"
-        if [[ -z "${new_csv}" || "${new_csv}" == "${previous_csv}" ]]; then
+        newCsv="$(GetCurrentCsv)"
+        if [[ -z "${newCsv}" || "${newCsv}" == "${previousCsv}" ]]; then
             sleep 10
             continue
         fi
 
-        phase="$(get_csv_phase "${new_csv}")"
-        echo "  CSV: ${new_csv}  Phase: ${phase}  (${elapsed}s elapsed)"
+        phase="$(GetCsvPhase "${newCsv}")"
+        echo "  CSV: ${newCsv}  Phase: ${phase}  (${elapsed}s elapsed)"
 
         case "${phase}" in
             Succeeded)
                 return 0
                 ;;
             Failed)
-                echo >&2 "ERROR: CSV ${new_csv} entered Failed phase"
+                echo >&2 "ERROR: CSV ${newCsv} entered Failed phase"
                 return 1
                 ;;
             *)
@@ -178,7 +178,7 @@ wait_for_csv_succeeded() {
     done
 }
 
-parse_timeout() {
+function ParseTimeout () {
     local input="$1"
     local minutes=0 seconds=0
     if [[ "${input}" =~ ^([0-9]+)m$ ]]; then
@@ -196,37 +196,37 @@ parse_timeout() {
     echo "$(( minutes * 60 + seconds ))"
 }
 
-validate_mce_upgrade() {
+function ValidateMceUpgrade () {
     echo "Validating MCE (MultiCluster Engine) upgrade..."
-    local mce_csv
-    mce_csv="$(oc get csv -n multicluster-engine \
+    local mceCsv
+    mceCsv="$(oc get csv -n multicluster-engine \
         -o jsonpath='{.items[?(@.spec.displayName=="multicluster engine for Kubernetes")].metadata.name}' \
-        2>/dev/null || true)"
+        || true)"
 
-    if [[ -z "${mce_csv}" ]]; then
-        mce_csv="$(oc get csv -n multicluster-engine \
+    if [[ -z "${mceCsv}" ]]; then
+        mceCsv="$(oc get csv -n multicluster-engine \
             -l operators.coreos.com/multicluster-engine.multicluster-engine= \
-            -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
+            -o jsonpath='{.items[0].metadata.name}' || true)"
     fi
 
-    if [[ -z "${mce_csv}" ]]; then
+    if [[ -z "${mceCsv}" ]]; then
         echo "WARNING: MCE CSV not found; skipping MCE validation"
         return 0
     fi
 
-    local mce_phase
-    mce_phase="$(oc get csv "${mce_csv}" -n multicluster-engine \
-        -o jsonpath='{.status.phase}' 2>/dev/null || true)"
-
-    echo "  MCE CSV: ${mce_csv}  Phase: ${mce_phase}"
-    if [[ "${mce_phase}" != "Succeeded" ]]; then
-        echo "WARNING: MCE CSV phase is ${mce_phase}, not Succeeded"
-        local timeout_end
-        timeout_end="$(( $(date +%s) + 300 ))"
-        while (( $(date +%s) < timeout_end )); do
-            mce_phase="$(oc get csv "${mce_csv}" -n multicluster-engine \
-                -o jsonpath='{.status.phase}' 2>/dev/null || true)"
-            if [[ "${mce_phase}" == "Succeeded" ]]; then
+    local mcePhase
+    mcePhase="$(oc get csv "${mceCsv}" -n multicluster-engine \
+        -o jsonpath='{.status.phase}' || true)"
+
+    echo "  MCE CSV: ${mceCsv}  Phase: ${mcePhase}"
+    if [[ "${mcePhase}" != "Succeeded" ]]; then
+        echo "WARNING: MCE CSV phase is ${mcePhase}, not Succeeded"
+        local timeoutEnd
+        timeoutEnd="$(( $(date +%s) + 300 ))"
+        while (( $(date +%s) < timeoutEnd )); do
+            mcePhase="$(oc get csv "${mceCsv}" -n multicluster-engine \
+                -o jsonpath='{.status.phase}' || true)"
+            if [[ "${mcePhase}" == "Succeeded" ]]; then
                 echo "  MCE CSV reached Succeeded phase"
                 return 0
             fi
@@ -238,47 +238,47 @@ validate_mce_upgrade() {
     return 0
 }
 
-validate_hub_health() {
+function ValidateHubHealth () {
     echo "Validating ACM hub health post-upgrade..."
 
-    local mch_status
-    mch_status="$(oc get multiclusterhub -A \
-        -o jsonpath='{.items[0].status.phase}' 2>/dev/null || true)"
-    echo "  MultiClusterHub phase: ${mch_status}"
+    local mchStatus
+    mchStatus="$(oc get multiclusterhub -A \
+        -o jsonpath='{.items[0].status.phase}' || true)"
+    echo "  MultiClusterHub phase: ${mchStatus}"
 
-    if [[ "${mch_status}" != "Running" ]]; then
+    if [[ "${mchStatus}" != "Running" ]]; then
         echo "  Waiting for MCH to reach Running phase (timeout: 5m)..."
-        local timeout_end
-        timeout_end="$(( $(date +%s) + 300 ))"
-        while (( $(date +%s) < timeout_end )); do
-            mch_status="$(oc get multiclusterhub -A \
-                -o jsonpath='{.items[0].status.phase}' 2>/dev/null || true)"
-            if [[ "${mch_status}" == "Running" ]]; then
+        local timeoutEnd
+        timeoutEnd="$(( $(date +%s) + 300 ))"
+        while (( $(date +%s) < timeoutEnd )); do
+            mchStatus="$(oc get multiclusterhub -A \
+                -o jsonpath='{.items[0].status.phase}' || true)"
+            if [[ "${mchStatus}" == "Running" ]]; then
                 break
             fi
             sleep 15
         done
-        if [[ "${mch_status}" != "Running" ]]; then
+        if [[ "${mchStatus}" != "Running" ]]; then
             echo >&2 "ERROR: MultiClusterHub did not reach Running phase"
             return 1
         fi
     fi
 
     echo "  Checking policy propagator..."
-    local propagator_ready
-    propagator_ready="$(oc get pods -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+    local propagatorReady
+    propagatorReady="$(oc get pods -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
         -l name=governance-policy-propagator \
         -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' \
-        2>/dev/null || true)"
-    echo "  Policy propagator ready: ${propagator_ready}"
+        || true)"
+    echo "  Policy propagator ready: ${propagatorReady}"
 
     echo "  Checking managed clusters..."
-    local cluster_count available_count
-    cluster_count="$(oc get managedclusters --no-headers 2>/dev/null | wc -l || echo 0)"
-    available_count="$(oc get managedclusters \
+    local clusterCount availableCount
+    clusterCount="$(oc get managedclusters --no-headers | wc -l || echo 0)"
+    availableCount="$(oc get managedclusters \
         -o jsonpath='{.items[?(@.status.conditions[?(@.type=="ManagedClusterConditionAvailable")].status=="True")].metadata.name}' \
-        2>/dev/null | wc -w || echo 0)"
-    echo "  Managed clusters: ${available_count}/${cluster_count} available"
+        | wc -w || echo 0)"
+    echo "  Managed clusters: ${availableCount}/${clusterCount} available"
 
     echo "ACM hub health validation complete"
     return 0
@@ -291,70 +291,70 @@ echo "Namespace: ${ACM_SUBSCRIPTION_NAMESPACE}"
 echo "Subscription: ${ACM_SUBSCRIPTION_NAME}"
 echo "Timeout: ${ACM_UPGRADE_TIMEOUT}"
 
-current_csv="$(get_current_csv)"
-if [[ -z "${current_csv}" ]]; then
+currentCsv="$(GetCurrentCsv)"
+if [[ -z "${currentCsv}" ]]; then
     echo >&2 "ERROR: No ACM subscription found or no currentCSV set"
     exit 3
 fi
 
-current_version="$(get_installed_version)"
-current_channel="$(get_current_channel)"
-echo "Current: CSV=${current_csv} Version=${current_version} Channel=${current_channel}"
+currentVersion="$(GetInstalledVersion)"
+currentChannel="$(GetCurrentChannel)"
+echo "Current: CSV=${currentCsv} Version=${currentVersion} Channel=${currentChannel}"
 
-target_channel="$(resolve_target_channel)"
-echo "Target channel: ${target_channel}"
+targetChannel="$(ResolveTargetChannel)"
+echo "Target channel: ${targetChannel}"
 
-if [[ "${target_channel}" == "${current_channel}" ]]; then
-    echo "Already on target channel ${target_channel}; checking if upgrade is available..."
-    install_plan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+if [[ "${targetChannel}" == "${currentChannel}" ]]; then
+    echo "Already on target channel ${targetChannel}; checking if upgrade is available..."
+    installPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.status.installPlanRef.name}' 2>/dev/null || true)"
-    if [[ -z "${install_plan}" ]]; then
+        -o jsonpath='{.status.installPlanRef.name}' || true)"
+    if [[ -z "${installPlan}" ]]; then
         echo "No pending upgrade on current channel; nothing to do"
         exit 0
     fi
-    plan_phase="$(oc get installplan "${install_plan}" \
+    planPhase="$(oc get installplan "${installPlan}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.status.phase}' 2>/dev/null || true)"
-    if [[ "${plan_phase}" == "Complete" ]]; then
-        echo "InstallPlan ${install_plan} already complete; no pending upgrade"
+        -o jsonpath='{.status.phase}' || true)"
+    if [[ "${planPhase}" == "Complete" ]]; then
+        echo "InstallPlan ${installPlan} already complete; no pending upgrade"
         exit 0
     fi
 fi
 
-echo "Patching subscription channel: ${current_channel} -> ${target_channel}"
+echo "Patching subscription channel: ${currentChannel} -> ${targetChannel}"
 oc patch subscription "${ACM_SUBSCRIPTION_NAME}" \
     -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
     --type merge \
-    -p "{\"spec\":{\"channel\":\"${target_channel}\"}}"
+    -p "{\"spec\":{\"channel\":\"${targetChannel}\"}}"
 
 echo "Waiting for InstallPlan to be created..."
 sleep 10
 
-install_plan=""
+installPlan=""
 for _ in $(seq 1 12); do
-    install_plan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+    installPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.status.installPlanRef.name}' 2>/dev/null || true)"
-    if [[ -z "${install_plan}" ]]; then
-        install_plan="$(oc get installplan -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        -o jsonpath='{.status.installPlanRef.name}' || true)"
+    if [[ -z "${installPlan}" ]]; then
+        installPlan="$(oc get installplan -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
             --sort-by=.metadata.creationTimestamp \
-            -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true)"
+            -o jsonpath='{.items[-1:].metadata.name}' || true)"
     fi
-    if [[ -n "${install_plan}" ]]; then
+    if [[ -n "${installPlan}" ]]; then
         break
     fi
     sleep 10
 done
 
-if [[ -n "${install_plan}" ]]; then
-    echo "InstallPlan: ${install_plan}"
-    local_approval="$(oc get installplan "${install_plan}" \
+if [[ -n "${installPlan}" ]]; then
+    echo "InstallPlan: ${installPlan}"
+    localApproval="$(oc get installplan "${installPlan}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.spec.approval}' 2>/dev/null || true)"
-    if [[ "${local_approval}" == "Manual" ]]; then
+        -o jsonpath='{.spec.approval}' || true)"
+    if [[ "${localApproval}" == "Manual" ]]; then
         echo "Approving manual InstallPlan..."
-        oc patch installplan "${install_plan}" \
+        oc patch installplan "${installPlan}" \
             -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
             --type merge \
             -p '{"spec":{"approved":true}}'
@@ -362,25 +362,26 @@ if [[ -n "${install_plan}" ]]; then
 fi
 
 echo "Waiting for ACM CSV to reach Succeeded phase..."
-wait_for_csv_succeeded "${current_csv}"
-new_csv="$(get_current_csv)"
-new_version="$(get_installed_version)"
-echo "Upgrade complete: ${current_version} -> ${new_version} (CSV: ${new_csv})"
+WaitForCsvSucceeded "${currentCsv}"
+newCsv="$(GetCurrentCsv)"
+newVersion="$(GetInstalledVersion)"
+echo "Upgrade complete: ${currentVersion} -> ${newVersion} (CSV: ${newCsv})"
 
-validate_mce_upgrade
-validate_hub_health
+ValidateMceUpgrade
+ValidateHubHealth
 
 {
     printf '=== ACM Operator Upgrade Summary ===\n'
-    printf 'Previous: %s (%s)\n' "${current_version}" "${current_channel}"
-    printf 'Current:  %s (%s)\n' "${new_version}" "${target_channel}"
-    printf 'CSV:      %s\n' "${new_csv}"
+    printf 'Previous: %s (%s)\n' "${currentVersion}" "${currentChannel}"
+    printf 'Current:  %s (%s)\n' "${newVersion}" "${targetChannel}"
+    printf 'CSV:      %s\n' "${newCsv}"
     printf 'Status:   SUCCESS\n'
 } > "${ARTIFACT_DIR}/acm-upgrade-summary.txt"
 
 if [[ -n "${SHARED_DIR:-}" ]]; then
-    echo "${new_version}" > "${SHARED_DIR}/acm-upgraded-version"
-    echo "${target_channel}" > "${SHARED_DIR}/acm-upgraded-channel"
+    echo "${newVersion}" > "${SHARED_DIR}/acm-upgraded-version"
+    echo "${targetChannel}" > "${SHARED_DIR}/acm-upgraded-channel"
 fi
 
 echo "=== ACM Operator Upgrade: SUCCESS ==="
+true

From 21bb75ef45fe73456523a349d6c622e3c3288807 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Thu, 6 Aug 2026 14:36:24 -0500
Subject: [PATCH 06/10] fix: Apply mpitt best practices (R2)

- typeset instead of local throughout
- Trap handler uses {( ... )} subshell form
- Terminal true in CollectDiagnostics, ResolveTargetChannel, ParseTimeout
- Separate oc get from wc to preserve pipefail semantics
- Brace expansion {1..12} instead of $(seq)
- Array-based channel iteration (SC2086)
---
 ...nterop-opp-product-upgrade-acm-commands.sh | 68 +++++++++++--------
 1 file changed, 39 insertions(+), 29 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
index a1fe763828489..f0fc6f935e612 100755
--- a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
@@ -14,7 +14,7 @@ mkdir -p "${ARTIFACT_DIR}"
 typeset -i exitCode=0
 
 function CollectDiagnostics () {
-    local artifactFile="${ARTIFACT_DIR}/acm-upgrade-diagnostics.txt"
+    typeset artifactFile="${ARTIFACT_DIR}/acm-upgrade-diagnostics.txt"
     {
         printf '=== ACM Operator Upgrade Diagnostics ===\n\n'
         printf '=== Subscription ===\n'
@@ -29,9 +29,10 @@ function CollectDiagnostics () {
         oc get pods -n "${ACM_SUBSCRIPTION_NAMESPACE}" --field-selector=status.phase!=Running,status.phase!=Succeeded 2>&1 || true
         oc get pods -n multicluster-engine --field-selector=status.phase!=Running,status.phase!=Succeeded 2>&1 || true
     } > "${artifactFile}"
+    true
 }
 
-trap 'exitCode=$?; if (( exitCode != 0 )); then CollectDiagnostics; fi' EXIT
+trap '{( exitCode=$?; if (( exitCode != 0 )); then CollectDiagnostics; fi )}' EXIT
 
 function GetCurrentCsv () {
     oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
@@ -40,14 +41,14 @@ function GetCurrentCsv () {
 }
 
 function GetCsvPhase () {
-    local csvName="$1"
+    typeset csvName="$1"
     oc get csv "${csvName}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
         -o jsonpath='{.status.phase}' || true
 }
 
 function GetInstalledVersion () {
-    local csvName
+    typeset csvName
     csvName="$(GetCurrentCsv)"
     if [[ -z "${csvName}" ]]; then
         return 1
@@ -69,24 +70,24 @@ function ResolveTargetChannel () {
         return 0
     fi
 
-    local currentChannel
+    typeset currentChannel
     currentChannel="$(GetCurrentChannel)"
     if [[ -z "${currentChannel}" ]]; then
         echo >&2 "ERROR: Cannot determine current subscription channel"
         return 3
     fi
 
-    local catalogNamespace
+    typeset catalogNamespace
     catalogNamespace="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
         -o jsonpath='{.spec.sourceNamespace}' || true)"
 
-    local packageName
+    typeset packageName
     packageName="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
         -o jsonpath='{.spec.name}' || true)"
 
-    local channels
+    typeset channels
     channels="$(oc get packagemanifest "${packageName}" \
         -n "${catalogNamespace}" \
         -o jsonpath='{.status.channels[*].name}' || true)"
@@ -96,11 +97,13 @@ function ResolveTargetChannel () {
         return 3
     fi
 
-    local currentVersion nextChannel=""
+    typeset currentVersion nextChannel=""
     currentVersion="$(echo "${currentChannel}" | grep -oE '[0-9]+\.[0-9]+' || true)"
 
-    for ch in ${channels}; do
-        local chVersion
+    typeset -a channelList
+    read -ra channelList <<< "${channels}"
+    for ch in "${channelList[@]}"; do
+        typeset chVersion
         chVersion="$(echo "${ch}" | grep -oE '[0-9]+\.[0-9]+' || true)"
         if [[ -z "${chVersion}" ]]; then
             continue
@@ -109,7 +112,7 @@ function ResolveTargetChannel () {
             nextChannel="${ch}"
             break
         fi
-        local currentMajor currentMinor chMajor chMinor
+        typeset currentMajor currentMinor chMajor chMinor
         currentMajor="${currentVersion%%.*}"
         currentMinor="${currentVersion##*.}"
         chMajor="${chVersion%%.*}"
@@ -120,7 +123,7 @@ function ResolveTargetChannel () {
             if [[ -z "${nextChannel}" ]]; then
                 nextChannel="${ch}"
             else
-                local nextVersion nextMajor nextMinor
+                typeset nextVersion nextMajor nextMinor
                 nextVersion="$(echo "${nextChannel}" | grep -oE '[0-9]+\.[0-9]+' || true)"
                 nextMajor="${nextVersion%%.*}"
                 nextMinor="${nextVersion##*.}"
@@ -138,13 +141,14 @@ function ResolveTargetChannel () {
     fi
 
     echo "${nextChannel}"
+    true
 }
 
 function WaitForCsvSucceeded () {
-    local previousCsv="$1"
-    local timeoutSeconds
+    typeset previousCsv="$1"
+    typeset timeoutSeconds
     timeoutSeconds="$(ParseTimeout "${ACM_UPGRADE_TIMEOUT}")"
-    local startTime elapsed newCsv phase
+    typeset startTime elapsed newCsv phase
     startTime="$(date +%s)"
 
     while true; do
@@ -179,8 +183,8 @@ function WaitForCsvSucceeded () {
 }
 
 function ParseTimeout () {
-    local input="$1"
-    local minutes=0 seconds=0
+    typeset input="$1"
+    typeset minutes=0 seconds=0
     if [[ "${input}" =~ ^([0-9]+)m$ ]]; then
         minutes="${BASH_REMATCH[1]}"
     elif [[ "${input}" =~ ^([0-9]+)s$ ]]; then
@@ -194,11 +198,12 @@ function ParseTimeout () {
         minutes=30
     fi
     echo "$(( minutes * 60 + seconds ))"
+    true
 }
 
 function ValidateMceUpgrade () {
     echo "Validating MCE (MultiCluster Engine) upgrade..."
-    local mceCsv
+    typeset mceCsv
     mceCsv="$(oc get csv -n multicluster-engine \
         -o jsonpath='{.items[?(@.spec.displayName=="multicluster engine for Kubernetes")].metadata.name}' \
         || true)"
@@ -214,14 +219,14 @@ function ValidateMceUpgrade () {
         return 0
     fi
 
-    local mcePhase
+    typeset mcePhase
     mcePhase="$(oc get csv "${mceCsv}" -n multicluster-engine \
         -o jsonpath='{.status.phase}' || true)"
 
     echo "  MCE CSV: ${mceCsv}  Phase: ${mcePhase}"
     if [[ "${mcePhase}" != "Succeeded" ]]; then
         echo "WARNING: MCE CSV phase is ${mcePhase}, not Succeeded"
-        local timeoutEnd
+        typeset timeoutEnd
         timeoutEnd="$(( $(date +%s) + 300 ))"
         while (( $(date +%s) < timeoutEnd )); do
             mcePhase="$(oc get csv "${mceCsv}" -n multicluster-engine \
@@ -241,14 +246,14 @@ function ValidateMceUpgrade () {
 function ValidateHubHealth () {
     echo "Validating ACM hub health post-upgrade..."
 
-    local mchStatus
+    typeset mchStatus
     mchStatus="$(oc get multiclusterhub -A \
         -o jsonpath='{.items[0].status.phase}' || true)"
     echo "  MultiClusterHub phase: ${mchStatus}"
 
     if [[ "${mchStatus}" != "Running" ]]; then
         echo "  Waiting for MCH to reach Running phase (timeout: 5m)..."
-        local timeoutEnd
+        typeset timeoutEnd
         timeoutEnd="$(( $(date +%s) + 300 ))"
         while (( $(date +%s) < timeoutEnd )); do
             mchStatus="$(oc get multiclusterhub -A \
@@ -265,7 +270,7 @@ function ValidateHubHealth () {
     fi
 
     echo "  Checking policy propagator..."
-    local propagatorReady
+    typeset propagatorReady
     propagatorReady="$(oc get pods -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
         -l name=governance-policy-propagator \
         -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' \
@@ -273,11 +278,16 @@ function ValidateHubHealth () {
     echo "  Policy propagator ready: ${propagatorReady}"
 
     echo "  Checking managed clusters..."
-    local clusterCount availableCount
-    clusterCount="$(oc get managedclusters --no-headers | wc -l || echo 0)"
-    availableCount="$(oc get managedclusters \
+    typeset clusterOutput=""
+    clusterOutput="$(oc get managedclusters --no-headers || true)"
+    typeset -i clusterCount=0
+    clusterCount="$(echo "${clusterOutput}" | grep -c . || true)"
+    typeset availableOutput=""
+    availableOutput="$(oc get managedclusters \
         -o jsonpath='{.items[?(@.status.conditions[?(@.type=="ManagedClusterConditionAvailable")].status=="True")].metadata.name}' \
-        | wc -w || echo 0)"
+        || true)"
+    typeset -i availableCount=0
+    availableCount="$(echo "${availableOutput}" | wc -w)"
     echo "  Managed clusters: ${availableCount}/${clusterCount} available"
 
     echo "ACM hub health validation complete"
@@ -332,7 +342,7 @@ echo "Waiting for InstallPlan to be created..."
 sleep 10
 
 installPlan=""
-for _ in $(seq 1 12); do
+for _ in {1..12}; do
     installPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
         -o jsonpath='{.status.installPlanRef.name}' || true)"

From a660eb2c630894cd3ad30ea524f33797ab1e3e8c Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Thu, 6 Aug 2026 14:20:56 -0500
Subject: [PATCH 07/10] Wire ACM product upgrade step into upgrade configs

Add interop-opp-product-upgrade-acm ref to both OPP upgrade configs
(4.22 and 5.0) so the ACM operator upgrade runs after OCP platform
upgrade and health check, before smoke tests.

Override ACM_SUBSCRIPTION_NAMESPACE to 'ocm' to match the namespace
used by the install-operators step.

Addresses INTEROP-9381 acceptance criteria:
- Post-upgrade health validation passes in CI
- Health check gates downstream product upgrades
---
 .../stolostron-policy-collection-main__ocp4.22-upgrade.yaml     | 2 ++
 .../stolostron-policy-collection-main__ocp5.0-upgrade.yaml      | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml
index a8e4a9187d0b1..52531c2abf28f 100644
--- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml
+++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml
@@ -44,6 +44,7 @@ tests:
     dependencies:
       OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:initial
     env:
+      ACM_SUBSCRIPTION_NAMESPACE: ocm
       BASE_DOMAIN: cspilp.interop.ccitredhat.com
       COMPUTE_NODE_REPLICAS: "6"
       COMPUTE_NODE_TYPE: m6a.2xlarge
@@ -79,6 +80,7 @@ tests:
     - ref: interop-opp-preflight
     - ref: interop-opp-upgrade
     - ref: cucushift-upgrade-healthcheck
+    - ref: interop-opp-product-upgrade-acm
     - ref: interop-opp-smoke
 zz_generated_metadata:
   branch: main
diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml
index 6573ee6e9e007..841a1a363a3f0 100644
--- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml
+++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml
@@ -44,6 +44,7 @@ tests:
     dependencies:
       OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:initial
     env:
+      ACM_SUBSCRIPTION_NAMESPACE: ocm
       BASE_DOMAIN: cspilp.interop.ccitredhat.com
       COMPUTE_NODE_REPLICAS: "6"
       COMPUTE_NODE_TYPE: m6a.2xlarge
@@ -77,6 +78,7 @@ tests:
     - ref: interop-opp-preflight
     - ref: interop-opp-upgrade
     - ref: cucushift-upgrade-healthcheck
+    - ref: interop-opp-product-upgrade-acm
     - ref: interop-opp-smoke
 zz_generated_metadata:
   branch: main

From 1ecf1027225a9e20011eafb10c1fe3553e062581 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Fri, 14 Aug 2026 07:31:39 -0500
Subject: [PATCH 08/10] Address review feedback: InstallPlan race and
 propagator validation

- Capture pre-patch InstallPlan ref before channel change; wait for a
  different ref to appear (removes stale-ref race condition)
- Remove namespace-wide InstallPlan fallback that could select another
  operator's plan
- Fail ValidateHubHealth when policy propagator is not ready after 5m
  timeout instead of only logging the value
---
 ...nterop-opp-product-upgrade-acm-commands.sh | 100 ++++++++++--------
 1 file changed, 57 insertions(+), 43 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
index f0fc6f935e612..97dbc9e4fbce5 100755
--- a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
@@ -270,12 +270,25 @@ function ValidateHubHealth () {
     fi
 
     echo "  Checking policy propagator..."
-    typeset propagatorReady
-    propagatorReady="$(oc get pods -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -l name=governance-policy-propagator \
-        -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' \
-        || true)"
+    typeset propagatorReady=""
+    typeset -i propTimeout=300
+    typeset -i propStart
+    propStart="$(date +%s)"
+    while (( $(date +%s) - propStart < propTimeout )); do
+        propagatorReady="$(oc get pods -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+            -l name=governance-policy-propagator \
+            -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' \
+            || true)"
+        if [[ "${propagatorReady}" == "True" ]]; then
+            break
+        fi
+        sleep 15
+    done
     echo "  Policy propagator ready: ${propagatorReady}"
+    if [[ "${propagatorReady}" != "True" ]]; then
+        echo >&2 "ERROR: Policy propagator not ready after ${propTimeout}s"
+        return 1
+    fi
 
     echo "  Checking managed clusters..."
     typeset clusterOutput=""
@@ -314,61 +327,62 @@ echo "Current: CSV=${currentCsv} Version=${currentVersion} Channel=${currentChan
 targetChannel="$(ResolveTargetChannel)"
 echo "Target channel: ${targetChannel}"
 
+prePatchPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+    -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+    -o jsonpath='{.status.installPlanRef.name}' || true)"
+
 if [[ "${targetChannel}" == "${currentChannel}" ]]; then
     echo "Already on target channel ${targetChannel}; checking if upgrade is available..."
-    installPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
-        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.status.installPlanRef.name}' || true)"
-    if [[ -z "${installPlan}" ]]; then
+    if [[ -z "${prePatchPlan}" ]]; then
         echo "No pending upgrade on current channel; nothing to do"
         exit 0
     fi
-    planPhase="$(oc get installplan "${installPlan}" \
+    planPhase="$(oc get installplan "${prePatchPlan}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
         -o jsonpath='{.status.phase}' || true)"
     if [[ "${planPhase}" == "Complete" ]]; then
-        echo "InstallPlan ${installPlan} already complete; no pending upgrade"
+        echo "InstallPlan ${prePatchPlan} already complete; no pending upgrade"
         exit 0
     fi
-fi
+    installPlan="${prePatchPlan}"
+else
+    echo "Patching subscription channel: ${currentChannel} -> ${targetChannel}"
+    oc patch subscription "${ACM_SUBSCRIPTION_NAME}" \
+        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+        --type merge \
+        -p "{\"spec\":{\"channel\":\"${targetChannel}\"}}"
 
-echo "Patching subscription channel: ${currentChannel} -> ${targetChannel}"
-oc patch subscription "${ACM_SUBSCRIPTION_NAME}" \
-    -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-    --type merge \
-    -p "{\"spec\":{\"channel\":\"${targetChannel}\"}}"
+    echo "Waiting for new InstallPlan (pre-patch ref: ${prePatchPlan:-none})..."
+    sleep 10
 
-echo "Waiting for InstallPlan to be created..."
-sleep 10
+    installPlan=""
+    for _ in {1..18}; do
+        installPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+            -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+            -o jsonpath='{.status.installPlanRef.name}' || true)"
+        if [[ -n "${installPlan}" && "${installPlan}" != "${prePatchPlan}" ]]; then
+            break
+        fi
+        installPlan=""
+        sleep 10
+    done
 
-installPlan=""
-for _ in {1..12}; do
-    installPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
-        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.status.installPlanRef.name}' || true)"
     if [[ -z "${installPlan}" ]]; then
-        installPlan="$(oc get installplan -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-            --sort-by=.metadata.creationTimestamp \
-            -o jsonpath='{.items[-1:].metadata.name}' || true)"
+        echo >&2 "ERROR: No new InstallPlan appeared after channel change (waited 3m)"
+        exit 2
     fi
-    if [[ -n "${installPlan}" ]]; then
-        break
-    fi
-    sleep 10
-done
+fi
 
-if [[ -n "${installPlan}" ]]; then
-    echo "InstallPlan: ${installPlan}"
-    localApproval="$(oc get installplan "${installPlan}" \
+echo "InstallPlan: ${installPlan}"
+localApproval="$(oc get installplan "${installPlan}" \
+    -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+    -o jsonpath='{.spec.approval}' || true)"
+if [[ "${localApproval}" == "Manual" ]]; then
+    echo "Approving manual InstallPlan..."
+    oc patch installplan "${installPlan}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.spec.approval}' || true)"
-    if [[ "${localApproval}" == "Manual" ]]; then
-        echo "Approving manual InstallPlan..."
-        oc patch installplan "${installPlan}" \
-            -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-            --type merge \
-            -p '{"spec":{"approved":true}}'
-    fi
+        --type merge \
+        -p '{"spec":{"approved":true}}'
 fi
 
 echo "Waiting for ACM CSV to reach Succeeded phase..."

From 41819c689e63139b5303f94afb2ac94dc02c7e0f Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Mon, 17 Aug 2026 08:10:08 -0500
Subject: [PATCH 09/10] Address review findings: ACM upgrade trap, hub health,
 InstallPlan logging

- Simplify EXIT trap: remove unnecessary {(...)} subshell nesting that
  prevented exitCode from propagating; use direct $? check instead
- Add managed cluster availability warnings in ValidateHubHealth when
  clusters are unavailable post-upgrade (warning only, not a failure,
  since transient unavailability is expected during operator upgrades)
- Make pre-patch InstallPlan capture explicit: log query failures
  separately from genuinely empty installPlanRef, aiding debugging
  when the poll loop fails to detect a new plan
---
 ...nterop-opp-product-upgrade-acm-commands.sh | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
index 97dbc9e4fbce5..e7fcddc8e2f1f 100755
--- a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
@@ -10,9 +10,6 @@ ACM_SUBSCRIPTION_NAMESPACE="${ACM_SUBSCRIPTION_NAMESPACE:-open-cluster-managemen
 ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/artifacts}"
 mkdir -p "${ARTIFACT_DIR}"
 
-# shellcheck disable=SC2034
-typeset -i exitCode=0
-
 function CollectDiagnostics () {
     typeset artifactFile="${ARTIFACT_DIR}/acm-upgrade-diagnostics.txt"
     {
@@ -32,7 +29,7 @@ function CollectDiagnostics () {
     true
 }
 
-trap '{( exitCode=$?; if (( exitCode != 0 )); then CollectDiagnostics; fi )}' EXIT
+trap 'if (( $? != 0 )); then CollectDiagnostics; fi' EXIT
 
 function GetCurrentCsv () {
     oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
@@ -303,6 +300,12 @@ function ValidateHubHealth () {
     availableCount="$(echo "${availableOutput}" | wc -w)"
     echo "  Managed clusters: ${availableCount}/${clusterCount} available"
 
+    if (( clusterCount > 0 && availableCount == 0 )); then
+        echo >&2 "WARNING: All ${clusterCount} managed cluster(s) are unavailable after upgrade"
+    elif (( clusterCount > 0 && availableCount < clusterCount )); then
+        echo >&2 "WARNING: ${availableCount}/${clusterCount} managed cluster(s) available (some may be reconciling post-upgrade)"
+    fi
+
     echo "ACM hub health validation complete"
     return 0
 }
@@ -327,9 +330,14 @@ echo "Current: CSV=${currentCsv} Version=${currentVersion} Channel=${currentChan
 targetChannel="$(ResolveTargetChannel)"
 echo "Target channel: ${targetChannel}"
 
-prePatchPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+prePatchPlan=""
+if ! prePatchPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
     -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-    -o jsonpath='{.status.installPlanRef.name}' || true)"
+    -o jsonpath='{.status.installPlanRef.name}' 2>/dev/null)"; then
+    echo "WARNING: Could not query current installPlanRef; treating as empty"
+    prePatchPlan=""
+fi
+echo "Pre-patch InstallPlan: ${prePatchPlan:-none}"
 
 if [[ "${targetChannel}" == "${currentChannel}" ]]; then
     echo "Already on target channel ${targetChannel}; checking if upgrade is available..."

From 9872e638fb43006c6dc0c11f96f781ffd389cb1d Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Mon, 17 Aug 2026 08:39:09 -0500
Subject: [PATCH 10/10] fix: Apply mpitt best practices

---
 ...nterop-opp-product-upgrade-acm-commands.sh | 198 +++++++++---------
 1 file changed, 103 insertions(+), 95 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
index e7fcddc8e2f1f..f1f28f1c71cef 100755
--- a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
+++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh
@@ -312,108 +312,116 @@ function ValidateHubHealth () {
 
 # === Main ===
 
-echo "=== ACM Operator Upgrade Step ==="
-echo "Namespace: ${ACM_SUBSCRIPTION_NAMESPACE}"
-echo "Subscription: ${ACM_SUBSCRIPTION_NAME}"
-echo "Timeout: ${ACM_UPGRADE_TIMEOUT}"
-
-currentCsv="$(GetCurrentCsv)"
-if [[ -z "${currentCsv}" ]]; then
-    echo >&2 "ERROR: No ACM subscription found or no currentCSV set"
-    exit 3
-fi
-
-currentVersion="$(GetInstalledVersion)"
-currentChannel="$(GetCurrentChannel)"
-echo "Current: CSV=${currentCsv} Version=${currentVersion} Channel=${currentChannel}"
-
-targetChannel="$(ResolveTargetChannel)"
-echo "Target channel: ${targetChannel}"
-
-prePatchPlan=""
-if ! prePatchPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
-    -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-    -o jsonpath='{.status.installPlanRef.name}' 2>/dev/null)"; then
-    echo "WARNING: Could not query current installPlanRef; treating as empty"
-    prePatchPlan=""
-fi
-echo "Pre-patch InstallPlan: ${prePatchPlan:-none}"
-
-if [[ "${targetChannel}" == "${currentChannel}" ]]; then
-    echo "Already on target channel ${targetChannel}; checking if upgrade is available..."
-    if [[ -z "${prePatchPlan}" ]]; then
-        echo "No pending upgrade on current channel; nothing to do"
-        exit 0
+function Main () {
+    typeset currentCsv currentVersion currentChannel targetChannel
+    typeset prePatchPlan planPhase installPlan localApproval
+    typeset newCsv newVersion
+
+    echo "=== ACM Operator Upgrade Step ==="
+    echo "Namespace: ${ACM_SUBSCRIPTION_NAMESPACE}"
+    echo "Subscription: ${ACM_SUBSCRIPTION_NAME}"
+    echo "Timeout: ${ACM_UPGRADE_TIMEOUT}"
+
+    currentCsv="$(GetCurrentCsv)"
+    if [[ -z "${currentCsv}" ]]; then
+        echo >&2 "ERROR: No ACM subscription found or no currentCSV set"
+        exit 3
     fi
-    planPhase="$(oc get installplan "${prePatchPlan}" \
+
+    currentVersion="$(GetInstalledVersion)"
+    currentChannel="$(GetCurrentChannel)"
+    echo "Current: CSV=${currentCsv} Version=${currentVersion} Channel=${currentChannel}"
+
+    targetChannel="$(ResolveTargetChannel)"
+    echo "Target channel: ${targetChannel}"
+
+    prePatchPlan=""
+    if ! prePatchPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        -o jsonpath='{.status.phase}' || true)"
-    if [[ "${planPhase}" == "Complete" ]]; then
-        echo "InstallPlan ${prePatchPlan} already complete; no pending upgrade"
-        exit 0
+        -o jsonpath='{.status.installPlanRef.name}' 2>/dev/null)"; then
+        echo "WARNING: Could not query current installPlanRef; treating as empty"
+        prePatchPlan=""
     fi
-    installPlan="${prePatchPlan}"
-else
-    echo "Patching subscription channel: ${currentChannel} -> ${targetChannel}"
-    oc patch subscription "${ACM_SUBSCRIPTION_NAME}" \
-        -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        --type merge \
-        -p "{\"spec\":{\"channel\":\"${targetChannel}\"}}"
-
-    echo "Waiting for new InstallPlan (pre-patch ref: ${prePatchPlan:-none})..."
-    sleep 10
+    echo "Pre-patch InstallPlan: ${prePatchPlan:-none}"
 
-    installPlan=""
-    for _ in {1..18}; do
-        installPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+    if [[ "${targetChannel}" == "${currentChannel}" ]]; then
+        echo "Already on target channel ${targetChannel}; checking if upgrade is available..."
+        if [[ -z "${prePatchPlan}" ]]; then
+            echo "No pending upgrade on current channel; nothing to do"
+            exit 0
+        fi
+        planPhase="$(oc get installplan "${prePatchPlan}" \
             -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-            -o jsonpath='{.status.installPlanRef.name}' || true)"
-        if [[ -n "${installPlan}" && "${installPlan}" != "${prePatchPlan}" ]]; then
-            break
+            -o jsonpath='{.status.phase}' || true)"
+        if [[ "${planPhase}" == "Complete" ]]; then
+            echo "InstallPlan ${prePatchPlan} already complete; no pending upgrade"
+            exit 0
         fi
-        installPlan=""
+        installPlan="${prePatchPlan}"
+    else
+        echo "Patching subscription channel: ${currentChannel} -> ${targetChannel}"
+        oc patch subscription "${ACM_SUBSCRIPTION_NAME}" \
+            -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+            --type merge \
+            -p "{\"spec\":{\"channel\":\"${targetChannel}\"}}"
+
+        echo "Waiting for new InstallPlan (pre-patch ref: ${prePatchPlan:-none})..."
         sleep 10
-    done
 
-    if [[ -z "${installPlan}" ]]; then
-        echo >&2 "ERROR: No new InstallPlan appeared after channel change (waited 3m)"
-        exit 2
+        installPlan=""
+        for _ in {1..18}; do
+            installPlan="$(oc get subscription "${ACM_SUBSCRIPTION_NAME}" \
+                -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+                -o jsonpath='{.status.installPlanRef.name}' || true)"
+            if [[ -n "${installPlan}" && "${installPlan}" != "${prePatchPlan}" ]]; then
+                break
+            fi
+            installPlan=""
+            sleep 10
+        done
+
+        if [[ -z "${installPlan}" ]]; then
+            echo >&2 "ERROR: No new InstallPlan appeared after channel change (waited 3m)"
+            exit 2
+        fi
     fi
-fi
-
-echo "InstallPlan: ${installPlan}"
-localApproval="$(oc get installplan "${installPlan}" \
-    -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-    -o jsonpath='{.spec.approval}' || true)"
-if [[ "${localApproval}" == "Manual" ]]; then
-    echo "Approving manual InstallPlan..."
-    oc patch installplan "${installPlan}" \
+
+    echo "InstallPlan: ${installPlan}"
+    localApproval="$(oc get installplan "${installPlan}" \
         -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
-        --type merge \
-        -p '{"spec":{"approved":true}}'
-fi
-
-echo "Waiting for ACM CSV to reach Succeeded phase..."
-WaitForCsvSucceeded "${currentCsv}"
-newCsv="$(GetCurrentCsv)"
-newVersion="$(GetInstalledVersion)"
-echo "Upgrade complete: ${currentVersion} -> ${newVersion} (CSV: ${newCsv})"
-
-ValidateMceUpgrade
-ValidateHubHealth
-
-{
-    printf '=== ACM Operator Upgrade Summary ===\n'
-    printf 'Previous: %s (%s)\n' "${currentVersion}" "${currentChannel}"
-    printf 'Current:  %s (%s)\n' "${newVersion}" "${targetChannel}"
-    printf 'CSV:      %s\n' "${newCsv}"
-    printf 'Status:   SUCCESS\n'
-} > "${ARTIFACT_DIR}/acm-upgrade-summary.txt"
-
-if [[ -n "${SHARED_DIR:-}" ]]; then
-    echo "${newVersion}" > "${SHARED_DIR}/acm-upgraded-version"
-    echo "${targetChannel}" > "${SHARED_DIR}/acm-upgraded-channel"
-fi
-
-echo "=== ACM Operator Upgrade: SUCCESS ==="
-true
+        -o jsonpath='{.spec.approval}' || true)"
+    if [[ "${localApproval}" == "Manual" ]]; then
+        echo "Approving manual InstallPlan..."
+        oc patch installplan "${installPlan}" \
+            -n "${ACM_SUBSCRIPTION_NAMESPACE}" \
+            --type merge \
+            -p '{"spec":{"approved":true}}'
+    fi
+
+    echo "Waiting for ACM CSV to reach Succeeded phase..."
+    WaitForCsvSucceeded "${currentCsv}"
+    newCsv="$(GetCurrentCsv)"
+    newVersion="$(GetInstalledVersion)"
+    echo "Upgrade complete: ${currentVersion} -> ${newVersion} (CSV: ${newCsv})"
+
+    ValidateMceUpgrade
+    ValidateHubHealth
+
+    {
+        printf '=== ACM Operator Upgrade Summary ===\n'
+        printf 'Previous: %s (%s)\n' "${currentVersion}" "${currentChannel}"
+        printf 'Current:  %s (%s)\n' "${newVersion}" "${targetChannel}"
+        printf 'CSV:      %s\n' "${newCsv}"
+        printf 'Status:   SUCCESS\n'
+    } > "${ARTIFACT_DIR}/acm-upgrade-summary.txt"
+
+    if [[ -n "${SHARED_DIR:-}" ]]; then
+        echo "${newVersion}" > "${SHARED_DIR}/acm-upgraded-version"
+        echo "${targetChannel}" > "${SHARED_DIR}/acm-upgraded-channel"
+    fi
+
+    echo "=== ACM Operator Upgrade: SUCCESS ==="
+    true
+}
+
+Main "$@"