From a620c7025336e5234c4685384f7e739926df2ef3 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 23 Jul 2026 14:20:19 -0500 Subject: [PATCH 1/3] INTEROP-9337: Fix OCP 5.0 upgrade config and harden upgrade script Remove rhacs-operator and odf-operator from the 5.0 upgrade config (absent from v5.0 catalog). Fix quay-operator channel from stable-3.18 to stable-3.17. Add interop-opp-backup and interop-opp-preflight steps. Pass explicit OPP_OPERATORS so backup, preflight, and upgrade steps only validate installed operators. Harden the upgrade script: source proxy-conf.sh, replace jq with oc templates and awk, use function PascalCase declarations with typeset, wrap oc wait in if-! guard with diagnostics, use trap subshell form, replace pipeline || true with variable capture, use here-strings instead of echo pipes, and add terminal true to all functions. --- ...olicy-collection-main__ocp5.0-upgrade.yaml | 7 +- .../upgrade/interop-opp-upgrade-commands.sh | 108 +++++++++--------- 2 files changed, 58 insertions(+), 57 deletions(-) diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml index cbacbcddcc40f..dc0228130a1d4 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml @@ -57,10 +57,9 @@ tests: OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}, - {"name": "rhacs-operator", "source": "redhat-operators", "channel": "stable", "install_namespace": "rhacs-operator", "target_namespaces": "rhacs-operator"}, - {"name": "odf-operator", "source": "redhat-operators", "channel": "stable-5.0", "install_namespace": "openshift-storage", "target_namespaces": "openshift-storage"}, - {"name": "quay-operator", "source": "redhat-operators", "channel": "stable-3.18", "install_namespace": "openshift-operators"} + {"name": "quay-operator", "source": "redhat-operators", "channel": "stable-3.17", "install_namespace": "openshift-operators"} ] + OPP_OPERATORS: advanced-cluster-management,quay-operator ZONES_COUNT: "3" post: - ref: gather-aws-console @@ -75,6 +74,8 @@ tests: - chain: ipi-install test: - ref: install-operators + - ref: interop-opp-backup + - ref: interop-opp-preflight - ref: interop-opp-upgrade - ref: cucushift-upgrade-healthcheck - ref: interop-opp-smoke diff --git a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh index 3cddc55452f5b..354060cdaed9e 100644 --- a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh +++ b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh @@ -14,6 +14,12 @@ export XDG_RUNTIME_DIR="${HOME}/run" export REGISTRY_AUTH_PREFERENCE=podman mkdir -p "${XDG_RUNTIME_DIR}" +if [[ -f "${SHARED_DIR}/proxy-conf.sh" ]]; then + set +x + source "${SHARED_DIR}/proxy-conf.sh" + set -x +fi + typeset -i exitCode=0 typeset upgradeTarget="" typeset targetVersion="" @@ -22,7 +28,7 @@ typeset sourceVersion="" typeset -i sourceMinorVersion=0 typeset isForceUpdate="false" -DebugOnExit() { +function DebugOnExit () { if (( exitCode != 0 )); then : "### DEBUG: Upgrade failure diagnostics ###" if [[ -n "${targetMinorVersion:-}" ]] && (( targetMinorVersion >= 16 )); then @@ -35,43 +41,39 @@ DebugOnExit() { oc get machineconfig || : "unavailable" : "# Abnormal nodes" - oc get node -o json | jq -r '.items[] | select(.status.conditions[] | select(.type=="Ready" and .status!="True")) | .metadata.name' | while read -r node; do + typeset node="" + for node in $(oc get node -o go-template='{{range .items}}{{$ready := ""}}{{range .status.conditions}}{{if eq .type "Ready"}}{{$ready = .status}}{{end}}{{end}}{{if ne $ready "True"}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' || true); do : "### oc describe node ${node} ###" oc describe node "${node}" || true - done || true + done : "# Abnormal ClusterOperators" - oc get co -o json | jq -r '.items[] | select( - (.status.conditions[] | select(.type=="Available")).status != "True" or - (.status.conditions[] | select(.type=="Progressing")).status != "False" or - (.status.conditions[] | select(.type=="Degraded")).status != "False" - ) | .metadata.name' | while read -r co; do + typeset co="" + for co in $(oc get co -o go-template='{{range .items}}{{$avail := ""}}{{$prog := ""}}{{$deg := ""}}{{range .status.conditions}}{{if eq .type "Available"}}{{$avail = .status}}{{end}}{{if eq .type "Progressing"}}{{$prog = .status}}{{end}}{{if eq .type "Degraded"}}{{$deg = .status}}{{end}}{{end}}{{if or (ne $avail "True") (ne $prog "False") (ne $deg "False")}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' || true); do : "### oc describe co ${co} ###" oc describe co "${co}" || true - done || true + done : "# Abnormal MachineConfigPools" - oc get machineconfigpools -o json | jq -r '.items[] | select( - (.status.conditions[] | select(.type=="Updated")).status != "True" or - (.status.conditions[] | select(.type=="Updating")).status != "False" or - (.status.conditions[] | select(.type=="Degraded")).status != "False" - ) | .metadata.name' | while read -r mcp; do + typeset mcp="" + for mcp in $(oc get machineconfigpools -o go-template='{{range .items}}{{$upd := ""}}{{$upting := ""}}{{$deg := ""}}{{range .status.conditions}}{{if eq .type "Updated"}}{{$upd = .status}}{{end}}{{if eq .type "Updating"}}{{$upting = .status}}{{end}}{{if eq .type "Degraded"}}{{$deg = .status}}{{end}}{{end}}{{if or (ne $upd "True") (ne $upting "False") (ne $deg "False")}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' || true); do : "### oc describe mcp ${mcp} ###" oc describe mcp "${mcp}" || true - done || true + done : "# OPP Operator CSVs" oc get csv -A || : "unavailable" fi + true } -trap 'exitCode=$?; DebugOnExit' EXIT TERM +trap '{( exitCode=$?; DebugOnExit )}' EXIT TERM set +x KUBECONFIG="" oc registry login set -x -ResolveTargetImage() { +function ResolveTargetImage () { typeset image="${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE:-}" if [[ -z "${image}" ]]; then : "OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE is not set; cannot resolve upgrade target" @@ -79,16 +81,17 @@ ResolveTargetImage() { fi : "Target image: ${image}" upgradeTarget="${image}" + true } -CheckSigned() { +function CheckSigned () { typeset payload="${1:-}"; (($#)) && shift typeset digest="" algorithm="" hashValue="" typeset -i response=0 try=0 maxRetries=3 if [[ "${payload}" =~ "@sha256:" ]]; then digest="$(echo "${payload}" | cut -f2 -d@)" else - digest="$(oc image info "${payload}" -o json | jq -r '.digest')" + digest="$(oc image info "${payload}" -o json | awk -F'"' '/"digest"/{print $4; exit}')" fi : "Image digest: ${digest}" algorithm="$(echo "${digest}" | cut -f1 -d:)" @@ -113,7 +116,7 @@ CheckSigned() { fi } -AdminAck() { +function AdminAck () { typeset -i srcMinor="${1:-0}"; (($#)) && shift typeset -i tgtMinor="${1:-0}"; (($#)) && shift if (( srcMinor == tgtMinor )) || (( srcMinor < 8 )); then @@ -122,8 +125,8 @@ AdminAck() { fi typeset gates="" - gates="$(oc -n openshift-config-managed get configmap admin-gates -o json | jq -r '.data')" || true - if [[ -z "${gates}" || "${gates}" == "null" ]]; then + gates="$(oc -n openshift-config-managed get configmap admin-gates -o go-template='{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}')" || true + if [[ -z "${gates}" ]]; then : "No admin gates found" return 0 fi @@ -135,10 +138,8 @@ AdminAck() { fi : "Patching admin acks for 4.${srcMinor} -> 4.${tgtMinor}" - typeset ackKeys="" - ackKeys="$(echo "${gates}" | jq -r 'keys[]')" typeset ack="" - for ack in ${ackKeys}; do + for ack in ${gates}; do if [[ "${ack}" == *"ack-4.${srcMinor}"* ]]; then : "Applying ack: ${ack}" oc -n openshift-config patch configmap admin-acks \ @@ -163,7 +164,7 @@ AdminAck() { return 1 } -UpdateCcoAnnotation() { +function UpdateCcoAnnotation () { typeset srcVersion="${1:-}"; (($#)) && shift typeset tgtVersion="${1:-}"; (($#)) && shift typeset -i srcMinor=0 tgtMinor=0 @@ -206,7 +207,7 @@ UpdateCcoAnnotation() { return 1 } -InitiateUpgrade() { +function InitiateUpgrade () { typeset isForce="${1:-}"; (($#)) && shift : "Initiating upgrade to ${upgradeTarget}" : "Force flag: ${isForce}" @@ -221,9 +222,10 @@ InitiateUpgrade() { else : "CVO confirmed Progressing=True" fi + true } -MonitorUpgrade() { +function MonitorUpgrade () { typeset -i pollCount=0 typeset -i lastProgressChange=0 lastProgressChange=$(date +%s) @@ -289,13 +291,20 @@ MonitorUpgrade() { exit 2 } -StabilizeCluster() { +function StabilizeCluster () { : "Waiting for cluster stability (minimum-stable-period=5m, timeout=30m)" - oc adm wait-for-stable-cluster --minimum-stable-period=5m --timeout=30m + if ! oc adm wait-for-stable-cluster --minimum-stable-period=5m --timeout=30m; then + : "Cluster stabilization failed; gathering diagnostics" + oc get co || true + oc get nodes || true + oc get machineconfigpools || true + exit 1 + fi : "Cluster is stable" + true } -ValidatePlatformHealth() { +function ValidatePlatformHealth () { : "Validating platform health" typeset avail="" progressing="" degraded="" @@ -309,11 +318,7 @@ ValidatePlatformHealth() { : "CVO: Available=True, Progressing=False, Degraded=False" typeset unhealthyCo="" - unhealthyCo="$(oc get co -o json | jq -r '.items[] | select( - (.status.conditions[] | select(.type=="Available")).status != "True" or - (.status.conditions[] | select(.type=="Progressing")).status != "False" or - (.status.conditions[] | select(.type=="Degraded")).status != "False" - ) | .metadata.name')" + unhealthyCo="$(oc get co -o go-template='{{range .items}}{{$avail := ""}}{{$prog := ""}}{{$deg := ""}}{{range .status.conditions}}{{if eq .type "Available"}}{{$avail = .status}}{{end}}{{if eq .type "Progressing"}}{{$prog = .status}}{{end}}{{if eq .type "Degraded"}}{{$deg = .status}}{{end}}{{end}}{{if or (ne $avail "True") (ne $prog "False") (ne $deg "False")}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}')" if [[ -n "${unhealthyCo}" ]]; then : "Unhealthy ClusterOperators: ${unhealthyCo}" return 1 @@ -321,7 +326,7 @@ ValidatePlatformHealth() { : "All ClusterOperators healthy" typeset unreadyNodes="" - unreadyNodes="$(oc get node -o json | jq -r '.items[] | select(.status.conditions[] | select(.type=="Ready" and .status!="True")) | .metadata.name')" + unreadyNodes="$(oc get node -o go-template='{{range .items}}{{$ready := ""}}{{range .status.conditions}}{{if eq .type "Ready"}}{{$ready = .status}}{{end}}{{end}}{{if ne $ready "True"}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}')" if [[ -n "${unreadyNodes}" ]]; then : "Not-Ready nodes: ${unreadyNodes}" return 1 @@ -329,11 +334,7 @@ ValidatePlatformHealth() { : "All nodes Ready" typeset mcpIssues="" - mcpIssues="$(oc get machineconfigpools -o json | jq -r '.items[] | select( - (.status.conditions[] | select(.type=="Updated")).status != "True" or - (.status.conditions[] | select(.type=="Updating")).status != "False" or - (.status.conditions[] | select(.type=="Degraded")).status != "False" - ) | .metadata.name')" + mcpIssues="$(oc get machineconfigpools -o go-template='{{range .items}}{{$upd := ""}}{{$upting := ""}}{{$deg := ""}}{{range .status.conditions}}{{if eq .type "Updated"}}{{$upd = .status}}{{end}}{{if eq .type "Updating"}}{{$upting = .status}}{{end}}{{if eq .type "Degraded"}}{{$deg = .status}}{{end}}{{end}}{{if or (ne $upd "True") (ne $upting "False") (ne $deg "False")}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}')" if [[ -n "${mcpIssues}" ]]; then : "Unhealthy MachineConfigPools: ${mcpIssues}" return 1 @@ -342,7 +343,7 @@ ValidatePlatformHealth() { true } -ValidateOppOperators() { +function ValidateOppOperators () { : "Validating OPP operator health" typeset -a operatorsArr=() IFS=',' read -ra operatorsArr <<< "${OPP_OPERATORS}" @@ -352,14 +353,14 @@ ValidateOppOperators() { typeset allCsvsJson="" typeset -i failCount=0 - allCsvsJson="$(oc get csv -A -o json)" || { + allCsvsJson="$(oc get csv -A -o go-template='{{range .items}}{{.metadata.namespace}}{{"\t"}}{{.metadata.name}}{{"\t"}}{{with .status}}{{.phase}}{{end}}{{"\n"}}{{end}}')" || { : "Failed to retrieve CSVs" return 1 } - typeset phase="" + typeset phase="" op="" for op in "${operatorsArr[@]}"; do - phase="$(echo "${allCsvsJson}" | jq -r --arg op "${op}" '[.items[] | select(.metadata.name | contains($op))][0].status.phase // empty')" || true + phase="$(awk -F'\t' -v op="${op}" 'index($2, op) == 1 {print $3; exit}' <<< "${allCsvsJson}")" if [[ -z "${phase}" ]]; then : "CSV not found for operator: ${op}" (( failCount += 1 )) @@ -376,16 +377,15 @@ ValidateOppOperators() { if (( failCount > 0 )); then : "${failCount} OPP operator(s) not healthy after upgrade" : "Full CSV listing:" - echo "${allCsvsJson}" | jq -r '.items[] | "\(.metadata.namespace)\t\(.metadata.name)\t\(.status.phase)"' + echo "${allCsvsJson}" return 1 fi : "Checking pod readiness for OPP operator namespaces" - typeset oppNamespaces="" - oppNamespaces="$(echo "${allCsvsJson}" | jq -r --arg ops "${OPP_OPERATORS}" '($ops | split(",")) as $opArr | [.items[] | select(.metadata.name as $n | $opArr | any(. as $op | $n | contains($op))) | .metadata.namespace] | unique | .[]')" - typeset notReady="" ns="" - for ns in ${oppNamespaces}; do - notReady="$(oc get pods -n "${ns}" --no-headers | grep -v 'Completed' | grep -v 'Running' | grep -v 'Succeeded')" || true + typeset notReady="" ns="" podList="" + for ns in $(awk -F'\t' -v ops="${OPP_OPERATORS}" 'BEGIN{n=split(ops,arr,",")} {for(i=1;i<=n;i++) if(index($2,arr[i])==1){ns[$1]=1;break}} END{for(k in ns) print k}' <<< "${allCsvsJson}"); do + podList="$(oc get pods -n "${ns}" --no-headers)" || true + notReady="$(awk '!/Completed/ && !/Running/ && !/Succeeded/' <<< "${podList}")" if [[ -n "${notReady}" ]]; then : "WARNING: Non-running pods in ${ns}:" echo "${notReady}" @@ -404,14 +404,14 @@ ValidateOppOperators() { true } -Main() { +function Main () { if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then export KUBECONFIG="${SHARED_DIR}/kubeconfig" fi ResolveTargetImage - targetVersion="$(oc adm release info "${upgradeTarget}" --output=json | jq -r '.metadata.version')" + targetVersion="$(oc adm release info "${upgradeTarget}" --output=json | awk -F'"' '/"version"/{print $4; exit}')" targetMinorVersion="$(echo "${targetVersion}" | cut -f2 -d.)" export targetVersion targetMinorVersion : "Target release: ${targetVersion} (minor: ${targetMinorVersion})" From 350b7d1319f4fbb99d691ca1fca2f9d0d4749c04 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 31 Jul 2026 13:29:05 -0500 Subject: [PATCH 2/3] Address review: replace awk JSON parsing with jsonpath, fix trap Replace awk-based JSON field extraction with oc's native -o jsonpath for version and digest lookups, eliminating fragile positional parsing that depends on key ordering. Fix trap handler to run in current shell (brace group) instead of a subshell, so exitCode propagates correctly to DebugOnExit. --- .../interop/opp/upgrade/interop-opp-upgrade-commands.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh index 354060cdaed9e..71feac7e963ca 100644 --- a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh +++ b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh @@ -67,7 +67,7 @@ function DebugOnExit () { true } -trap '{( exitCode=$?; DebugOnExit )}' EXIT TERM +trap '{ exitCode=$?; DebugOnExit; }' EXIT TERM set +x KUBECONFIG="" oc registry login @@ -91,7 +91,7 @@ function CheckSigned () { if [[ "${payload}" =~ "@sha256:" ]]; then digest="$(echo "${payload}" | cut -f2 -d@)" else - digest="$(oc image info "${payload}" -o json | awk -F'"' '/"digest"/{print $4; exit}')" + digest="$(oc image info "${payload}" -o jsonpath='{.digest}')" fi : "Image digest: ${digest}" algorithm="$(echo "${digest}" | cut -f1 -d:)" @@ -411,7 +411,7 @@ function Main () { ResolveTargetImage - targetVersion="$(oc adm release info "${upgradeTarget}" --output=json | awk -F'"' '/"version"/{print $4; exit}')" + targetVersion="$(oc adm release info "${upgradeTarget}" -o jsonpath='{.metadata.version}')" targetMinorVersion="$(echo "${targetVersion}" | cut -f2 -d.)" export targetVersion targetMinorVersion : "Target release: ${targetVersion} (minor: ${targetMinorVersion})" From caf977ca1e5384a0a1ffda60993626f6c2694fba Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 31 Jul 2026 13:38:13 -0500 Subject: [PATCH 3/3] Apply mpitt best practices: fix pipeline masking, add defensive true, drop unnecessary exports - Replace `|| true` on eval'd pipeline with explicit error handling to avoid masking oc failures under pipefail - Add `true` as final statement in functions with early returns - Replace `export` with `typeset -g` for script-local variables --- .../opp/upgrade/interop-opp-upgrade-commands.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh index 71feac7e963ca..d71390ae1e54b 100644 --- a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh +++ b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh @@ -114,6 +114,7 @@ function CheckSigned () { : "Image is not signed" return 1 fi + true } function AdminAck () { @@ -162,6 +163,7 @@ function AdminAck () { done : "Timed out waiting for admin acks" return 1 + true } function UpdateCcoAnnotation () { @@ -205,6 +207,7 @@ function UpdateCcoAnnotation () { done : "Timed out waiting for CCO annotation" return 1 + true } function InitiateUpgrade () { @@ -250,7 +253,9 @@ function MonitorUpgrade () { (( pollCount += 1 )) typeset currentStatus="" - currentStatus="$(eval "${statCmd}")" || true + if ! currentStatus="$(eval "${statCmd}")"; then + currentStatus="" + fi if [[ -n "${currentStatus}" && "${currentStatus}" != "${prevStatus}" ]]; then : "=== Upgrade Status $(date '+%T') ===" echo "${currentStatus}" @@ -289,6 +294,7 @@ function MonitorUpgrade () { : "Upgrade timed out after ${UPGRADE_TIMEOUT} minutes at $(date '+%F %T')" : "Elapsed: $(( (endTime - startTime) / 60 ))m" exit 2 + true } function StabilizeCluster () { @@ -413,12 +419,12 @@ function Main () { targetVersion="$(oc adm release info "${upgradeTarget}" -o jsonpath='{.metadata.version}')" targetMinorVersion="$(echo "${targetVersion}" | cut -f2 -d.)" - export targetVersion targetMinorVersion + typeset -g targetVersion targetMinorVersion : "Target release: ${targetVersion} (minor: ${targetMinorVersion})" sourceVersion="$(oc get clusterversion version -o jsonpath='{.status.desired.version}')" sourceMinorVersion="$(echo "${sourceVersion}" | cut -f2 -d.)" - export sourceVersion sourceMinorVersion + typeset -g sourceVersion sourceMinorVersion : "Source release: ${sourceVersion} (minor: ${sourceMinorVersion})" isForceUpdate="false"