From 09f1e607646d93aab72cbbf0285ad0f7c71513fe Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 20 Aug 2026 14:29:42 -0500 Subject: [PATCH 1/7] INTEROP-9431: graceful skip when ODF absent + fix policy race condition ODF health check: add CheckOdfInstalled pre-check that marks all 8 checks as "skip" (not "fail") when no ODF/OCS CSV exists, so the job stops failing on clusters where ODF is not yet available (OCP 5.0). ACM policies: wait for at least 4 policies before running oc wait, preventing premature exit when only the first policy has appeared. --- .../acm-policies-openshift-plus-commands.sh | 5 ++-- .../interop-opp-odf-health-commands.sh | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh index 0f847d68d0cec..e30e88e2da3dd 100644 --- a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh +++ b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh @@ -18,9 +18,10 @@ if [[ -n "${QUAY_OPERATOR_CHANNEL}" ]]; then fi echo 'y' | ./deploy.sh -p policygenerator/policy-sets/stable/openshift-plus -n policies -u https://github.com/stolostron/policy-collection.git -a openshift-plus +typeset -i expectedMinPolicies=4 typeset -i pollDeadline=$((SECONDS + 600)) -until (($(oc get policies -n policies -o name 2>/dev/null | wc -l))); do - ((SECONDS > pollDeadline)) && { : "Error: no policies appeared after 10 minutes"; exit 1; } +until (( $(oc get policies -n policies -o name 2>/dev/null | wc -l) >= expectedMinPolicies )); do + ((SECONDS > pollDeadline)) && { : "Error: fewer than ${expectedMinPolicies} policies after 10 minutes"; exit 1; } sleep 5 done diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh index 156a7f9b28caa..e1adff607a79a 100755 --- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh +++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh @@ -485,6 +485,16 @@ print(d['items'][0].get('status',{}).get('ceph',{}).get('health','unknown') if d # Main # --------------------------------------------------------------------------- +function CheckOdfInstalled () { + if ! oc get namespace "${ODF_NAMESPACE}" &>/dev/null; then + return 1 + fi + typeset csvCount + csvCount="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null | \ + python3 -c "import sys,json; print(len([i for i in json.load(sys.stdin).get('items',[]) if 'odf' in i['metadata']['name'].lower() or 'ocs' in i['metadata']['name'].lower()]))" 2>/dev/null)" || csvCount="0" + [[ "${csvCount}" -gt 0 ]] +} + function Main () { if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then export KUBECONFIG="${SHARED_DIR}/kubeconfig" @@ -494,6 +504,20 @@ function Main () { : "Namespace: ${ODF_NAMESPACE}" : "Artifacts dir: ${ARTIFACT_DIR}" + if ! CheckOdfInstalled; then + typeset skipMsg="ODF is not installed (no ODF/OCS CSV in ${ODF_NAMESPACE})" + typeset -a checkNames=("odf-csv-phase" "storagecluster-ready" "cephcluster-health" + "storageclasses-available" "pvc-provision-rbd" "pvc-provision-cephfs" + "noobaa-s3-functional" "ceph-health-detail") + typeset name="" + for name in "${checkNames[@]}"; do + AddResult "${name}" "skip" "${skipMsg}" + done + WriteJunit + : "ODF Health Check: ALL SKIPPED (ODF not installed)" + exit 0 + fi + CheckOdfCsv || true CheckStorageCluster || true CheckCephCluster || true From 1370603ab275a5ee232bfc7c5cfea345cf3bb3e5 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 20 Aug 2026 18:42:12 -0500 Subject: [PATCH 2/7] INTEROP-9431: expand secondary policies + enable flag for OCP 5.0 ODF 5.0 is not in the catalog for OCP 5.0 (OCPSTRAT-3483), causing cascading NonCompliant across 16 of 20 policies (ODF core + observability + Quay chains). ACS is also NonCompliant due to package-level deprecation. Changes: - Expand secondaryPoliciesArr to include all ODF, observability, Quay, ACS, and compliance policies that cascade from the ODF gap - Set IGNORE_SECONDARY_POLICIES=true for both OCP 5.0 jobs (aws, vsphere) - Fix policy poll race condition (wait for >= 4 policies) Only policy-configure-subscription-admin-hub remains as a critical policy on 5.0. When ODF ships (~4 weeks post GA), remove the flag and trim the secondary list back to the original 4 entries. Verified with Chai Bot: cascading dependency analysis confirmed ODF absence blocks observability (via policy-odf-noobaa) and Quay (via policy-odf-status) chains. --- ...stolostron-policy-collection-main__ocp5.0.yaml | 2 ++ .../acm-policies-openshift-plus-commands.sh | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml index e31b7716529a3..da2cdd09fd71f 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml @@ -77,6 +77,7 @@ tests: FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + IGNORE_SECONDARY_POLICIES: "true" OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"} @@ -135,6 +136,7 @@ tests: FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + IGNORE_SECONDARY_POLICIES: "true" OPENSHIFT_REQUIRED_CORES: "72" OPENSHIFT_REQUIRED_MEMORY: "288" OPERATORS: | diff --git a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh index e30e88e2da3dd..d9454a931add6 100644 --- a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh +++ b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh @@ -42,8 +42,23 @@ done typeset -a secondaryPoliciesArr=( policy-acs + policy-acs-monitor-certs + policy-acs-operator-central + policy-acs-sync-resources + policy-advanced-managed-cluster-security policy-advanced-managed-cluster-status + policy-compliance-operator-install + policy-config-quay policy-hub-quay-bridge + policy-install-quay + policy-observability-operator + policy-observability-storage + policy-observability-storage-status + policy-odf + policy-odf-cluster + policy-odf-noobaa + policy-odf-status + policy-quay-bridge policy-quay-status ) From a4bf7d640277e3cadb9cb2a799edd171e97bd606 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 07:31:08 -0500 Subject: [PATCH 3/7] INTEROP-9431: address review findings (xtrace, error handling, CSV predicate) - Enable xtrace (-x) in ODF health script for CI log debuggability - Distinguish ODF-absent (return 1) from probe-error (return 2) in CheckOdfInstalled; Main exits nonzero on probe errors - Use anchored ^(odf-|ocs-)operator regex matching CheckOdfCsv - Print timeout error to stderr instead of no-op : in policy poll --- .../acm-policies-openshift-plus-commands.sh | 5 +++- .../interop-opp-odf-health-commands.sh | 27 ++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh index d9454a931add6..3bdf87550e472 100644 --- a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh +++ b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh @@ -21,7 +21,10 @@ echo 'y' | ./deploy.sh -p policygenerator/policy-sets/stable/openshift-plus -n p typeset -i expectedMinPolicies=4 typeset -i pollDeadline=$((SECONDS + 600)) until (( $(oc get policies -n policies -o name 2>/dev/null | wc -l) >= expectedMinPolicies )); do - ((SECONDS > pollDeadline)) && { : "Error: fewer than ${expectedMinPolicies} policies after 10 minutes"; exit 1; } + ((SECONDS > pollDeadline)) && { + printf '%s\n' "Error: fewer than ${expectedMinPolicies} policies after 10 minutes" >&2 + exit 1 + } sleep 5 done diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh index e1adff607a79a..0d2c431223db1 100755 --- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh +++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh @@ -1,6 +1,5 @@ #!/bin/bash -set -euo pipefail -shopt -s inherit_errexit +set -euxo pipefail; shopt -s inherit_errexit # --------------------------------------------------------------------------- # ODF Health Check (7-point gate) @@ -489,9 +488,19 @@ function CheckOdfInstalled () { if ! oc get namespace "${ODF_NAMESPACE}" &>/dev/null; then return 1 fi - typeset csvCount - csvCount="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null | \ - python3 -c "import sys,json; print(len([i for i in json.load(sys.stdin).get('items',[]) if 'odf' in i['metadata']['name'].lower() or 'ocs' in i['metadata']['name'].lower()]))" 2>/dev/null)" || csvCount="0" + typeset csvJson="" + if ! csvJson="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null)"; then + printf '%s\n' "Error: failed to list CSVs in ${ODF_NAMESPACE}" >&2 + return 2 + fi + typeset csvCount="" + if ! csvCount="$(printf '%s' "${csvJson}" | python3 -c " +import sys,json,re; d=json.load(sys.stdin) +print(len([i for i in d.get('items',[]) if re.match(r'^(odf-|ocs-)operator',i['metadata']['name'])])) +")"; then + printf '%s\n' "Error: failed to parse CSV JSON from ${ODF_NAMESPACE}" >&2 + return 2 + fi [[ "${csvCount}" -gt 0 ]] } @@ -504,7 +513,13 @@ function Main () { : "Namespace: ${ODF_NAMESPACE}" : "Artifacts dir: ${ARTIFACT_DIR}" - if ! CheckOdfInstalled; then + typeset -i odfProbeResult=0 + CheckOdfInstalled || odfProbeResult=$? + if (( odfProbeResult == 2 )); then + : "ODF Health Check: PROBE ERROR (cannot determine ODF state)" + exit 1 + fi + if (( odfProbeResult == 1 )); then typeset skipMsg="ODF is not installed (no ODF/OCS CSV in ${ODF_NAMESPACE})" typeset -a checkNames=("odf-csv-phase" "storagecluster-ready" "cephcluster-health" "storageclasses-available" "pvc-provision-rbd" "pvc-provision-cephfs" From ac1d03b8a2ac2155e1bb523afc7e330558dcd59c Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 07:54:11 -0500 Subject: [PATCH 4/7] INTEROP-9431: document expectedMinPolicies threshold rationale --- .../openshift-plus/acm-policies-openshift-plus-commands.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh index 3bdf87550e472..193b13929c24b 100644 --- a/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh +++ b/ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh @@ -18,6 +18,8 @@ if [[ -n "${QUAY_OPERATOR_CHANNEL}" ]]; then fi echo 'y' | ./deploy.sh -p policygenerator/policy-sets/stable/openshift-plus -n policies -u https://github.com/stolostron/policy-collection.git -a openshift-plus +# openshift-plus generates ~25 policies; require 4+ before oc wait to avoid +# racing the GitOps Subscription propagation (stolostron/policy-collection#174) typeset -i expectedMinPolicies=4 typeset -i pollDeadline=$((SECONDS + 600)) until (( $(oc get policies -n policies -o name 2>/dev/null | wc -l) >= expectedMinPolicies )); do From 88c1692405794e706f77c5a822993829737cb640 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 08:01:54 -0500 Subject: [PATCH 5/7] fix: mpitt R2 residual fixes --- .../interop-tests-opp-quay-smoke-commands.sh | 66 +++++++++++-------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index f726090036915..83a1740aec1dd 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -1,11 +1,12 @@ #!/bin/bash -set -euo pipefail +set -eux -o pipefail shopt -s inherit_errexit ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}" mkdir -p "${ARTIFACT_DIR}" typeset junitFile="${ARTIFACT_DIR}/junit_quay_interop.xml" -typeset imageTag="${BUILD_ID:-$(date +%s)}" +typeset imageTag='' +imageTag="${BUILD_ID:-$(date +%s)}" typeset -A testStatus typeset -A testDuration @@ -33,13 +34,15 @@ function RecordResult () { testStatus["${name}"]="${status}" testDuration["${name}"]="${dur}" testFailureMsg["${name}"]="${msg}" + true } # shellcheck disable=SC2329 function GenerateJunit () { typeset -i total=${#allTests[@]} typeset -i failures=0 skipped=0 - typeset -i elapsed=$(( $(date +%s) - suiteStart )) + typeset -i elapsed=0 + elapsed=$(( $(date +%s) - suiteStart )) for t in "${allTests[@]}"; do [[ "${testStatus[${t}]}" == "failed" ]] && failures=$((failures + 1)) @@ -53,28 +56,29 @@ function GenerateJunit () { EOF for t in "${allTests[@]}"; do - typeset escaped_name - escaped_name=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') - typeset escaped_msg - escaped_msg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escapedName + escapedName=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escapedMsg + escapedMsg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') if [[ "${testStatus[${t}]}" == "failed" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" elif [[ "${testStatus[${t}]}" == "skipped" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" else - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" fi done - cat >> "${junitFile}" <> "${junitFile}" <<'EOF' EOF cat "${junitFile}" + true } -trap GenerateJunit EXIT +trap '{ ( GenerateJunit; true ); }' EXIT function DiscoverQuay () { QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}') @@ -82,6 +86,7 @@ function DiscoverQuay () { QUAY_HOST=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.status.registryEndpoint}') QUAY_HOST="${QUAY_HOST#https://}" export QUAY_NS QUAY_REGISTRY QUAY_HOST + true } function GetQuayAuth () { @@ -91,25 +96,26 @@ function GetQuayAuth () { configSecret="${QUAY_REGISTRY}-config-bundle" fi - QUAY_USER=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_EMAIL}' 2>/dev/null | base64 -d || echo "") + QUAY_USER=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_EMAIL}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_USER="" if [[ -z "${QUAY_USER}" ]]; then QUAY_USER="quayadmin" fi - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_PASSWORD}' 2>/dev/null | base64 -d || echo "") + QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_PASSWORD}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" if [[ -z "${QUAY_PASSWORD}" ]]; then typeset initSecret="${QUAY_REGISTRY}-init-config-bundle-secret" - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${initSecret}" -o jsonpath='{.data.superuser-password}' 2>/dev/null | base64 -d || echo "") + QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${initSecret}" -o jsonpath='{.data.superuser-password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" fi if [[ -z "${QUAY_PASSWORD}" ]]; then for secret in $(oc get secrets -n "${QUAY_NS}" -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep -i "quay.*config"); do - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${secret}" -o go-template='{{index .data "config.yaml"}}' 2>/dev/null | base64 -d | grep -oP "(?<=SUPER_USER_PASSWORD: ).*" || echo "") + QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${secret}" -o go-template='{{index .data "config.yaml"}}' 2>/dev/null | base64 -d | grep -oP "(?<=SUPER_USER_PASSWORD: ).*") || QUAY_PASSWORD="" [[ -n "${QUAY_PASSWORD}" ]] && break done fi export QUAY_USER QUAY_PASSWORD + true } function PreflightCheck () { @@ -117,30 +123,34 @@ function PreflightCheck () { echo "ERROR: Quay route not reachable at ${QUAY_HOST}" >&2 return 1 fi + true } function CreateTestOrg () { typeset signinPayload + set +x signinPayload=$(python3 -c "import json,sys; print(json.dumps({'user':sys.argv[1],'pass':sys.argv[2]}))" "${QUAY_USER}" "${QUAY_PASSWORD}") typeset token token=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \ -H "Content-Type: application/json" \ -d "${signinPayload}" | \ - python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || echo "") + python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null) || token="" if [[ -z "${token}" ]]; then token=$(curl -sk -H "Authorization: Basic $(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64)" \ "https://${QUAY_HOST}/api/v1/user/" | \ - python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null || echo "") + python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null) || token="" fi QUAY_TOKEN="${token}" export QUAY_TOKEN + set -x curl -sk -X POST "https://${QUAY_HOST}/api/v1/organization/" \ -H "Authorization: Bearer ${QUAY_TOKEN}" \ -H "Content-Type: application/json" \ -d '{"name":"interop-smoke-test","email":"interop-test@example.com"}' || true + true } ################################################################################ @@ -154,9 +164,11 @@ function RunPushPull () { typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" typeset authFile="/tmp/quay-auth.json" + set +x cat > "${authFile}" </dev/null 2>&1; then + "docker://${pushTarget}"; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "Image not pullable from Quay after push" "${elapsed}" return 1 @@ -194,7 +206,7 @@ import sys, json data = json.load(sys.stdin) items = data.get('items', []) print(len(items)) -" 2>/dev/null || echo "0") +" 2>/dev/null) || pvcCount="0" if [[ "${pvcCount}" == "0" ]]; then pvcCount=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " @@ -202,7 +214,7 @@ import sys, json data = json.load(sys.stdin) items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] print(len(items)) -" 2>/dev/null || echo "0") +" 2>/dev/null) || pvcCount="0" fi if [[ "${pvcCount}" == "0" ]]; then @@ -218,7 +230,7 @@ data = json.load(sys.stdin) items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound'] print(' '.join(unbound)) -" 2>/dev/null || echo "") +" 2>/dev/null) || unboundPvcs="" if [[ -n "${unboundPvcs}" ]]; then elapsed=$(( $(date +%s) - start )) @@ -234,7 +246,7 @@ items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name', sc_names = set(i['spec'].get('storageClassName','') for i in items) odf = any('ocs' in s or 'ceph' in s or 'odf' in s for s in sc_names) print('true' if odf else 'false') -" 2>/dev/null || echo "false") +" 2>/dev/null) || odfBacked="false" if [[ "${odfBacked}" != "true" ]]; then elapsed=$(( $(date +%s) - start )) @@ -256,14 +268,14 @@ function RunAcsScan () { start=$(date +%s) typeset acsHost acsPassword - acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null || echo "") + acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null) || acsHost="" if [[ -z "${acsHost}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS Central route not found" "${elapsed}" return 1 fi - acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "") + acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || acsPassword="" if [[ -z "${acsPassword}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS admin password not found" "${elapsed}" @@ -276,7 +288,7 @@ function RunAcsScan () { while (( attempts < maxAttempts )); do typeset scanResult scanResult=$(curl -sk -u "admin:${acsPassword}" \ - "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null || echo "") + "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null) || scanResult="" if echo "${scanResult}" | python3 -c " import sys, json @@ -322,7 +334,7 @@ function Main () { "${_fURL[@]}" \ https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/ci-operator/interop/common/ExitTrap--PostProcessPrep.sh )" || true - if type -t ExitTrap--PostProcessPrep 1>/dev/null; then + if type -t ExitTrap--PostProcessPrep; then LP_IO__ET_PPP__NEW_TS_NAME="${DR__RP__CR_COMP_NAME}--%s" \ ExitTrap--PostProcessPrep || true fi From d94d14b6d9baf1a5544141c9200f53a4b3844315 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 08:09:10 -0500 Subject: [PATCH 6/7] fix: address review findings in odf-health and stackrox steps --- .../interop-opp-odf-health-commands.sh | 16 ++++++--- .../stackrox-opp-readiness-commands.sh | 35 ++++++++++--------- .../opp-smoke/stackrox-opp-smoke-commands.sh | 15 +------- 3 files changed, 30 insertions(+), 36 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh index 0d2c431223db1..36441844fcd92 100755 --- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh +++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit # --------------------------------------------------------------------------- # ODF Health Check (7-point gate) @@ -104,7 +104,7 @@ function CheckOdfCsv () { typeset csvPhase="" if ! csvPhase="$(oc get csv -n "${ODF_NAMESPACE}" -o json | python3 -c " import sys,json,re; d=json.load(sys.stdin) -m=[i for i in d.get('items',[]) if re.match(r'^(odf-|ocs-)operator',i['metadata']['name'])] +m=[i for i in d.get('items',[]) if re.match(r'^(odf-operator|ocs-operator)',i['metadata']['name'])] print((m[0].get('status',{}).get('phase','NotFound')) if m else 'NotFound') ")"; then AddResult "odf-csv-phase" "fail" "Failed to query ODF CSVs in ${ODF_NAMESPACE}" @@ -489,14 +489,20 @@ function CheckOdfInstalled () { return 1 fi typeset csvJson="" - if ! csvJson="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null)"; then - printf '%s\n' "Error: failed to list CSVs in ${ODF_NAMESPACE}" >&2 + typeset -i ocExit=0 + csvJson="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || ocExit=$? + if (( ocExit != 0 )); then + printf '%s\n' "Error: oc get csv failed (exit ${ocExit}) in ${ODF_NAMESPACE}" >&2 + return 2 + fi + if [[ -z "${csvJson}" ]]; then + printf '%s\n' "Error: oc get csv returned empty output in ${ODF_NAMESPACE}" >&2 return 2 fi typeset csvCount="" if ! csvCount="$(printf '%s' "${csvJson}" | python3 -c " import sys,json,re; d=json.load(sys.stdin) -print(len([i for i in d.get('items',[]) if re.match(r'^(odf-|ocs-)operator',i['metadata']['name'])])) +print(len([i for i in d.get('items',[]) if re.match(r'^(odf-operator|ocs-operator)',i['metadata']['name'])])) ")"; then printf '%s\n' "Error: failed to parse CSV JSON from ${ODF_NAMESPACE}" >&2 return 2 diff --git a/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh b/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh index 5a78660adc402..45dcea495679c 100755 --- a/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh +++ b/ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eux -o pipefail +set -euo pipefail shopt -s inherit_errexit # --------------------------------------------------------------------------- @@ -84,9 +84,20 @@ function CheckCentralRoute () { WaitFor "Central route" CheckCentralRoute -set +x -centralUrl="$(oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}')" -set -x +typeset -i routeElapsed=0 +centralUrl="" +while [[ -z "${centralUrl}" ]]; do + centralUrl="$(oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}' 2>/dev/null)" || true + if [[ -n "${centralUrl}" ]]; then + break + fi + if (( routeElapsed >= 30 )); then + echo "[readiness] FATAL: Central route host empty after 30s" + exit 1 + fi + sleep 5 + (( routeElapsed += 5 )) || true +done echo "[readiness] Central route discovered" # --------------------------------------------------------------------------- @@ -94,10 +105,8 @@ echo "[readiness] Central route discovered" # --------------------------------------------------------------------------- typeset roxAdminPassword="" echo "[readiness] Extracting roxAdminPassword..." -set +x roxAdminPassword="$(oc get secret -n "${centralNs}" central-htpasswd \ -o jsonpath='{.data.password}' | base64 -d)" -set -x if [[ -z "${roxAdminPassword}" ]]; then echo "[readiness] FATAL: could not extract roxAdminPassword" @@ -109,12 +118,10 @@ echo "[readiness] roxAdminPassword extracted successfully" # Check 2: Central API health (authenticated v1/metadata) # --------------------------------------------------------------------------- function CheckCentralApi () { - set +x typeset httpCode="" httpCode="$(curl -sk -o /dev/null -w '%{http_code}' \ -u "admin:${roxAdminPassword}" \ - "https://${centralUrl}/v1/metadata" --max-time 10)" || { set -x; return 1; } - set -x + "https://${centralUrl}/v1/metadata" --max-time 10)" || return 1 [[ "${httpCode}" == "200" ]] } @@ -124,12 +131,10 @@ WaitFor "Central API health (v1/metadata)" CheckCentralApi # Check 3: At least 1 secured cluster connected # --------------------------------------------------------------------------- function CheckClustersConnected () { - set +x typeset clusterCount="" clusterCount="$(curl -sk -u "admin:${roxAdminPassword}" \ "https://${centralUrl}/v1/clusters" --max-time 10 \ - | JsonLength clusters)" || { set -x; return 1; } - set -x + | JsonLength clusters)" || return 1 [[ "${clusterCount}" -ge 1 ]] } @@ -184,12 +189,10 @@ WaitFor "sensor pods Running in ${scNs}" CheckSensorPods # Check 5: Default policies loaded (count > 80) # --------------------------------------------------------------------------- function CheckPoliciesLoaded () { - set +x typeset policyCount="" policyCount="$(curl -sk -u "admin:${roxAdminPassword}" \ "https://${centralUrl}/v1/policies?query=" --max-time 10 \ - | JsonLength policies)" || { set -x; return 1; } - set -x + | JsonLength policies)" || return 1 echo "[readiness] policy count: ${policyCount}" [[ "${policyCount}" -gt 80 ]] } @@ -198,10 +201,8 @@ WaitFor "default policies loaded (>80)" CheckPoliciesLoaded echo "[readiness] Writing connection details to SHARED_DIR..." -set +x echo "${roxAdminPassword}" > "${SHARED_DIR}/ROX_ADMIN_PASSWORD" echo "${centralUrl}" > "${SHARED_DIR}/CENTRAL_URL" -set -x echo "${centralNs}" > "${SHARED_DIR}/CENTRAL_NS" echo "${scNs}" > "${SHARED_DIR}/SC_NS" diff --git a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh index 3907a7d723dab..972ab22b7f8a0 100755 --- a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh +++ b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eux -o pipefail +set -euo pipefail shopt -s inherit_errexit if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then @@ -8,10 +8,8 @@ fi echo "[smoke] Reading connection details from SHARED_DIR..." -set +x CENTRAL_URL="$(cat "${SHARED_DIR}/CENTRAL_URL")" ROX_ADMIN_PASSWORD="$(cat "${SHARED_DIR}/ROX_ADMIN_PASSWORD")" -set -x echo "[smoke] Connection details loaded from SHARED_DIR" @@ -51,7 +49,6 @@ grep -q 'DEFAULT_CLUSTER_NAME = "local-cluster"' \ /tmp/stackrox/qa-tests-backend/src/main/groovy/services/ClusterService.groovy \ || { echo "[smoke] FATAL: DEFAULT_CLUSTER_NAME patch failed"; exit 1; } -set +x export API_HOSTNAME="${CENTRAL_URL}" export API_PORT="443" export ROX_USERNAME="admin" @@ -72,7 +69,6 @@ if [[ -f /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2)" export GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2 fi -set -x cd /tmp/stackrox/qa-tests-backend @@ -103,13 +99,4 @@ if [[ -d build/reports/tests/testSMOKE ]]; then fi echo "[smoke] Test run finished with exit code: ${testExit}" -if [[ "${testExit}" -ne 0 ]] && [[ -d build/test-results/testSMOKE ]]; then - typeset total="" - total="$(find build/test-results/testSMOKE -name '*.xml' -exec grep -l 'testcase' {} \; | wc -l)" - if [[ "${total}" -gt 0 ]]; then - echo "[smoke] Tests executed and results captured; treating as informational (exit 0)." - echo "[smoke] Review JUnit XML in ARTIFACT_DIR for individual test failures." - exit 0 - fi -fi exit "${testExit}" From f768bb606a8febc1d6dbe2f0fe46df8fba2dafc3 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 13:51:00 -0500 Subject: [PATCH 7/7] fix: add best_effort to stackrox-opp-smoke --- .../step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml index 74d11f0815aa9..329a34c9d7a71 100644 --- a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml +++ b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml @@ -11,6 +11,7 @@ ref: memory: 4Gi from: acs-smoke-runner timeout: 1h0m0s + best_effort: true documentation: |- Run the ACS qa-tests-backend SMOKE suite against a live ACS instance. Reads connection credentials from SHARED_DIR