INTEROP-9431: ODF health check graceful skip + policy race fix

ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml

@@ -77,6 +77,7 @@ tests:
       FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323
       FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP
       FIREWATCH_FAIL_WITH_TEST_FAILURES: "true"
+      IGNORE_SECONDARY_POLICIES: "true"
       OPERATORS: |
         [
           {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}
@@ -135,6 +136,7 @@ tests:
       FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323
       FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP
       FIREWATCH_FAIL_WITH_TEST_FAILURES: "true"
+      IGNORE_SECONDARY_POLICIES: "true"
       OPENSHIFT_REQUIRED_CORES: "72"
       OPENSHIFT_REQUIRED_MEMORY: "288"
       OPERATORS: |

ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh

@@ -18,9 +18,15 @@ if [[ -n "${QUAY_OPERATOR_CHANNEL}" ]]; then
 fi
 echo 'y' | ./deploy.sh -p policygenerator/policy-sets/stable/openshift-plus -n policies -u https://github.com/stolostron/policy-collection.git -a openshift-plus
 
+# openshift-plus generates ~25 policies; require 4+ before oc wait to avoid
+# racing the GitOps Subscription propagation (stolostron/policy-collection#174)
+typeset -i expectedMinPolicies=4
 typeset -i pollDeadline=$((SECONDS + 600))
-until (($(oc get policies -n policies -o name 2>/dev/null | wc -l))); do
-  ((SECONDS > pollDeadline)) && { : "Error: no policies appeared after 10 minutes"; exit 1; }
+until (( $(oc get policies -n policies -o name 2>/dev/null | wc -l) >= expectedMinPolicies )); do
+  ((SECONDS > pollDeadline)) && {
+    printf '%s\n' "Error: fewer than ${expectedMinPolicies} policies after 10 minutes" >&2
+    exit 1
+  }
   sleep 5
 done
 
@@ -41,8 +47,23 @@ done
 
 typeset -a secondaryPoliciesArr=(
   policy-acs
+  policy-acs-monitor-certs
+  policy-acs-operator-central
+  policy-acs-sync-resources
+  policy-advanced-managed-cluster-security
   policy-advanced-managed-cluster-status
+  policy-compliance-operator-install
+  policy-config-quay
   policy-hub-quay-bridge
+  policy-install-quay
+  policy-observability-operator
+  policy-observability-storage
+  policy-observability-storage-status
+  policy-odf
+  policy-odf-cluster
+  policy-odf-noobaa
+  policy-odf-status
+  policy-quay-bridge
   policy-quay-status
 )
 

ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh

@@ -1,11 +1,12 @@
 #!/bin/bash
-set -euo pipefail
+set -eux -o pipefail
 shopt -s inherit_errexit
 
 ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}"
 mkdir -p "${ARTIFACT_DIR}"
 typeset junitFile="${ARTIFACT_DIR}/junit_quay_interop.xml"
-typeset imageTag="${BUILD_ID:-$(date +%s)}"
+typeset imageTag=''
+imageTag="${BUILD_ID:-$(date +%s)}"
 
 typeset -A testStatus
 typeset -A testDuration
@@ -33,13 +34,15 @@ function RecordResult () {
     testStatus["${name}"]="${status}"
     testDuration["${name}"]="${dur}"
     testFailureMsg["${name}"]="${msg}"
+    true
 }
 
 # shellcheck disable=SC2329
 function GenerateJunit () {
     typeset -i total=${#allTests[@]}
     typeset -i failures=0 skipped=0
-    typeset -i elapsed=$(( $(date +%s) - suiteStart ))
+    typeset -i elapsed=0
+    elapsed=$(( $(date +%s) - suiteStart ))
 
     for t in "${allTests[@]}"; do
         [[ "${testStatus[${t}]}" == "failed" ]] && failures=$((failures + 1))
@@ -53,35 +56,37 @@ function GenerateJunit () {
 EOF
 
     for t in "${allTests[@]}"; do
-        typeset escaped_name
-        escaped_name=$(printf '%s' "${t}" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g')
-        typeset escaped_msg
-        escaped_msg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g')
+        typeset escapedName
+        escapedName=$(printf '%s' "${t}" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g')
+        typeset escapedMsg
+        escapedMsg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g')
 
         if [[ "${testStatus[${t}]}" == "failed" ]]; then
-            echo "    <testcase name=\"${escaped_name}\" classname=\"interop-tests-opp-quay-smoke\" time=\"${testDuration[${t}]}\"><failure message=\"${escaped_msg}\"><![CDATA[${testFailureMsg[${t}]}]]></failure></testcase>" >> "${junitFile}"
+            echo "    <testcase name=\"${escapedName}\" classname=\"interop-tests-opp-quay-smoke\" time=\"${testDuration[${t}]}\"><failure message=\"${escapedMsg}\"><![CDATA[${testFailureMsg[${t}]}]]></failure></testcase>" >> "${junitFile}"
         elif [[ "${testStatus[${t}]}" == "skipped" ]]; then
-            echo "    <testcase name=\"${escaped_name}\" classname=\"interop-tests-opp-quay-smoke\" time=\"${testDuration[${t}]}\"><skipped message=\"${escaped_msg}\"/></testcase>" >> "${junitFile}"
+            echo "    <testcase name=\"${escapedName}\" classname=\"interop-tests-opp-quay-smoke\" time=\"${testDuration[${t}]}\"><skipped message=\"${escapedMsg}\"/></testcase>" >> "${junitFile}"
         else
-            echo "    <testcase name=\"${escaped_name}\" classname=\"interop-tests-opp-quay-smoke\" time=\"${testDuration[${t}]}\"/>" >> "${junitFile}"
+            echo "    <testcase name=\"${escapedName}\" classname=\"interop-tests-opp-quay-smoke\" time=\"${testDuration[${t}]}\"/>" >> "${junitFile}"
         fi
     done
 
-    cat >> "${junitFile}" <<EOF
+    cat >> "${junitFile}" <<'EOF'
   </testsuite>
 </testsuites>
 EOF
     cat "${junitFile}"
+    true
 }
 
-trap GenerateJunit EXIT
+trap '{ ( GenerateJunit; true ); }' EXIT
 
 function DiscoverQuay () {
     QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}')
     QUAY_REGISTRY=$(oc get quayregistry -n "${QUAY_NS}" -o jsonpath='{.items[0].metadata.name}')
     QUAY_HOST=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.status.registryEndpoint}')
     QUAY_HOST="${QUAY_HOST#https://}"
     export QUAY_NS QUAY_REGISTRY QUAY_HOST
+    true
 }
 
 function GetQuayAuth () {
@@ -91,56 +96,61 @@ function GetQuayAuth () {
         configSecret="${QUAY_REGISTRY}-config-bundle"
     fi
 
-    QUAY_USER=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_EMAIL}' 2>/dev/null | base64 -d || echo "")
+    QUAY_USER=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_EMAIL}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_USER=""
     if [[ -z "${QUAY_USER}" ]]; then
         QUAY_USER="quayadmin"
     fi
-    QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_PASSWORD}' 2>/dev/null | base64 -d || echo "")
+    QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_PASSWORD}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD=""
 
     if [[ -z "${QUAY_PASSWORD}" ]]; then
         typeset initSecret="${QUAY_REGISTRY}-init-config-bundle-secret"
-        QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${initSecret}" -o jsonpath='{.data.superuser-password}' 2>/dev/null | base64 -d || echo "")
+        QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${initSecret}" -o jsonpath='{.data.superuser-password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD=""
     fi
 
     if [[ -z "${QUAY_PASSWORD}" ]]; then
         for secret in $(oc get secrets -n "${QUAY_NS}" -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep -i "quay.*config"); do
-            QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${secret}" -o go-template='{{index .data "config.yaml"}}' 2>/dev/null | base64 -d | grep -oP "(?<=SUPER_USER_PASSWORD: ).*" || echo "")
+            QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${secret}" -o go-template='{{index .data "config.yaml"}}' 2>/dev/null | base64 -d | grep -oP "(?<=SUPER_USER_PASSWORD: ).*") || QUAY_PASSWORD=""
             [[ -n "${QUAY_PASSWORD}" ]] && break
         done
     fi
 
     export QUAY_USER QUAY_PASSWORD
+    true
 }
 
 function PreflightCheck () {
     if ! curl -sk --connect-timeout 15 "https://${QUAY_HOST}/api/v1/discovery" | grep -qi "quay"; then
         echo "ERROR: Quay route not reachable at ${QUAY_HOST}" >&2
         return 1
     fi
+    true
 }
 
 function CreateTestOrg () {
     typeset signinPayload
+    set +x
     signinPayload=$(python3 -c "import json,sys; print(json.dumps({'user':sys.argv[1],'pass':sys.argv[2]}))" "${QUAY_USER}" "${QUAY_PASSWORD}")
     typeset token
     token=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \
         -H "Content-Type: application/json" \
         -d "${signinPayload}" | \
-        python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || echo "")
+        python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null) || token=""
 
     if [[ -z "${token}" ]]; then
         token=$(curl -sk -H "Authorization: Basic $(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64)" \
             "https://${QUAY_HOST}/api/v1/user/" | \
-            python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null || echo "")
+            python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null) || token=""
     fi
 
     QUAY_TOKEN="${token}"
     export QUAY_TOKEN
+    set -x
 
     curl -sk -X POST "https://${QUAY_HOST}/api/v1/organization/" \
         -H "Authorization: Bearer ${QUAY_TOKEN}" \
         -H "Content-Type: application/json" \
         -d '{"name":"interop-smoke-test","email":"interop-test@example.com"}' || true
+    true
 }
 
 ################################################################################
@@ -154,9 +164,11 @@ function RunPushPull () {
     typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}"
     typeset authFile="/tmp/quay-auth.json"
 
+    set +x
     cat > "${authFile}" <<EOF
 {"auths":{"${QUAY_HOST}":{"auth":"$(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64)"}}}
 EOF
+    set -x
 
     if ! skopeo copy --dest-tls-verify=false \
         --dest-authfile="${authFile}" \
@@ -169,7 +181,7 @@ EOF
 
     if ! skopeo inspect --tls-verify=false \
         --authfile="${authFile}" \
-        "docker://${pushTarget}" >/dev/null 2>&1; then
+        "docker://${pushTarget}"; then
         elapsed=$(( $(date +%s) - start ))
         RecordResult "${testName}" "failed" "Image not pullable from Quay after push" "${elapsed}"
         return 1
@@ -194,15 +206,15 @@ import sys, json
 data = json.load(sys.stdin)
 items = data.get('items', [])
 print(len(items))
-" 2>/dev/null || echo "0")
+" 2>/dev/null) || pvcCount="0"
 
     if [[ "${pvcCount}" == "0" ]]; then
         pvcCount=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c "
 import sys, json
 data = json.load(sys.stdin)
 items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()]
 print(len(items))
-" 2>/dev/null || echo "0")
+" 2>/dev/null) || pvcCount="0"
     fi
 
     if [[ "${pvcCount}" == "0" ]]; then
@@ -218,7 +230,7 @@ data = json.load(sys.stdin)
 items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()]
 unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound']
 print(' '.join(unbound))
-" 2>/dev/null || echo "")
+" 2>/dev/null) || unboundPvcs=""
 
     if [[ -n "${unboundPvcs}" ]]; then
         elapsed=$(( $(date +%s) - start ))
@@ -234,7 +246,7 @@ items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name',
 sc_names = set(i['spec'].get('storageClassName','') for i in items)
 odf = any('ocs' in s or 'ceph' in s or 'odf' in s for s in sc_names)
 print('true' if odf else 'false')
-" 2>/dev/null || echo "false")
+" 2>/dev/null) || odfBacked="false"
 
     if [[ "${odfBacked}" != "true" ]]; then
         elapsed=$(( $(date +%s) - start ))
@@ -256,14 +268,14 @@ function RunAcsScan () {
     start=$(date +%s)
 
     typeset acsHost acsPassword
-    acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null || echo "")
+    acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null) || acsHost=""
     if [[ -z "${acsHost}" ]]; then
         elapsed=$(( $(date +%s) - start ))
         RecordResult "${testName}" "failed" "ACS Central route not found" "${elapsed}"
         return 1
     fi
 
-    acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "")
+    acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || acsPassword=""
     if [[ -z "${acsPassword}" ]]; then
         elapsed=$(( $(date +%s) - start ))
         RecordResult "${testName}" "failed" "ACS admin password not found" "${elapsed}"
@@ -276,7 +288,7 @@ function RunAcsScan () {
     while (( attempts < maxAttempts )); do
         typeset scanResult
         scanResult=$(curl -sk -u "admin:${acsPassword}" \
-            "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null || echo "")
+            "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null) || scanResult=""
 
         if echo "${scanResult}" | python3 -c "
 import sys, json
@@ -322,7 +334,7 @@ function Main () {
             "${_fURL[@]}" \
                 https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/ci-operator/interop/common/ExitTrap--PostProcessPrep.sh
         )" || true
-        if type -t ExitTrap--PostProcessPrep 1>/dev/null; then
+        if type -t ExitTrap--PostProcessPrep; then
             LP_IO__ET_PPP__NEW_TS_NAME="${DR__RP__CR_COMP_NAME}--%s" \
                 ExitTrap--PostProcessPrep || true
         fi

ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
-set -euo pipefail
-shopt -s inherit_errexit
+set -euo pipefail; shopt -s inherit_errexit
 
 # ---------------------------------------------------------------------------
 # ODF Health Check (7-point gate)
@@ -105,7 +104,7 @@ function CheckOdfCsv () {
     typeset csvPhase=""
     if ! csvPhase="$(oc get csv -n "${ODF_NAMESPACE}" -o json | python3 -c "
 import sys,json,re; d=json.load(sys.stdin)
-m=[i for i in d.get('items',[]) if re.match(r'^(odf-|ocs-)operator',i['metadata']['name'])]
+m=[i for i in d.get('items',[]) if re.match(r'^(odf-operator|ocs-operator)',i['metadata']['name'])]
 print((m[0].get('status',{}).get('phase','NotFound')) if m else 'NotFound')
 ")"; then
         AddResult "odf-csv-phase" "fail" "Failed to query ODF CSVs in ${ODF_NAMESPACE}"
@@ -485,6 +484,32 @@ print(d['items'][0].get('status',{}).get('ceph',{}).get('health','unknown') if d
 # Main
 # ---------------------------------------------------------------------------
 
+function CheckOdfInstalled () {
+    if ! oc get namespace "${ODF_NAMESPACE}" &>/dev/null; then
+        return 1
+    fi
+    typeset csvJson=""
+    typeset -i ocExit=0
+    csvJson="$(oc get csv -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || ocExit=$?
+    if (( ocExit != 0 )); then
+        printf '%s\n' "Error: oc get csv failed (exit ${ocExit}) in ${ODF_NAMESPACE}" >&2
+        return 2
+    fi
+    if [[ -z "${csvJson}" ]]; then
+        printf '%s\n' "Error: oc get csv returned empty output in ${ODF_NAMESPACE}" >&2
+        return 2
+    fi
+    typeset csvCount=""
+    if ! csvCount="$(printf '%s' "${csvJson}" | python3 -c "
+import sys,json,re; d=json.load(sys.stdin)
+print(len([i for i in d.get('items',[]) if re.match(r'^(odf-operator|ocs-operator)',i['metadata']['name'])]))
+")"; then
+        printf '%s\n' "Error: failed to parse CSV JSON from ${ODF_NAMESPACE}" >&2
+        return 2
+    fi
+    [[ "${csvCount}" -gt 0 ]]
+}
+
 function Main () {
     if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then
         export KUBECONFIG="${SHARED_DIR}/kubeconfig"
@@ -494,6 +519,26 @@ function Main () {
     : "Namespace: ${ODF_NAMESPACE}"
     : "Artifacts dir: ${ARTIFACT_DIR}"
 
+    typeset -i odfProbeResult=0
+    CheckOdfInstalled || odfProbeResult=$?
+    if (( odfProbeResult == 2 )); then
+        : "ODF Health Check: PROBE ERROR (cannot determine ODF state)"
+        exit 1
+    fi
+    if (( odfProbeResult == 1 )); then
+        typeset skipMsg="ODF is not installed (no ODF/OCS CSV in ${ODF_NAMESPACE})"
+        typeset -a checkNames=("odf-csv-phase" "storagecluster-ready" "cephcluster-health"
+            "storageclasses-available" "pvc-provision-rbd" "pvc-provision-cephfs"
+            "noobaa-s3-functional" "ceph-health-detail")
+        typeset name=""
+        for name in "${checkNames[@]}"; do
+            AddResult "${name}" "skip" "${skipMsg}"
+        done
+        WriteJunit
+        : "ODF Health Check: ALL SKIPPED (ODF not installed)"
+        exit 0
+    fi
+
     CheckOdfCsv          || true
     CheckStorageCluster  || true
     CheckCephCluster     || true