From 828f2d5e9bda3142e4bbadf367a802ddc95f25c6 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Wed, 5 Aug 2026 14:56:38 -0500
Subject: [PATCH 01/10] INTEROP-9384: Replace OCS acceptance tests with ODF
 health check

The interop-tests-ocs-tests step runs 28 single-product ODF acceptance
tests after deliberately deleting the ACM subscription, making them
fundamentally misaligned with interop testing. The step has a 3h timeout,
depends on an external ocs-ci image, and the NooBaa endpoint timeout
(OCSQE-5148) has been unresponsive for 16+ days.

Replace with a lightweight 7-point ODF health gate (~2 min):
1. ODF Operator CSV in Succeeded phase
2. StorageCluster phase == Ready
3. CephCluster health == HEALTH_OK or HEALTH_WARN
4. Default StorageClasses available (ceph-rbd, cephfs)
5. PVC provisionable (create, bind, delete)
6. NooBaa system Ready + S3 put/get/delete functional check
7. Ceph overall health detail

The NooBaa S3 functional check (check 6) is critical because Quay has
a hard dependency on NooBaa for image blob storage. If NooBaa accepts
connections but cannot serve object operations, Quay will silently
fail on push/pull.

Produces JUnit XML for Prow/Sippy/TestGrid. No external image dependency.
Does NOT delete the ACM subscription (keeps stack intact for true interop).
---
 ...stron-policy-collection-main__ocp4.22.yaml |   4 +-
 ...ostron-policy-collection-main__ocp5.0.yaml |   4 +-
 .../interop/opp/odf-health/OWNERS             |   3 +
 .../interop-opp-odf-health-commands.sh        | 448 ++++++++++++++++++
 .../interop-opp-odf-health-ref.metadata.json  |  11 +
 .../interop-opp-odf-health-ref.yaml           |  18 +
 6 files changed, 484 insertions(+), 4 deletions(-)
 create mode 100644 ci-operator/step-registry/interop/opp/odf-health/OWNERS
 create mode 100755 ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
 create mode 100644 ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.metadata.json
 create mode 100644 ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml

diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml
index 7c648df07abcf..1472f4c4384f1 100644
--- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml
+++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml
@@ -125,7 +125,7 @@ tests:
     - ref: acm-tests-clc-create
     - ref: acm-fetch-managed-clusters
     - ref: acm-opp-app
-    - ref: interop-tests-ocs-tests
+    - ref: interop-opp-odf-health
     - ref: quay-tests-quay-interop-test
     - ref: acm-tests-observability
     - ref: acm-tests-grc
@@ -184,7 +184,7 @@ tests:
     - ref: acm-policies-openshift-plus-setup
     - ref: acm-policies-openshift-plus
     - chain: cucushift-installer-check-cluster-health
-    - ref: interop-tests-ocs-tests
+    - ref: interop-opp-odf-health
     - ref: acm-tests-observability
     - ref: acm-opp-app
     - ref: openshift-extended-test
diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml
index 2efc0cc8ee622..f17a2101b1c4b 100644
--- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml
+++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml
@@ -123,7 +123,7 @@ tests:
     - ref: acm-tests-clc-create
     - ref: acm-fetch-managed-clusters
     - ref: acm-opp-app
-    - ref: interop-tests-ocs-tests
+    - ref: interop-opp-odf-health
     - ref: quay-tests-quay-interop-test
     - ref: acm-tests-observability
     - ref: acm-tests-grc
@@ -180,7 +180,7 @@ tests:
     - ref: acm-policies-openshift-plus-setup
     - ref: acm-policies-openshift-plus
     - chain: cucushift-installer-check-cluster-health
-    - ref: interop-tests-ocs-tests
+    - ref: interop-opp-odf-health
     - ref: acm-tests-observability
     - ref: acm-opp-app
     - ref: openshift-extended-test
diff --git a/ci-operator/step-registry/interop/opp/odf-health/OWNERS b/ci-operator/step-registry/interop/opp/odf-health/OWNERS
new file mode 100644
index 0000000000000..41d144d3728a2
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/odf-health/OWNERS
@@ -0,0 +1,3 @@
+approvers: &owners
+- cspi-qe-ocp-lp
+reviewers: *owners
diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
new file mode 100755
index 0000000000000..95bcf854a6306
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
@@ -0,0 +1,448 @@
+#!/bin/bash
+set -euxo pipefail
+shopt -s inherit_errexit
+
+# ---------------------------------------------------------------------------
+# ODF Health Check (7-point gate)
+#
+# Validates that ODF is healthy and functional in the OPP interop cluster.
+# Replaces the former interop-tests-ocs-tests step which ran single-product
+# ODF acceptance tests unrelated to cross-product interop.
+#
+# Produces JUnit XML consumed by Prow / Sippy / TestGrid.
+# ---------------------------------------------------------------------------
+
+ODF_NAMESPACE="${ODF_NAMESPACE:-openshift-storage}"
+NOOBAA_S3_TIMEOUT="${NOOBAA_S3_TIMEOUT:-30}"
+
+typeset junitFile="${ARTIFACT_DIR}/junit_odf_health.xml"
+
+typeset -a tcNamesArr=()
+typeset -a tcResultsArr=()
+typeset -a tcMessagesArr=()
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+AddResult() {
+    typeset name="${1:-}"; (($#)) && shift
+    typeset result="${1:-}"; (($#)) && shift
+    typeset message="${1:-}"; (($#)) && shift
+    tcNamesArr+=("${name}")
+    tcResultsArr+=("${result}")
+    tcMessagesArr+=("${message}")
+    true
+}
+
+XmlEscape() {
+    typeset text="${1:-}"; (($#)) && shift
+    text="${text//&/&amp;}"
+    text="${text//</&lt;}"
+    text="${text//>/&gt;}"
+    text="${text//\"/&quot;}"
+    text="${text//\'/&apos;}"
+    printf '%s' "${text}"
+}
+
+WriteJunit() {
+    typeset -i total=${#tcNamesArr[@]}
+    typeset -i failCount=0
+    typeset -i skipCount=0
+    for r in "${tcResultsArr[@]}"; do
+        if [[ "${r}" == "fail" ]]; then
+            (( failCount++ )) || true
+        elif [[ "${r}" == "skip" ]]; then
+            (( skipCount++ )) || true
+        fi
+    done
+
+    {
+        echo '<?xml version="1.0" encoding="UTF-8"?>'
+        echo "<testsuite name=\"odf-health\" tests=\"${total}\" failures=\"${failCount}\" skipped=\"${skipCount}\">"
+        for i in "${!tcNamesArr[@]}"; do
+            typeset name=""
+            name="$(XmlEscape "${tcNamesArr[$i]}")"
+            echo "  <testcase classname=\"odf-health\" name=\"${name}\">"
+            if [[ "${tcResultsArr[$i]}" == "fail" ]]; then
+                typeset msg=""
+                msg="$(XmlEscape "${tcMessagesArr[$i]}")"
+                echo "    <failure message=\"${msg}\"></failure>"
+            elif [[ "${tcResultsArr[$i]}" == "skip" ]]; then
+                typeset msg=""
+                msg="$(XmlEscape "${tcMessagesArr[$i]}")"
+                echo "    <skipped message=\"${msg}\"/>"
+            fi
+            echo "  </testcase>"
+        done
+        echo "</testsuite>"
+    } > "${junitFile}"
+    : "JUnit XML written to ${junitFile}"
+}
+
+# shellcheck disable=SC2317
+CollectExitArtifacts() {
+    : "Collecting ODF diagnostics..."
+    oc get csv -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/odf-csvs.yaml" || true
+    oc get storagecluster -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/storagecluster.yaml" || true
+    oc get cephcluster -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/cephcluster.yaml" || true
+    oc get noobaa -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/noobaa.yaml" || true
+    oc get sc -o yaml > "${ARTIFACT_DIR}/storageclasses.yaml" || true
+}
+
+trap CollectExitArtifacts EXIT
+
+# ---------------------------------------------------------------------------
+# Check 1: ODF Operator CSV in Succeeded phase
+# ---------------------------------------------------------------------------
+
+CheckOdfCsv() {
+    : "=== Check 1: ODF Operator CSV ==="
+
+    typeset csvPhase=""
+    if ! csvPhase="$(oc get csv -n "${ODF_NAMESPACE}" -o json | jq -r '
+        [.items[] | select(.metadata.name | test("^(odf-|ocs-)operator"))] |
+        first | .status.phase // "NotFound"
+    ')"; then
+        AddResult "odf-csv-phase" "fail" "Failed to query ODF CSVs in ${ODF_NAMESPACE}"
+        return
+    fi
+
+    if [[ "${csvPhase}" == "Succeeded" ]]; then
+        : "PASS: ODF CSV phase=Succeeded"
+        AddResult "odf-csv-phase" "pass"
+    elif [[ "${csvPhase}" == "NotFound" ]]; then
+        AddResult "odf-csv-phase" "fail" "No ODF/OCS operator CSV found in ${ODF_NAMESPACE}"
+    else
+        AddResult "odf-csv-phase" "fail" "ODF CSV phase=${csvPhase} (expected Succeeded)"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Check 2: StorageCluster phase == Ready
+# ---------------------------------------------------------------------------
+
+CheckStorageCluster() {
+    : "=== Check 2: StorageCluster Ready ==="
+
+    typeset scPhase=""
+    if ! scPhase="$(oc get storagecluster -n "${ODF_NAMESPACE}" -o json | jq -r '
+        .items[0].status.phase // "NotFound"
+    ')"; then
+        AddResult "storagecluster-ready" "fail" "Failed to query StorageCluster"
+        return
+    fi
+
+    if [[ "${scPhase}" == "Ready" ]]; then
+        : "PASS: StorageCluster phase=Ready"
+        AddResult "storagecluster-ready" "pass"
+    elif [[ "${scPhase}" == "NotFound" ]]; then
+        AddResult "storagecluster-ready" "fail" "No StorageCluster found in ${ODF_NAMESPACE}"
+    else
+        AddResult "storagecluster-ready" "fail" "StorageCluster phase=${scPhase} (expected Ready)"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Check 3: CephCluster health == HEALTH_OK or HEALTH_WARN
+# ---------------------------------------------------------------------------
+
+CheckCephCluster() {
+    : "=== Check 3: CephCluster health ==="
+
+    typeset cephHealth=""
+    if ! cephHealth="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | jq -r '
+        .items[0].status.ceph.health // "NotFound"
+    ')"; then
+        AddResult "cephcluster-health" "fail" "Failed to query CephCluster"
+        return
+    fi
+
+    if [[ "${cephHealth}" == "HEALTH_OK" || "${cephHealth}" == "HEALTH_WARN" ]]; then
+        : "PASS: CephCluster health=${cephHealth}"
+        AddResult "cephcluster-health" "pass"
+    elif [[ "${cephHealth}" == "NotFound" ]]; then
+        AddResult "cephcluster-health" "fail" "No CephCluster found in ${ODF_NAMESPACE}"
+    else
+        AddResult "cephcluster-health" "fail" "CephCluster health=${cephHealth} (expected HEALTH_OK or HEALTH_WARN)"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Check 4: Default StorageClasses available (ceph-rbd, cephfs)
+# ---------------------------------------------------------------------------
+
+CheckStorageClasses() {
+    : "=== Check 4: StorageClasses ==="
+    typeset failMsg=""
+
+    for scName in ocs-storagecluster-ceph-rbd ocs-storagecluster-cephfs; do
+        if ! oc get sc "${scName}" &>/dev/null; then
+            if [[ -n "${failMsg}" ]]; then
+                failMsg="${failMsg}; StorageClass ${scName} not found"
+            else
+                failMsg="StorageClass ${scName} not found"
+            fi
+        else
+            : "PASS: StorageClass ${scName} exists"
+        fi
+    done
+
+    if [[ -z "${failMsg}" ]]; then
+        AddResult "storageclasses-available" "pass"
+    else
+        AddResult "storageclasses-available" "fail" "${failMsg}"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Check 5: PVC provisionable (create, bind, delete)
+# ---------------------------------------------------------------------------
+
+CheckPvcProvision() {
+    : "=== Check 5: PVC provisioning ==="
+
+    typeset pvcName="odf-health-check-pvc-$$"
+    typeset pvcYaml
+    pvcYaml=$(cat <<EOF
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ${pvcName}
+  namespace: ${ODF_NAMESPACE}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: ocs-storagecluster-ceph-rbd
+EOF
+)
+
+    if ! echo "${pvcYaml}" | oc apply -f - 2>/dev/null; then
+        AddResult "pvc-provision" "fail" "Failed to create test PVC"
+        return
+    fi
+
+    typeset -i maxWait=60
+    typeset -i elapsed=0
+    typeset phase=""
+    while (( elapsed < maxWait )); do
+        phase="$(oc get pvc "${pvcName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.status.phase}' 2>/dev/null || echo "")"
+        if [[ "${phase}" == "Bound" ]]; then
+            break
+        fi
+        sleep 5
+        (( elapsed += 5 )) || true
+    done
+
+    oc delete pvc "${pvcName}" -n "${ODF_NAMESPACE}" --wait=false 2>/dev/null || true
+
+    if [[ "${phase}" == "Bound" ]]; then
+        : "PASS: PVC provisioned and bound in ${elapsed}s"
+        AddResult "pvc-provision" "pass"
+    else
+        AddResult "pvc-provision" "fail" "PVC did not bind within ${maxWait}s (phase=${phase:-unknown})"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Check 6: NooBaa system Ready + S3 put/get/delete functional check
+# ---------------------------------------------------------------------------
+
+CheckNoobaa() {
+    : "=== Check 6: NooBaa S3 functional ==="
+
+    typeset nbPhase=""
+    if ! nbPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | jq -r '
+        .items[0].status.phase // "NotFound"
+    ')"; then
+        AddResult "noobaa-s3-functional" "fail" "Failed to query NooBaa"
+        return
+    fi
+
+    if [[ "${nbPhase}" == "NotFound" ]]; then
+        AddResult "noobaa-s3-functional" "fail" "No NooBaa system found in ${ODF_NAMESPACE}"
+        return
+    fi
+
+    if [[ "${nbPhase}" != "Ready" ]]; then
+        AddResult "noobaa-s3-functional" "fail" "NooBaa phase=${nbPhase} (expected Ready)"
+        return
+    fi
+
+    : "NooBaa phase=Ready, running S3 functional check..."
+
+    typeset s3Endpoint="" accessKey="" secretKey=""
+    s3Endpoint="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | jq -r '
+        .items[0].status.services.serviceS3.internalDNS[0] // empty
+    ')" || true
+
+    if [[ -z "${s3Endpoint}" ]]; then
+        s3Endpoint="https://s3.${ODF_NAMESPACE}.svc:443"
+    fi
+
+    typeset secretName=""
+    secretName="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | jq -r '
+        .items[0].status.accounts.admin.secretRef.name // "noobaa-admin"
+    ')" || true
+
+    if ! accessKey="$(oc get secret "${secretName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)"; then
+        AddResult "noobaa-s3-functional" "fail" "Cannot read NooBaa admin credentials"
+        return
+    fi
+    if ! secretKey="$(oc get secret "${secretName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)"; then
+        AddResult "noobaa-s3-functional" "fail" "Cannot read NooBaa admin credentials"
+        return
+    fi
+
+    typeset bucket="odf-health-check-$$"
+    typeset testKey="health-check-test"
+    typeset testData="odf-health-$(date +%s)"
+
+    typeset -i s3ok=1
+
+    # Use a temporary pod to run the S3 operations from inside the cluster
+    typeset podName="odf-health-s3-check-$$"
+    typeset s3Script
+    s3Script=$(cat <<'EOSCRIPT'
+#!/bin/sh
+set -e
+export AWS_ACCESS_KEY_ID="$1"
+export AWS_SECRET_ACCESS_KEY="$2"
+S3_ENDPOINT="$3"
+BUCKET="$4"
+TEST_KEY="$5"
+TEST_DATA="$6"
+
+alias s3="aws --endpoint-url ${S3_ENDPOINT} --no-verify-ssl s3"
+alias s3api="aws --endpoint-url ${S3_ENDPOINT} --no-verify-ssl s3api"
+
+# Put
+echo "${TEST_DATA}" | aws --endpoint-url "${S3_ENDPOINT}" --no-verify-ssl s3 cp - "s3://first.bucket/${TEST_KEY}" 2>/dev/null
+# Get
+RETRIEVED=$(aws --endpoint-url "${S3_ENDPOINT}" --no-verify-ssl s3 cp "s3://first.bucket/${TEST_KEY}" - 2>/dev/null)
+# Delete
+aws --endpoint-url "${S3_ENDPOINT}" --no-verify-ssl s3 rm "s3://first.bucket/${TEST_KEY}" 2>/dev/null
+# Verify
+if [ "${RETRIEVED}" = "${TEST_DATA}" ]; then
+    echo "S3_CHECK_PASS"
+else
+    echo "S3_CHECK_FAIL: data mismatch"
+fi
+EOSCRIPT
+)
+
+    typeset s3Result=""
+    if s3Result="$(oc run "${podName}" -n "${ODF_NAMESPACE}" \
+        --image=amazon/aws-cli:latest \
+        --restart=Never \
+        --rm=true \
+        --attach=true \
+        --timeout="${NOOBAA_S3_TIMEOUT}s" \
+        --command -- sh -c "$(cat <<EOF
+export AWS_ACCESS_KEY_ID='${accessKey}'
+export AWS_SECRET_ACCESS_KEY='${secretKey}'
+S3_ENDPOINT='${s3Endpoint}'
+TEST_KEY='${testKey}'
+TEST_DATA='${testData}'
+echo "\${TEST_DATA}" | aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 cp - "s3://first.bucket/\${TEST_KEY}" 2>/dev/null && \
+RETRIEVED=\$(aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 cp "s3://first.bucket/\${TEST_KEY}" - 2>/dev/null) && \
+aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 rm "s3://first.bucket/\${TEST_KEY}" 2>/dev/null && \
+if [ "\${RETRIEVED}" = "\${TEST_DATA}" ]; then echo "S3_CHECK_PASS"; else echo "S3_CHECK_FAIL: data mismatch"; fi
+EOF
+)" 2>/dev/null)"; then
+        : "S3 check pod completed"
+    else
+        : "S3 check pod failed or timed out"
+    fi
+
+    oc delete pod "${podName}" -n "${ODF_NAMESPACE}" --ignore-not-found=true --wait=false 2>/dev/null || true
+
+    if echo "${s3Result}" | grep -q "S3_CHECK_PASS"; then
+        : "PASS: NooBaa S3 put/get/delete cycle succeeded"
+        AddResult "noobaa-s3-functional" "pass"
+    else
+        typeset s3Msg="NooBaa S3 functional check failed"
+        if echo "${s3Result}" | grep -q "S3_CHECK_FAIL"; then
+            s3Msg="NooBaa S3: $(echo "${s3Result}" | grep "S3_CHECK_FAIL")"
+        fi
+        AddResult "noobaa-s3-functional" "fail" "${s3Msg}"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Check 7: Ceph overall health (via toolbox or CephCluster status)
+# ---------------------------------------------------------------------------
+
+CheckCephHealth() {
+    : "=== Check 7: Ceph health detail ==="
+
+    typeset cephDetail=""
+    if ! cephDetail="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | jq -r '
+        .items[0].status.ceph.details // {} | to_entries[] |
+        select(.value.severity != "HEALTH_OK") |
+        "\(.key): \(.value.message // "unknown")"
+    ')"; then
+        AddResult "ceph-health-detail" "fail" "Failed to query CephCluster details"
+        return
+    fi
+
+    typeset cephHealth=""
+    cephHealth="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | jq -r '
+        .items[0].status.ceph.health // "unknown"
+    ')" || true
+
+    if [[ "${cephHealth}" == "HEALTH_OK" ]]; then
+        : "PASS: Ceph health=HEALTH_OK"
+        AddResult "ceph-health-detail" "pass"
+    elif [[ "${cephHealth}" == "HEALTH_WARN" ]]; then
+        : "PASS (warn): Ceph health=HEALTH_WARN: ${cephDetail}"
+        AddResult "ceph-health-detail" "pass"
+    else
+        AddResult "ceph-health-detail" "fail" "Ceph health=${cephHealth}: ${cephDetail}"
+    fi
+}
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+Main() {
+    if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then
+        export KUBECONFIG="${SHARED_DIR}/kubeconfig"
+    fi
+
+    : "ODF Health Check (7-point gate) starting"
+    : "Namespace: ${ODF_NAMESPACE}"
+    : "Artifacts dir: ${ARTIFACT_DIR}"
+
+    CheckOdfCsv          || true
+    CheckStorageCluster  || true
+    CheckCephCluster     || true
+    CheckStorageClasses  || true
+    CheckPvcProvision    || true
+    CheckNoobaa          || true
+    CheckCephHealth      || true
+
+    WriteJunit
+
+    typeset -i hasAnyFail=0
+    for r in "${tcResultsArr[@]}"; do
+        if [[ "${r}" == "fail" ]]; then
+            hasAnyFail=1
+            break
+        fi
+    done
+
+    if (( hasAnyFail )); then
+        : "ODF Health Check: SOME CHECKS FAILED"
+        exit 1
+    fi
+
+    : "ODF Health Check: ALL PASSED"
+    exit 0
+}
+
+Main "$@"
diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.metadata.json b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.metadata.json
new file mode 100644
index 0000000000000..f3a48106962b9
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.metadata.json
@@ -0,0 +1,11 @@
+{
+	"path": "interop/opp/odf-health/interop-opp-odf-health-ref.yaml",
+	"owners": {
+		"approvers": [
+			"cspi-qe-ocp-lp"
+		],
+		"reviewers": [
+			"cspi-qe-ocp-lp"
+		]
+	}
+}
diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml
new file mode 100644
index 0000000000000..21db18ae818fb
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml
@@ -0,0 +1,18 @@
+ref:
+  as: interop-opp-odf-health
+  from: cli
+  grace_period: 30s
+  commands: interop-opp-odf-health-commands.sh
+  timeout: 5m
+  resources:
+    requests:
+      cpu: 100m
+      memory: 200Mi
+  documentation: |-
+    Validates ODF health in the OPP interop cluster via a 7-point gate:
+    ODF Operator CSV phase, StorageCluster readiness, CephCluster health,
+    default StorageClass availability, PVC provisioning, NooBaa system
+    readiness with S3 functional check, and Ceph overall health.
+    Replaces the former interop-tests-ocs-tests step which ran 28
+    single-product acceptance tests with a 3h timeout.
+    Produces JUnit XML for Prow / Sippy / TestGrid consumption.

From 7df8d1aeb766fad6af7a19c9335e3d82a2bef89d Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Thu, 6 Aug 2026 09:54:47 -0500
Subject: [PATCH 02/10] fix: resolve CI failures and apply mpitt best practices

- Remove ODF_VERSION_MAJOR_MINOR and DISABLE_ENVIRONMENT_CHECKER env
  overrides (only declared by removed interop-tests-ocs-tests step)
- Remove unused ocs-ci-tests base_image (required by validateUnused)
- Remove dead code: bucket, s3ok, s3Script variables (SC2034)
- Fix SC2155: split testData declaration and assignment
- Guard credentials with set +x/set -x to prevent log exposure
- Add true terminators to all functions (mpex best practice)
- Bump step timeout from 5m to 8m (PVC bind + cold image pull buffer)
- Fix metadata.json trailing newline
---
 ...stron-policy-collection-main__ocp4.22.yaml | 10 ----
 ...ostron-policy-collection-main__ocp5.0.yaml | 10 ----
 .../interop-opp-odf-health-commands.sh        | 48 ++++++-------------
 .../interop-opp-odf-health-ref.metadata.json  |  2 +-
 .../interop-opp-odf-health-ref.yaml           |  2 +-
 5 files changed, 16 insertions(+), 56 deletions(-)

diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml
index 1472f4c4384f1..c24a4aab33455 100644
--- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml
+++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml
@@ -23,10 +23,6 @@ base_images:
     name: "2.17"
     namespace: acm-qe
     tag: multicluster-observability-operator-opp
-  ocs-ci-tests:
-    name: ocs-ci-container
-    namespace: ci
-    tag: stable
   tests-private:
     name: tests-private
     namespace: ci
@@ -90,8 +86,6 @@ tests:
       FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP
       FIREWATCH_FAIL_WITH_TEST_FAILURES: "true"
       IGNORE_SECONDARY_POLICIES: "true"
-      MAP_TESTS: "true"
-      ODF_VERSION_MAJOR_MINOR: "4.21"
       OPERATORS: |
         [
           {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.16", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}
@@ -150,8 +144,6 @@ tests:
     cluster_profile: vsphere-connected-2
     env:
       COMPUTE_NODE_REPLICAS: "6"
-      DISABLE_ENVIRONMENT_CHECKER: "true"
-      DR__RP__CR_COMP_NAME: lp-interop--OPP
       FIREWATCH_CONFIG_FILE_PATH: https://raw.githubusercontent.com/CSPI-QE/cspi-utils/refs/heads/main/firewatch-base-configs/opp/lp-interop-vsphere.json
       FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS: '["4.22-lp","opp-vsphere-lp","opp-lp"]'
       FIREWATCH_DEFAULT_JIRA_ASSIGNEE: ftan@redhat.com
@@ -159,8 +151,6 @@ tests:
       FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP
       FIREWATCH_FAIL_WITH_TEST_FAILURES: "true"
       IGNORE_SECONDARY_POLICIES: "true"
-      MAP_TESTS: "true"
-      ODF_VERSION_MAJOR_MINOR: "4.21"
       OPENSHIFT_REQUIRED_CORES: "72"
       OPENSHIFT_REQUIRED_MEMORY: "288"
       OPERATORS: |
diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml
index f17a2101b1c4b..610fc395e85fb 100644
--- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml
+++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml
@@ -23,10 +23,6 @@ base_images:
     name: "2.17"
     namespace: acm-qe
     tag: multicluster-observability-operator-opp
-  ocs-ci-tests:
-    name: ocs-ci-container
-    namespace: ci
-    tag: stable
   tests-private:
     name: tests-private
     namespace: ci
@@ -89,8 +85,6 @@ tests:
       FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323
       FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP
       FIREWATCH_FAIL_WITH_TEST_FAILURES: "true"
-      MAP_TESTS: "true"
-      ODF_VERSION_MAJOR_MINOR: "5.0"
       OPERATORS: |
         [
           {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}
@@ -148,16 +142,12 @@ tests:
     cluster_profile: vsphere-connected-2
     env:
       COMPUTE_NODE_REPLICAS: "6"
-      DISABLE_ENVIRONMENT_CHECKER: "true"
-      DR__RP__CR_COMP_NAME: lp-interop--OPP
       FIREWATCH_CONFIG_FILE_PATH: https://raw.githubusercontent.com/CSPI-QE/cspi-utils/refs/heads/main/firewatch-base-configs/opp/lp-interop-vsphere.json
       FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS: '["5.0-lp","opp-vsphere-lp","opp-lp"]'
       FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com
       FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323
       FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP
       FIREWATCH_FAIL_WITH_TEST_FAILURES: "true"
-      MAP_TESTS: "true"
-      ODF_VERSION_MAJOR_MINOR: "5.0"
       OPENSHIFT_REQUIRED_CORES: "72"
       OPENSHIFT_REQUIRED_MEMORY: "288"
       OPERATORS: |
diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
index 95bcf854a6306..5a8b1cc0d9d2c 100755
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
@@ -43,6 +43,7 @@ XmlEscape() {
     text="${text//\"/&quot;}"
     text="${text//\'/&apos;}"
     printf '%s' "${text}"
+    true
 }
 
 WriteJunit() {
@@ -116,6 +117,7 @@ CheckOdfCsv() {
     else
         AddResult "odf-csv-phase" "fail" "ODF CSV phase=${csvPhase} (expected Succeeded)"
     fi
+    true
 }
 
 # ---------------------------------------------------------------------------
@@ -141,6 +143,7 @@ CheckStorageCluster() {
     else
         AddResult "storagecluster-ready" "fail" "StorageCluster phase=${scPhase} (expected Ready)"
     fi
+    true
 }
 
 # ---------------------------------------------------------------------------
@@ -166,6 +169,7 @@ CheckCephCluster() {
     else
         AddResult "cephcluster-health" "fail" "CephCluster health=${cephHealth} (expected HEALTH_OK or HEALTH_WARN)"
     fi
+    true
 }
 
 # ---------------------------------------------------------------------------
@@ -193,6 +197,7 @@ CheckStorageClasses() {
     else
         AddResult "storageclasses-available" "fail" "${failMsg}"
     fi
+    true
 }
 
 # ---------------------------------------------------------------------------
@@ -245,6 +250,7 @@ EOF
     else
         AddResult "pvc-provision" "fail" "PVC did not bind within ${maxWait}s (phase=${phase:-unknown})"
     fi
+    true
 }
 
 # ---------------------------------------------------------------------------
@@ -288,52 +294,23 @@ CheckNoobaa() {
         .items[0].status.accounts.admin.secretRef.name // "noobaa-admin"
     ')" || true
 
+    set +x
     if ! accessKey="$(oc get secret "${secretName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)"; then
+        set -x
         AddResult "noobaa-s3-functional" "fail" "Cannot read NooBaa admin credentials"
         return
     fi
     if ! secretKey="$(oc get secret "${secretName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)"; then
+        set -x
         AddResult "noobaa-s3-functional" "fail" "Cannot read NooBaa admin credentials"
         return
     fi
 
-    typeset bucket="odf-health-check-$$"
     typeset testKey="health-check-test"
-    typeset testData="odf-health-$(date +%s)"
+    typeset testData=""
+    testData="odf-health-$(date +%s)"
 
-    typeset -i s3ok=1
-
-    # Use a temporary pod to run the S3 operations from inside the cluster
     typeset podName="odf-health-s3-check-$$"
-    typeset s3Script
-    s3Script=$(cat <<'EOSCRIPT'
-#!/bin/sh
-set -e
-export AWS_ACCESS_KEY_ID="$1"
-export AWS_SECRET_ACCESS_KEY="$2"
-S3_ENDPOINT="$3"
-BUCKET="$4"
-TEST_KEY="$5"
-TEST_DATA="$6"
-
-alias s3="aws --endpoint-url ${S3_ENDPOINT} --no-verify-ssl s3"
-alias s3api="aws --endpoint-url ${S3_ENDPOINT} --no-verify-ssl s3api"
-
-# Put
-echo "${TEST_DATA}" | aws --endpoint-url "${S3_ENDPOINT}" --no-verify-ssl s3 cp - "s3://first.bucket/${TEST_KEY}" 2>/dev/null
-# Get
-RETRIEVED=$(aws --endpoint-url "${S3_ENDPOINT}" --no-verify-ssl s3 cp "s3://first.bucket/${TEST_KEY}" - 2>/dev/null)
-# Delete
-aws --endpoint-url "${S3_ENDPOINT}" --no-verify-ssl s3 rm "s3://first.bucket/${TEST_KEY}" 2>/dev/null
-# Verify
-if [ "${RETRIEVED}" = "${TEST_DATA}" ]; then
-    echo "S3_CHECK_PASS"
-else
-    echo "S3_CHECK_FAIL: data mismatch"
-fi
-EOSCRIPT
-)
-
     typeset s3Result=""
     if s3Result="$(oc run "${podName}" -n "${ODF_NAMESPACE}" \
         --image=amazon/aws-cli:latest \
@@ -357,6 +334,7 @@ EOF
     else
         : "S3 check pod failed or timed out"
     fi
+    set -x
 
     oc delete pod "${podName}" -n "${ODF_NAMESPACE}" --ignore-not-found=true --wait=false 2>/dev/null || true
 
@@ -370,6 +348,7 @@ EOF
         fi
         AddResult "noobaa-s3-functional" "fail" "${s3Msg}"
     fi
+    true
 }
 
 # ---------------------------------------------------------------------------
@@ -403,6 +382,7 @@ CheckCephHealth() {
     else
         AddResult "ceph-health-detail" "fail" "Ceph health=${cephHealth}: ${cephDetail}"
     fi
+    true
 }
 
 # ---------------------------------------------------------------------------
diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.metadata.json b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.metadata.json
index f3a48106962b9..4db42591de65c 100644
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.metadata.json
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.metadata.json
@@ -8,4 +8,4 @@
 			"cspi-qe-ocp-lp"
 		]
 	}
-}
+}
\ No newline at end of file
diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml
index 21db18ae818fb..c2f77a06cf50c 100644
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml
@@ -3,7 +3,7 @@ ref:
   from: cli
   grace_period: 30s
   commands: interop-opp-odf-health-commands.sh
-  timeout: 5m
+  timeout: 8m
   resources:
     requests:
       cpu: 100m

From e5f280c697834f423949bc3ac80556e7c46ccc5f Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Thu, 6 Aug 2026 12:28:31 -0500
Subject: [PATCH 03/10] fix: use function keyword for mpitt best practices

All function declarations now use `function FuncName () {` form
per MPEX Integrity Engineering Best Practices.
---
 .../interop-opp-odf-health-commands.sh        | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
index 5a8b1cc0d9d2c..d17a518e2fc99 100755
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
@@ -25,7 +25,7 @@ typeset -a tcMessagesArr=()
 # Helpers
 # ---------------------------------------------------------------------------
 
-AddResult() {
+function AddResult () {
     typeset name="${1:-}"; (($#)) && shift
     typeset result="${1:-}"; (($#)) && shift
     typeset message="${1:-}"; (($#)) && shift
@@ -35,7 +35,7 @@ AddResult() {
     true
 }
 
-XmlEscape() {
+function XmlEscape () {
     typeset text="${1:-}"; (($#)) && shift
     text="${text//&/&amp;}"
     text="${text//</&lt;}"
@@ -46,7 +46,7 @@ XmlEscape() {
     true
 }
 
-WriteJunit() {
+function WriteJunit () {
     typeset -i total=${#tcNamesArr[@]}
     typeset -i failCount=0
     typeset -i skipCount=0
@@ -82,7 +82,7 @@ WriteJunit() {
 }
 
 # shellcheck disable=SC2317
-CollectExitArtifacts() {
+function CollectExitArtifacts () {
     : "Collecting ODF diagnostics..."
     oc get csv -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/odf-csvs.yaml" || true
     oc get storagecluster -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/storagecluster.yaml" || true
@@ -97,7 +97,7 @@ trap CollectExitArtifacts EXIT
 # Check 1: ODF Operator CSV in Succeeded phase
 # ---------------------------------------------------------------------------
 
-CheckOdfCsv() {
+function CheckOdfCsv () {
     : "=== Check 1: ODF Operator CSV ==="
 
     typeset csvPhase=""
@@ -124,7 +124,7 @@ CheckOdfCsv() {
 # Check 2: StorageCluster phase == Ready
 # ---------------------------------------------------------------------------
 
-CheckStorageCluster() {
+function CheckStorageCluster () {
     : "=== Check 2: StorageCluster Ready ==="
 
     typeset scPhase=""
@@ -150,7 +150,7 @@ CheckStorageCluster() {
 # Check 3: CephCluster health == HEALTH_OK or HEALTH_WARN
 # ---------------------------------------------------------------------------
 
-CheckCephCluster() {
+function CheckCephCluster () {
     : "=== Check 3: CephCluster health ==="
 
     typeset cephHealth=""
@@ -176,7 +176,7 @@ CheckCephCluster() {
 # Check 4: Default StorageClasses available (ceph-rbd, cephfs)
 # ---------------------------------------------------------------------------
 
-CheckStorageClasses() {
+function CheckStorageClasses () {
     : "=== Check 4: StorageClasses ==="
     typeset failMsg=""
 
@@ -204,7 +204,7 @@ CheckStorageClasses() {
 # Check 5: PVC provisionable (create, bind, delete)
 # ---------------------------------------------------------------------------
 
-CheckPvcProvision() {
+function CheckPvcProvision () {
     : "=== Check 5: PVC provisioning ==="
 
     typeset pvcName="odf-health-check-pvc-$$"
@@ -257,7 +257,7 @@ EOF
 # Check 6: NooBaa system Ready + S3 put/get/delete functional check
 # ---------------------------------------------------------------------------
 
-CheckNoobaa() {
+function CheckNoobaa () {
     : "=== Check 6: NooBaa S3 functional ==="
 
     typeset nbPhase=""
@@ -355,7 +355,7 @@ EOF
 # Check 7: Ceph overall health (via toolbox or CephCluster status)
 # ---------------------------------------------------------------------------
 
-CheckCephHealth() {
+function CheckCephHealth () {
     : "=== Check 7: Ceph health detail ==="
 
     typeset cephDetail=""
@@ -389,7 +389,7 @@ CheckCephHealth() {
 # Main
 # ---------------------------------------------------------------------------
 
-Main() {
+function Main () {
     if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then
         export KUBECONFIG="${SHARED_DIR}/kubeconfig"
     fi

From 5dfc8d783e5ce13daf728cbea85d856560cc9be1 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Thu, 6 Aug 2026 12:49:45 -0500
Subject: [PATCH 04/10] fix: inject NooBaa credentials via secretRef, pin
 aws-cli image

Address CodeRabbit security finding: credentials were interpolated
into the pod command spec. Now uses envFrom with the existing NooBaa
admin secret. Pins aws-cli to 2.22.35 instead of :latest.
---
 .../interop-opp-odf-health-commands.sh        | 75 ++++++++++---------
 1 file changed, 40 insertions(+), 35 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
index d17a518e2fc99..511fed663fa6f 100755
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
@@ -280,7 +280,7 @@ function CheckNoobaa () {
 
     : "NooBaa phase=Ready, running S3 functional check..."
 
-    typeset s3Endpoint="" accessKey="" secretKey=""
+    typeset s3Endpoint=""
     s3Endpoint="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | jq -r '
         .items[0].status.services.serviceS3.internalDNS[0] // empty
     ')" || true
@@ -294,47 +294,52 @@ function CheckNoobaa () {
         .items[0].status.accounts.admin.secretRef.name // "noobaa-admin"
     ')" || true
 
-    set +x
-    if ! accessKey="$(oc get secret "${secretName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)"; then
-        set -x
-        AddResult "noobaa-s3-functional" "fail" "Cannot read NooBaa admin credentials"
-        return
-    fi
-    if ! secretKey="$(oc get secret "${secretName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)"; then
-        set -x
-        AddResult "noobaa-s3-functional" "fail" "Cannot read NooBaa admin credentials"
-        return
-    fi
-
     typeset testKey="health-check-test"
     typeset testData=""
     testData="odf-health-$(date +%s)"
 
     typeset podName="odf-health-s3-check-$$"
-    typeset s3Result=""
-    if s3Result="$(oc run "${podName}" -n "${ODF_NAMESPACE}" \
-        --image=amazon/aws-cli:latest \
-        --restart=Never \
-        --rm=true \
-        --attach=true \
-        --timeout="${NOOBAA_S3_TIMEOUT}s" \
-        --command -- sh -c "$(cat <<EOF
-export AWS_ACCESS_KEY_ID='${accessKey}'
-export AWS_SECRET_ACCESS_KEY='${secretKey}'
-S3_ENDPOINT='${s3Endpoint}'
-TEST_KEY='${testKey}'
-TEST_DATA='${testData}'
-echo "\${TEST_DATA}" | aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 cp - "s3://first.bucket/\${TEST_KEY}" 2>/dev/null && \
-RETRIEVED=\$(aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 cp "s3://first.bucket/\${TEST_KEY}" - 2>/dev/null) && \
-aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 rm "s3://first.bucket/\${TEST_KEY}" 2>/dev/null && \
-if [ "\${RETRIEVED}" = "\${TEST_DATA}" ]; then echo "S3_CHECK_PASS"; else echo "S3_CHECK_FAIL: data mismatch"; fi
+    typeset podManifest=""
+    podManifest=$(cat <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ${podName}
+  namespace: ${ODF_NAMESPACE}
+spec:
+  restartPolicy: Never
+  containers:
+  - name: s3check
+    image: amazon/aws-cli:2.22.35
+    envFrom:
+    - secretRef:
+        name: ${secretName}
+    env:
+    - name: S3_ENDPOINT
+      value: "${s3Endpoint}"
+    - name: TEST_KEY
+      value: "${testKey}"
+    - name: TEST_DATA
+      value: "${testData}"
+    command:
+    - sh
+    - -c
+    - |
+      echo "\${TEST_DATA}" | aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 cp - "s3://first.bucket/\${TEST_KEY}" 2>/dev/null && \
+      RETRIEVED=\$(aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 cp "s3://first.bucket/\${TEST_KEY}" - 2>/dev/null) && \
+      aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 rm "s3://first.bucket/\${TEST_KEY}" 2>/dev/null && \
+      if [ "\${RETRIEVED}" = "\${TEST_DATA}" ]; then echo "S3_CHECK_PASS"; else echo "S3_CHECK_FAIL: data mismatch"; fi
+  activeDeadlineSeconds: ${NOOBAA_S3_TIMEOUT}
 EOF
-)" 2>/dev/null)"; then
-        : "S3 check pod completed"
-    else
-        : "S3 check pod failed or timed out"
+)
+
+    typeset s3Result=""
+    if echo "${podManifest}" | oc apply -f - 2>/dev/null; then
+        if ! oc wait pod "${podName}" -n "${ODF_NAMESPACE}" --for=condition=Ready --timeout="${NOOBAA_S3_TIMEOUT}s" 2>/dev/null; then
+            : "Pod did not reach Ready, checking logs anyway"
+        fi
+        s3Result="$(oc logs "${podName}" -n "${ODF_NAMESPACE}" 2>/dev/null || echo "")"
     fi
-    set -x
 
     oc delete pod "${podName}" -n "${ODF_NAMESPACE}" --ignore-not-found=true --wait=false 2>/dev/null || true
 

From 1a5c80561a2dda099112298c8f125e0f2166f83a Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Thu, 6 Aug 2026 13:12:21 -0500
Subject: [PATCH 05/10] fix: add CephFS PVC test, switch NooBaa to OBC-based
 pattern

Check 5 now validates both ceph-rbd and cephfs provisioning.
Check 6 creates an ObjectBucketClaim instead of using the legacy
first.bucket, avoiding future breakage and key collisions.
---
 .../interop-opp-odf-health-commands.sh        | 130 ++++++++++++------
 1 file changed, 90 insertions(+), 40 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
index 511fed663fa6f..15a3ba7d548b7 100755
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
@@ -205,11 +205,18 @@ function CheckStorageClasses () {
 # ---------------------------------------------------------------------------
 
 function CheckPvcProvision () {
-    : "=== Check 5: PVC provisioning ==="
-
-    typeset pvcName="odf-health-check-pvc-$$"
-    typeset pvcYaml
-    pvcYaml=$(cat <<EOF
+    : "=== Check 5: PVC provisioning (RBD + CephFS) ==="
+
+    typeset -a scTests=("ocs-storagecluster-ceph-rbd" "ocs-storagecluster-cephfs")
+    typeset -a scModes=("ReadWriteOnce" "ReadWriteMany")
+
+    for idx in "${!scTests[@]}"; do
+        typeset scName="${scTests[$idx]}"
+        typeset accessMode="${scModes[$idx]}"
+        typeset testId="pvc-provision-${scName##*-}"
+        typeset pvcName="odf-health-${scName##*-}-$$"
+        typeset pvcYaml
+        pvcYaml=$(cat <<EOF
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -217,39 +224,40 @@ metadata:
   namespace: ${ODF_NAMESPACE}
 spec:
   accessModes:
-    - ReadWriteOnce
+    - ${accessMode}
   resources:
     requests:
       storage: 1Gi
-  storageClassName: ocs-storagecluster-ceph-rbd
+  storageClassName: ${scName}
 EOF
 )
 
-    if ! echo "${pvcYaml}" | oc apply -f - 2>/dev/null; then
-        AddResult "pvc-provision" "fail" "Failed to create test PVC"
-        return
-    fi
+        if ! echo "${pvcYaml}" | oc apply -f - 2>/dev/null; then
+            AddResult "${testId}" "fail" "Failed to create test PVC for ${scName}"
+            continue
+        fi
+
+        typeset -i maxWait=60
+        typeset -i elapsed=0
+        typeset phase=""
+        while (( elapsed < maxWait )); do
+            phase="$(oc get pvc "${pvcName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.status.phase}' 2>/dev/null || echo "")"
+            if [[ "${phase}" == "Bound" ]]; then
+                break
+            fi
+            sleep 5
+            (( elapsed += 5 )) || true
+        done
+
+        oc delete pvc "${pvcName}" -n "${ODF_NAMESPACE}" --wait=false 2>/dev/null || true
 
-    typeset -i maxWait=60
-    typeset -i elapsed=0
-    typeset phase=""
-    while (( elapsed < maxWait )); do
-        phase="$(oc get pvc "${pvcName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.status.phase}' 2>/dev/null || echo "")"
         if [[ "${phase}" == "Bound" ]]; then
-            break
+            : "PASS: ${scName} PVC bound in ${elapsed}s"
+            AddResult "${testId}" "pass"
+        else
+            AddResult "${testId}" "fail" "${scName} PVC did not bind within ${maxWait}s (phase=${phase:-unknown})"
         fi
-        sleep 5
-        (( elapsed += 5 )) || true
     done
-
-    oc delete pvc "${pvcName}" -n "${ODF_NAMESPACE}" --wait=false 2>/dev/null || true
-
-    if [[ "${phase}" == "Bound" ]]; then
-        : "PASS: PVC provisioned and bound in ${elapsed}s"
-        AddResult "pvc-provision" "pass"
-    else
-        AddResult "pvc-provision" "fail" "PVC did not bind within ${maxWait}s (phase=${phase:-unknown})"
-    fi
     true
 }
 
@@ -278,23 +286,62 @@ function CheckNoobaa () {
         return
     fi
 
-    : "NooBaa phase=Ready, running S3 functional check..."
+    : "NooBaa phase=Ready, creating OBC for S3 functional check..."
+
+    typeset obcName="odf-health-obc-$$"
+    typeset obcYaml
+    obcYaml=$(cat <<EOF
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: ${obcName}
+  namespace: ${ODF_NAMESPACE}
+spec:
+  generateBucketName: odf-health-check
+  storageClassName: openshift-storage.noobaa.io
+EOF
+)
+
+    if ! echo "${obcYaml}" | oc apply -f - 2>/dev/null; then
+        AddResult "noobaa-s3-functional" "fail" "Failed to create ObjectBucketClaim"
+        return
+    fi
+
+    typeset -i maxWait=60
+    typeset -i elapsed=0
+    typeset obcPhase=""
+    while (( elapsed < maxWait )); do
+        obcPhase="$(oc get obc "${obcName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.status.phase}' 2>/dev/null || echo "")"
+        if [[ "${obcPhase}" == "Bound" ]]; then
+            break
+        fi
+        sleep 5
+        (( elapsed += 5 )) || true
+    done
+
+    if [[ "${obcPhase}" != "Bound" ]]; then
+        oc delete obc "${obcName}" -n "${ODF_NAMESPACE}" --ignore-not-found=true --wait=false 2>/dev/null || true
+        AddResult "noobaa-s3-functional" "fail" "OBC did not bind within ${maxWait}s (phase=${obcPhase:-unknown})"
+        return
+    fi
+
+    typeset bucketName=""
+    bucketName="$(oc get obc "${obcName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.spec.bucketName}' 2>/dev/null)" || true
+    typeset secretRef=""
+    secretRef="$(oc get obc "${obcName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.spec.secretName}' 2>/dev/null)"
+    if [[ -z "${secretRef}" ]]; then
+        secretRef="${obcName}"
+    fi
 
     typeset s3Endpoint=""
     s3Endpoint="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | jq -r '
         .items[0].status.services.serviceS3.internalDNS[0] // empty
     ')" || true
-
     if [[ -z "${s3Endpoint}" ]]; then
         s3Endpoint="https://s3.${ODF_NAMESPACE}.svc:443"
     fi
 
-    typeset secretName=""
-    secretName="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | jq -r '
-        .items[0].status.accounts.admin.secretRef.name // "noobaa-admin"
-    ')" || true
-
-    typeset testKey="health-check-test"
+    typeset testKey="health-check-$$"
     typeset testData=""
     testData="odf-health-$(date +%s)"
 
@@ -313,10 +360,12 @@ spec:
     image: amazon/aws-cli:2.22.35
     envFrom:
     - secretRef:
-        name: ${secretName}
+        name: ${secretRef}
     env:
     - name: S3_ENDPOINT
       value: "${s3Endpoint}"
+    - name: BUCKET_NAME
+      value: "${bucketName}"
     - name: TEST_KEY
       value: "${testKey}"
     - name: TEST_DATA
@@ -325,9 +374,9 @@ spec:
     - sh
     - -c
     - |
-      echo "\${TEST_DATA}" | aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 cp - "s3://first.bucket/\${TEST_KEY}" 2>/dev/null && \
-      RETRIEVED=\$(aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 cp "s3://first.bucket/\${TEST_KEY}" - 2>/dev/null) && \
-      aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 rm "s3://first.bucket/\${TEST_KEY}" 2>/dev/null && \
+      echo "\${TEST_DATA}" | aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 cp - "s3://\${BUCKET_NAME}/\${TEST_KEY}" 2>/dev/null && \
+      RETRIEVED=\$(aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 cp "s3://\${BUCKET_NAME}/\${TEST_KEY}" - 2>/dev/null) && \
+      aws --endpoint-url "\${S3_ENDPOINT}" --no-verify-ssl s3 rm "s3://\${BUCKET_NAME}/\${TEST_KEY}" 2>/dev/null && \
       if [ "\${RETRIEVED}" = "\${TEST_DATA}" ]; then echo "S3_CHECK_PASS"; else echo "S3_CHECK_FAIL: data mismatch"; fi
   activeDeadlineSeconds: ${NOOBAA_S3_TIMEOUT}
 EOF
@@ -342,6 +391,7 @@ EOF
     fi
 
     oc delete pod "${podName}" -n "${ODF_NAMESPACE}" --ignore-not-found=true --wait=false 2>/dev/null || true
+    oc delete obc "${obcName}" -n "${ODF_NAMESPACE}" --ignore-not-found=true --wait=false 2>/dev/null || true
 
     if echo "${s3Result}" | grep -q "S3_CHECK_PASS"; then
         : "PASS: NooBaa S3 put/get/delete cycle succeeded"

From 4cddd8d001826b708b7a9e12bb4ff372512dd2e3 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Fri, 7 Aug 2026 11:19:39 -0500
Subject: [PATCH 06/10] Polish ODF health check shell conventions

- Use `set -eux -o pipefail` (separated -o flag)
- Replace &>/dev/null with `-o name 2>/dev/null` on sc existence check
- Remove stderr suppression on oc apply/wait where error context aids debugging
---
 .../odf-health/interop-opp-odf-health-commands.sh    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
index 15a3ba7d548b7..abf119a200f17 100755
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -euxo pipefail
+set -eux -o pipefail
 shopt -s inherit_errexit
 
 # ---------------------------------------------------------------------------
@@ -181,7 +181,7 @@ function CheckStorageClasses () {
     typeset failMsg=""
 
     for scName in ocs-storagecluster-ceph-rbd ocs-storagecluster-cephfs; do
-        if ! oc get sc "${scName}" &>/dev/null; then
+        if ! oc get sc "${scName}" -o name 2>/dev/null; then
             if [[ -n "${failMsg}" ]]; then
                 failMsg="${failMsg}; StorageClass ${scName} not found"
             else
@@ -232,7 +232,7 @@ spec:
 EOF
 )
 
-        if ! echo "${pvcYaml}" | oc apply -f - 2>/dev/null; then
+        if ! echo "${pvcYaml}" | oc apply -f -; then
             AddResult "${testId}" "fail" "Failed to create test PVC for ${scName}"
             continue
         fi
@@ -302,7 +302,7 @@ spec:
 EOF
 )
 
-    if ! echo "${obcYaml}" | oc apply -f - 2>/dev/null; then
+    if ! echo "${obcYaml}" | oc apply -f -; then
         AddResult "noobaa-s3-functional" "fail" "Failed to create ObjectBucketClaim"
         return
     fi
@@ -383,8 +383,8 @@ EOF
 )
 
     typeset s3Result=""
-    if echo "${podManifest}" | oc apply -f - 2>/dev/null; then
-        if ! oc wait pod "${podName}" -n "${ODF_NAMESPACE}" --for=condition=Ready --timeout="${NOOBAA_S3_TIMEOUT}s" 2>/dev/null; then
+    if echo "${podManifest}" | oc apply -f -; then
+        if ! oc wait pod "${podName}" -n "${ODF_NAMESPACE}" --for=condition=Ready --timeout="${NOOBAA_S3_TIMEOUT}s"; then
             : "Pod did not reach Ready, checking logs anyway"
         fi
         s3Result="$(oc logs "${podName}" -n "${ODF_NAMESPACE}" 2>/dev/null || echo "")"

From a03d8ff9d78a2806d23c2fed201ca387636ceb03 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Fri, 7 Aug 2026 12:17:52 -0500
Subject: [PATCH 07/10] fix: fail early on empty bucket name and wait for pod
 completion

Address CodeRabbit review findings:
- Guard against empty OBC bucket name before launching S3 pod
- Wait for pod phase=Succeeded instead of condition=Ready
- Add 60s image pull budget on top of S3 timeout
---
 .../odf-health/interop-opp-odf-health-commands.sh    | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
index abf119a200f17..4d92338d19933 100755
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
@@ -327,6 +327,11 @@ EOF
 
     typeset bucketName=""
     bucketName="$(oc get obc "${obcName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.spec.bucketName}' 2>/dev/null)" || true
+    if [[ -z "${bucketName}" ]]; then
+        oc delete obc "${obcName}" -n "${ODF_NAMESPACE}" --ignore-not-found=true --wait=false 2>/dev/null || true
+        AddResult "noobaa-s3-functional" "fail" "OBC bound but bucket name is empty"
+        return
+    fi
     typeset secretRef=""
     secretRef="$(oc get obc "${obcName}" -n "${ODF_NAMESPACE}" -o jsonpath='{.spec.secretName}' 2>/dev/null)"
     if [[ -z "${secretRef}" ]]; then
@@ -383,9 +388,12 @@ EOF
 )
 
     typeset s3Result=""
+    typeset -i podWait=$(( NOOBAA_S3_TIMEOUT + 60 ))
     if echo "${podManifest}" | oc apply -f -; then
-        if ! oc wait pod "${podName}" -n "${ODF_NAMESPACE}" --for=condition=Ready --timeout="${NOOBAA_S3_TIMEOUT}s"; then
-            : "Pod did not reach Ready, checking logs anyway"
+        if ! oc wait pod "${podName}" -n "${ODF_NAMESPACE}" \
+            --for=jsonpath='{.status.phase}'=Succeeded \
+            --timeout="${podWait}s" 2>/dev/null; then
+            : "Pod did not succeed within ${podWait}s, checking logs anyway"
         fi
         s3Result="$(oc logs "${podName}" -n "${ODF_NAMESPACE}" 2>/dev/null || echo "")"
     fi

From 84f2cde22e91dc341242c6b7c34a9613961a730c Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Fri, 7 Aug 2026 12:37:00 -0500
Subject: [PATCH 08/10] fix: apply mpitt and Chai Bot review findings

- Rename JUnit testsuite to lp-interop--ODF for Sippy allowlist
- Declare all loop iterator variables with typeset (mpitt)
- Add env declarations to ref.yaml for discoverability (mpitt)
- Add securityContext to S3 check pod (runAsNonRoot, no escalation)
---
 .../odf-health/interop-opp-odf-health-commands.sh  | 14 ++++++++++++--
 .../opp/odf-health/interop-opp-odf-health-ref.yaml |  7 +++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
index 4d92338d19933..eca36a74c454d 100755
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
@@ -50,6 +50,7 @@ function WriteJunit () {
     typeset -i total=${#tcNamesArr[@]}
     typeset -i failCount=0
     typeset -i skipCount=0
+    typeset r=""
     for r in "${tcResultsArr[@]}"; do
         if [[ "${r}" == "fail" ]]; then
             (( failCount++ )) || true
@@ -60,11 +61,12 @@ function WriteJunit () {
 
     {
         echo '<?xml version="1.0" encoding="UTF-8"?>'
-        echo "<testsuite name=\"odf-health\" tests=\"${total}\" failures=\"${failCount}\" skipped=\"${skipCount}\">"
+        echo "<testsuite name=\"lp-interop--ODF\" tests=\"${total}\" failures=\"${failCount}\" skipped=\"${skipCount}\">"
+        typeset -i i=0
         for i in "${!tcNamesArr[@]}"; do
             typeset name=""
             name="$(XmlEscape "${tcNamesArr[$i]}")"
-            echo "  <testcase classname=\"odf-health\" name=\"${name}\">"
+            echo "  <testcase classname=\"lp-interop--ODF\" name=\"${name}\">"
             if [[ "${tcResultsArr[$i]}" == "fail" ]]; then
                 typeset msg=""
                 msg="$(XmlEscape "${tcMessagesArr[$i]}")"
@@ -179,6 +181,7 @@ function CheckCephCluster () {
 function CheckStorageClasses () {
     : "=== Check 4: StorageClasses ==="
     typeset failMsg=""
+    typeset scName=""
 
     for scName in ocs-storagecluster-ceph-rbd ocs-storagecluster-cephfs; do
         if ! oc get sc "${scName}" -o name 2>/dev/null; then
@@ -209,6 +212,7 @@ function CheckPvcProvision () {
 
     typeset -a scTests=("ocs-storagecluster-ceph-rbd" "ocs-storagecluster-cephfs")
     typeset -a scModes=("ReadWriteOnce" "ReadWriteMany")
+    typeset -i idx=0
 
     for idx in "${!scTests[@]}"; do
         typeset scName="${scTests[$idx]}"
@@ -360,9 +364,14 @@ metadata:
   namespace: ${ODF_NAMESPACE}
 spec:
   restartPolicy: Never
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
   containers:
   - name: s3check
     image: amazon/aws-cli:2.22.35
+    securityContext:
+      allowPrivilegeEscalation: false
     envFrom:
     - secretRef:
         name: ${secretRef}
@@ -472,6 +481,7 @@ function Main () {
     WriteJunit
 
     typeset -i hasAnyFail=0
+    typeset r=""
     for r in "${tcResultsArr[@]}"; do
         if [[ "${r}" == "fail" ]]; then
             hasAnyFail=1
diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml
index c2f77a06cf50c..9f8c8a5523239 100644
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-ref.yaml
@@ -4,6 +4,13 @@ ref:
   grace_period: 30s
   commands: interop-opp-odf-health-commands.sh
   timeout: 8m
+  env:
+  - name: ODF_NAMESPACE
+    default: "openshift-storage"
+    documentation: Namespace where ODF is installed
+  - name: NOOBAA_S3_TIMEOUT
+    default: "30"
+    documentation: Timeout in seconds for the NooBaa S3 functional check pod
   resources:
     requests:
       cpu: 100m

From 9f5233b467b3ff561163a464feac4405632655cd Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Fri, 14 Aug 2026 07:30:16 -0500
Subject: [PATCH 09/10] Address review feedback: security hardening

- Remove xtrace (-x) to avoid logging S3 endpoint in CI output
- Add readOnlyRootFilesystem, drop ALL capabilities, resource limits,
  and emptyDir volume mount to the s3check Pod manifest
---
 .../interop-opp-odf-health-commands.sh         | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
index eca36a74c454d..2452dfbde4211 100755
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -eux -o pipefail
+set -euo pipefail
 shopt -s inherit_errexit
 
 # ---------------------------------------------------------------------------
@@ -367,11 +367,27 @@ spec:
   securityContext:
     runAsNonRoot: true
     runAsUser: 65534
+  volumes:
+  - name: tmp
+    emptyDir: {}
   containers:
   - name: s3check
     image: amazon/aws-cli:2.22.35
     securityContext:
       allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop: ["ALL"]
+    resources:
+      limits:
+        cpu: 200m
+        memory: 256Mi
+      requests:
+        cpu: 100m
+        memory: 128Mi
+    volumeMounts:
+    - name: tmp
+      mountPath: /tmp
     envFrom:
     - secretRef:
         name: ${secretRef}

From c59faf8cba4914c316cd7a459c059eda5d5c342f Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Mon, 17 Aug 2026 12:31:28 -0500
Subject: [PATCH 10/10] fix: Replace jq with python3 and fix $$ expansion in
 ODF health check

- jq not available in CI container image; replaced all 7 jq calls
  with python3 -c equivalents using json module
- $$ doesn't reliably expand to PID in CI containers; replaced with
  $RANDOM for unique resource names
---
 .../interop-opp-odf-health-commands.sh        | 64 +++++++++++--------
 1 file changed, 36 insertions(+), 28 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
index 2452dfbde4211..156a7f9b28caa 100755
--- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
+++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh
@@ -103,10 +103,11 @@ function CheckOdfCsv () {
     : "=== Check 1: ODF Operator CSV ==="
 
     typeset csvPhase=""
-    if ! csvPhase="$(oc get csv -n "${ODF_NAMESPACE}" -o json | jq -r '
-        [.items[] | select(.metadata.name | test("^(odf-|ocs-)operator"))] |
-        first | .status.phase // "NotFound"
-    ')"; then
+    if ! csvPhase="$(oc get csv -n "${ODF_NAMESPACE}" -o json | python3 -c "
+import sys,json,re; d=json.load(sys.stdin)
+m=[i for i in d.get('items',[]) if re.match(r'^(odf-|ocs-)operator',i['metadata']['name'])]
+print((m[0].get('status',{}).get('phase','NotFound')) if m else 'NotFound')
+")"; then
         AddResult "odf-csv-phase" "fail" "Failed to query ODF CSVs in ${ODF_NAMESPACE}"
         return
     fi
@@ -130,9 +131,10 @@ function CheckStorageCluster () {
     : "=== Check 2: StorageCluster Ready ==="
 
     typeset scPhase=""
-    if ! scPhase="$(oc get storagecluster -n "${ODF_NAMESPACE}" -o json | jq -r '
-        .items[0].status.phase // "NotFound"
-    ')"; then
+    if ! scPhase="$(oc get storagecluster -n "${ODF_NAMESPACE}" -o json | python3 -c "
+import sys,json; d=json.load(sys.stdin)
+print(d['items'][0]['status'].get('phase','NotFound') if d.get('items') else 'NotFound')
+")"; then
         AddResult "storagecluster-ready" "fail" "Failed to query StorageCluster"
         return
     fi
@@ -156,9 +158,10 @@ function CheckCephCluster () {
     : "=== Check 3: CephCluster health ==="
 
     typeset cephHealth=""
-    if ! cephHealth="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | jq -r '
-        .items[0].status.ceph.health // "NotFound"
-    ')"; then
+    if ! cephHealth="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | python3 -c "
+import sys,json; d=json.load(sys.stdin)
+print(d['items'][0].get('status',{}).get('ceph',{}).get('health','NotFound') if d.get('items') else 'NotFound')
+")"; then
         AddResult "cephcluster-health" "fail" "Failed to query CephCluster"
         return
     fi
@@ -218,7 +221,7 @@ function CheckPvcProvision () {
         typeset scName="${scTests[$idx]}"
         typeset accessMode="${scModes[$idx]}"
         typeset testId="pvc-provision-${scName##*-}"
-        typeset pvcName="odf-health-${scName##*-}-$$"
+        typeset pvcName="odf-health-${scName##*-}-${RANDOM}"
         typeset pvcYaml
         pvcYaml=$(cat <<EOF
 apiVersion: v1
@@ -273,9 +276,10 @@ function CheckNoobaa () {
     : "=== Check 6: NooBaa S3 functional ==="
 
     typeset nbPhase=""
-    if ! nbPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | jq -r '
-        .items[0].status.phase // "NotFound"
-    ')"; then
+    if ! nbPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | python3 -c "
+import sys,json; d=json.load(sys.stdin)
+print(d['items'][0]['status'].get('phase','NotFound') if d.get('items') else 'NotFound')
+")"; then
         AddResult "noobaa-s3-functional" "fail" "Failed to query NooBaa"
         return
     fi
@@ -292,7 +296,7 @@ function CheckNoobaa () {
 
     : "NooBaa phase=Ready, creating OBC for S3 functional check..."
 
-    typeset obcName="odf-health-obc-$$"
+    typeset obcName="odf-health-obc-${RANDOM}"
     typeset obcYaml
     obcYaml=$(cat <<EOF
 apiVersion: objectbucket.io/v1alpha1
@@ -343,18 +347,20 @@ EOF
     fi
 
     typeset s3Endpoint=""
-    s3Endpoint="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | jq -r '
-        .items[0].status.services.serviceS3.internalDNS[0] // empty
-    ')" || true
+    s3Endpoint="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | python3 -c "
+import sys,json; d=json.load(sys.stdin)
+v=d['items'][0].get('status',{}).get('services',{}).get('serviceS3',{}).get('internalDNS',[])
+print(v[0] if v else '')
+")" || true
     if [[ -z "${s3Endpoint}" ]]; then
         s3Endpoint="https://s3.${ODF_NAMESPACE}.svc:443"
     fi
 
-    typeset testKey="health-check-$$"
+    typeset testKey="health-check-${RANDOM}"
     typeset testData=""
     testData="odf-health-$(date +%s)"
 
-    typeset podName="odf-health-s3-check-$$"
+    typeset podName="odf-health-s3-check-${RANDOM}"
     typeset podManifest=""
     podManifest=$(cat <<EOF
 apiVersion: v1
@@ -447,19 +453,21 @@ function CheckCephHealth () {
     : "=== Check 7: Ceph health detail ==="
 
     typeset cephDetail=""
-    if ! cephDetail="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | jq -r '
-        .items[0].status.ceph.details // {} | to_entries[] |
-        select(.value.severity != "HEALTH_OK") |
-        "\(.key): \(.value.message // "unknown")"
-    ')"; then
+    if ! cephDetail="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | python3 -c "
+import sys,json; d=json.load(sys.stdin)
+details=d['items'][0].get('status',{}).get('ceph',{}).get('details',{}) if d.get('items') else {}
+for k,v in details.items():
+ if v.get('severity')!='HEALTH_OK': print(f\"{k}: {v.get('message','unknown')}\")
+")"; then
         AddResult "ceph-health-detail" "fail" "Failed to query CephCluster details"
         return
     fi
 
     typeset cephHealth=""
-    cephHealth="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | jq -r '
-        .items[0].status.ceph.health // "unknown"
-    ')" || true
+    cephHealth="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | python3 -c "
+import sys,json; d=json.load(sys.stdin)
+print(d['items'][0].get('status',{}).get('ceph',{}).get('health','unknown') if d.get('items') else 'unknown')
+")" || true
 
     if [[ "${cephHealth}" == "HEALTH_OK" ]]; then
         : "PASS: Ceph health=HEALTH_OK"