From 44fc025b7b80f6f5803b7e63b2b4a0af01bc2f58 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Wed, 19 Aug 2026 09:53:33 -0500
Subject: [PATCH 01/10] INTEROP-9416: Add opp-observability-odf step for
 ACM+ODF interop validation

Validates the cross-product integration between ACM Observability (Thanos)
and ODF (Ceph RGW/NooBaa) object storage with a 6-point gate:

1. ODF Ceph RGW infrastructure ready
2. MultiClusterObservability CR exists and is Ready
3. Object storage secret references ODF-backed endpoint
4. Thanos components healthy (with missing-component detection)
5. ObjectBucketClaim bound
6. Thanos query endpoint functional (strict HTTP 200 only)

Addresses CodeRabbit review findings from v1:
- Do not leak decoded secret content into JUnit artifacts
- Detect missing Thanos components instead of silently passing
- Fix unreachable OBC fallback by checking parsed item count
- Only accept HTTP 200 for functional query check (not 401/403)
- Use only metricObjectStorage.name for secret lookup (not key)
---
 .../interop/opp/observability-odf/OWNERS      |   3 +
 .../interop-opp-observability-odf-commands.sh | 524 ++++++++++++++++++
 ...op-opp-observability-odf-ref.metadata.json |  11 +
 .../interop-opp-observability-odf-ref.yaml    |  36 ++
 4 files changed, 574 insertions(+)
 create mode 100644 ci-operator/step-registry/interop/opp/observability-odf/OWNERS
 create mode 100755 ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
 create mode 100644 ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json
 create mode 100644 ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml

diff --git a/ci-operator/step-registry/interop/opp/observability-odf/OWNERS b/ci-operator/step-registry/interop/opp/observability-odf/OWNERS
new file mode 100644
index 0000000000000..41d144d3728a2
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/observability-odf/OWNERS
@@ -0,0 +1,3 @@
+approvers: &owners
+- cspi-qe-ocp-lp
+reviewers: *owners
diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
new file mode 100755
index 0000000000000..3510619b51e83
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
@@ -0,0 +1,524 @@
+#!/bin/bash
+set -euo pipefail
+shopt -s inherit_errexit
+
+# ---------------------------------------------------------------------------
+# ACM Observability + ODF Interop Validation (6-point gate)
+#
+# Validates that ACM's observability stack (Thanos) correctly uses
+# ODF-provided object storage (Ceph RGW or NooBaa S3) as its backend.
+# This is a cross-product interop test exercising the ACM <-> ODF boundary.
+#
+# Produces JUnit XML consumed by Prow / Sippy / TestGrid.
+# ---------------------------------------------------------------------------
+
+ACM_NAMESPACE="${ACM_NAMESPACE:-open-cluster-management}"
+OBS_NAMESPACE="${OBS_NAMESPACE:-open-cluster-management-observability}"
+ODF_NAMESPACE="${ODF_NAMESPACE:-openshift-storage}"
+
+typeset junitFile="${ARTIFACT_DIR}/junit_observability_odf.xml"
+
+typeset -a tcNamesArr=()
+typeset -a tcResultsArr=()
+typeset -a tcMessagesArr=()
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+function AddResult () {
+    typeset name="${1:-}"; (($#)) && shift
+    typeset result="${1:-}"; (($#)) && shift
+    typeset message="${1:-}"; (($#)) && shift
+    tcNamesArr+=("${name}")
+    tcResultsArr+=("${result}")
+    tcMessagesArr+=("${message}")
+    true
+}
+
+function XmlEscape () {
+    typeset text="${1:-}"; (($#)) && shift
+    text="${text//&/&amp;}"
+    text="${text//</&lt;}"
+    text="${text//>/&gt;}"
+    text="${text//\"/&quot;}"
+    text="${text//\'/&apos;}"
+    printf '%s' "${text}"
+    true
+}
+
+function WriteJunit () {
+    typeset -i total=${#tcNamesArr[@]}
+    typeset -i failCount=0
+    typeset -i skipCount=0
+    typeset r=""
+    for r in "${tcResultsArr[@]}"; do
+        if [[ "${r}" == "fail" ]]; then
+            (( failCount++ )) || true
+        elif [[ "${r}" == "skip" ]]; then
+            (( skipCount++ )) || true
+        fi
+    done
+
+    {
+        echo '<?xml version="1.0" encoding="UTF-8"?>'
+        echo "<testsuite name=\"lp-interop--ACM-OBS-ODF\" tests=\"${total}\" failures=\"${failCount}\" skipped=\"${skipCount}\">"
+        typeset -i i=0
+        for i in "${!tcNamesArr[@]}"; do
+            typeset name=""
+            name="$(XmlEscape "${tcNamesArr[$i]}")"
+            echo "  <testcase classname=\"lp-interop--ACM-OBS-ODF\" name=\"${name}\">"
+            if [[ "${tcResultsArr[$i]}" == "fail" ]]; then
+                typeset msg=""
+                msg="$(XmlEscape "${tcMessagesArr[$i]}")"
+                echo "    <failure message=\"${msg}\"></failure>"
+            elif [[ "${tcResultsArr[$i]}" == "skip" ]]; then
+                typeset msg=""
+                msg="$(XmlEscape "${tcMessagesArr[$i]}")"
+                echo "    <skipped message=\"${msg}\"/>"
+            fi
+            echo "  </testcase>"
+        done
+        echo "</testsuite>"
+    } > "${junitFile}"
+    : "JUnit XML written to ${junitFile}"
+}
+
+# shellcheck disable=SC2317
+function CollectExitArtifacts () {
+    : "Collecting observability + ODF diagnostics..."
+    oc get multiclusterobservabilities.observability.open-cluster-management.io --all-namespaces -o yaml > "${ARTIFACT_DIR}/mco.yaml" 2>/dev/null || true
+    oc get pods -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-pods.yaml" 2>/dev/null || true
+    oc get obc -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-obc.yaml" 2>/dev/null || true
+    oc get secret -n "${OBS_NAMESPACE}" -o name > "${ARTIFACT_DIR}/obs-secrets-list.txt" 2>/dev/null || true
+    oc get cephobjectstore -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/cephobjectstore.yaml" 2>/dev/null || true
+    oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw -o yaml > "${ARTIFACT_DIR}/rgw-pods.yaml" 2>/dev/null || true
+    oc get noobaa -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/noobaa.yaml" 2>/dev/null || true
+}
+
+trap CollectExitArtifacts EXIT
+
+# ---------------------------------------------------------------------------
+# Check 1: ODF Ceph RGW infrastructure ready
+# ---------------------------------------------------------------------------
+
+function CheckRgwReady () {
+    : "=== Check 1: ODF Ceph RGW infrastructure ==="
+
+    typeset rgwPhase=""
+    if ! rgwPhase="$(oc get cephobjectstore -n "${ODF_NAMESPACE}" -o json 2>/dev/null | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+items=d.get('items',[])
+if not items:
+    print('NotFound')
+else:
+    print(items[0].get('status',{}).get('phase','Unknown'))
+")"; then
+        AddResult "rgw-ready" "fail" "Failed to query CephObjectStore"
+        return
+    fi
+
+    if [[ "${rgwPhase}" == "NotFound" ]]; then
+        AddResult "rgw-ready" "skip" "No CephObjectStore found; ODF RGW not deployed"
+        return
+    fi
+
+    typeset failMsg=""
+    if [[ "${rgwPhase}" != "Ready" ]]; then
+        failMsg="CephObjectStore phase=${rgwPhase} (expected Ready)"
+    fi
+
+    typeset rgwPodCount=""
+    rgwPodCount="$(oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw \
+        --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l)" || true
+
+    if [[ "${rgwPodCount}" -eq 0 ]]; then
+        typeset rgwMsg="No rook-ceph-rgw pods Running in ${ODF_NAMESPACE}"
+        if [[ -n "${failMsg}" ]]; then
+            failMsg="${failMsg}; ${rgwMsg}"
+        else
+            failMsg="${rgwMsg}"
+        fi
+    fi
+
+    typeset scExists=""
+    scExists="$(oc get sc ocs-storagecluster-ceph-rgw -o name 2>/dev/null)" || true
+    if [[ -z "${scExists}" ]]; then
+        typeset scMsg="StorageClass ocs-storagecluster-ceph-rgw not found"
+        if [[ -n "${failMsg}" ]]; then
+            failMsg="${failMsg}; ${scMsg}"
+        else
+            failMsg="${scMsg}"
+        fi
+    fi
+
+    if [[ -z "${failMsg}" ]]; then
+        : "PASS: CephObjectStore Ready, RGW pods Running, StorageClass exists"
+        AddResult "rgw-ready" "pass"
+    else
+        AddResult "rgw-ready" "fail" "${failMsg}"
+    fi
+    true
+}
+
+# ---------------------------------------------------------------------------
+# Check 2: MultiClusterObservability CR exists and is Ready
+# ---------------------------------------------------------------------------
+
+function CheckMcoReady () {
+    : "=== Check 2: MultiClusterObservability CR ==="
+
+    typeset mcoStatus=""
+    if ! mcoStatus="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \
+        --all-namespaces -o json 2>/dev/null | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+items=d.get('items',[])
+if not items:
+    print('NotFound')
+else:
+    conds=items[0].get('status',{}).get('conditions',[])
+    ready=[c for c in conds if c.get('type')=='Ready']
+    print(ready[0].get('status','Unknown') if ready else 'NoCondition')
+")"; then
+        AddResult "mco-ready" "fail" "Failed to query MultiClusterObservability CR"
+        return
+    fi
+
+    if [[ "${mcoStatus}" == "True" ]]; then
+        : "PASS: MultiClusterObservability Ready=True"
+        AddResult "mco-ready" "pass"
+    elif [[ "${mcoStatus}" == "NotFound" ]]; then
+        AddResult "mco-ready" "skip" "MultiClusterObservability CR not found; observability not deployed"
+    else
+        AddResult "mco-ready" "fail" "MultiClusterObservability Ready=${mcoStatus} (expected True)"
+    fi
+    true
+}
+
+# ---------------------------------------------------------------------------
+# Check 3: Object storage secret references ODF-backed endpoint
+# ---------------------------------------------------------------------------
+
+function CheckStorageEndpoint () {
+    : "=== Check 3: Object storage endpoint ==="
+
+    typeset secretName=""
+    if ! secretName="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \
+        --all-namespaces -o json 2>/dev/null | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+items=d.get('items',[])
+if not items:
+    print('')
+else:
+    spec=items[0].get('spec',{})
+    storage=spec.get('storageConfig',{}).get('metricObjectStorage',{})
+    print(storage.get('name',''))
+")"; then
+        AddResult "storage-endpoint" "fail" "Failed to read MCO storage config"
+        return
+    fi
+
+    if [[ -z "${secretName}" ]]; then
+        AddResult "storage-endpoint" "skip" "No metricObjectStorage secret configured in MCO"
+        return
+    fi
+
+    typeset endpointCheck=""
+    endpointCheck="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json 2>/dev/null \
+        | python3 -c "
+import sys,json,base64,re
+d=json.load(sys.stdin)
+odf_pattern=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs)', re.IGNORECASE)
+for k,v in d.get('data',{}).items():
+    decoded=base64.b64decode(v).decode('utf-8','replace')
+    if 'endpoint' in decoded.lower() or 'bucket' in decoded.lower():
+        if odf_pattern.search(decoded):
+            print('odf-backed')
+        else:
+            print('external')
+        sys.exit(0)
+print('no-endpoint')
+" 2>/dev/null)" || true
+
+    if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then
+        AddResult "storage-endpoint" "fail" "Secret ${secretName} exists but no endpoint/bucket config found"
+        return
+    fi
+
+    if [[ "${endpointCheck}" == "odf-backed" ]]; then
+        AddResult "storage-endpoint" "pass"
+    else
+        AddResult "storage-endpoint" "fail" "Storage endpoint does not reference ODF-backed service"
+    fi
+    true
+}
+
+# ---------------------------------------------------------------------------
+# Check 4: Thanos components healthy
+# ---------------------------------------------------------------------------
+
+function CheckThanosHealth () {
+    : "=== Check 4: Thanos components healthy ==="
+
+    if ! oc get namespace "${OBS_NAMESPACE}" &>/dev/null; then
+        AddResult "thanos-health" "skip" "Observability namespace ${OBS_NAMESPACE} does not exist"
+        return
+    fi
+
+    typeset failMsg=""
+    typeset -i foundCount=0
+    typeset -a missingComponents=()
+
+    typeset -a componentNames=("thanos-receive"     "thanos-compact"     "thanos-store"       "thanos-query"       "alertmanager"       "rbac-query-proxy")
+    typeset -a componentLabels=("app=thanos-receive" "app=thanos-compact" "app=thanos-store"   "app=thanos-query"   "alertmanager=observability" "app=rbac-query-proxy")
+
+    typeset -i idx=0
+    for idx in "${!componentNames[@]}"; do
+        typeset component="${componentNames[$idx]}"
+        typeset labelSelector="${componentLabels[$idx]}"
+
+        typeset podCount=""
+        podCount="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \
+            --no-headers 2>/dev/null | wc -l)" || true
+
+        if [[ "${podCount}" -eq 0 ]]; then
+            podCount="$(oc get pods -n "${OBS_NAMESPACE}" \
+                --no-headers 2>/dev/null | grep -c "^${component}")" || true
+        fi
+
+        if [[ "${podCount}" -eq 0 ]]; then
+            missingComponents+=("${component}")
+            continue
+        fi
+
+        (( foundCount++ )) || true
+
+        typeset notReady=""
+        notReady="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \
+            --no-headers 2>/dev/null \
+            | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')" || true
+
+        if [[ -n "${notReady}" ]]; then
+            typeset compMsg="${component}: ${notReady//$'\n'/, }"
+            if [[ -n "${failMsg}" ]]; then
+                failMsg="${failMsg}; ${compMsg}"
+            else
+                failMsg="${compMsg}"
+            fi
+        fi
+    done
+
+    if (( foundCount == 0 )); then
+        AddResult "thanos-health" "fail" "No Thanos/observability components found in ${OBS_NAMESPACE}"
+    elif [[ -n "${failMsg}" ]]; then
+        AddResult "thanos-health" "fail" "Unhealthy Thanos components: ${failMsg}"
+    elif (( ${#missingComponents[@]} > 0 )); then
+        AddResult "thanos-health" "pass" "Running (missing: ${missingComponents[*]})"
+    else
+        AddResult "thanos-health" "pass"
+    fi
+    true
+}
+
+# ---------------------------------------------------------------------------
+# Check 5: ObjectBucketClaim bound (if used by observability)
+# ---------------------------------------------------------------------------
+
+function CheckObcBound () {
+    : "=== Check 5: Observability ObjectBucketClaim ==="
+
+    typeset obcList=""
+    obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true
+
+    typeset obcItemCount=""
+    if [[ -n "${obcList}" ]]; then
+        obcItemCount="$(echo "${obcList}" | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+print(len(d.get('items',[])))
+" 2>/dev/null)" || true
+    fi
+
+    if [[ "${obcItemCount:-0}" -eq 0 ]]; then
+        obcList="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null \
+            | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()]
+print(json.dumps({'items':obs}))
+" 2>/dev/null)" || true
+        obcItemCount="$(echo "${obcList}" | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+print(len(d.get('items',[])))
+" 2>/dev/null)" || true
+    fi
+
+    if [[ "${obcItemCount:-0}" -eq 0 ]]; then
+        AddResult "obc-bound" "skip" "No ObjectBucketClaim found for observability"
+        return
+    fi
+
+    typeset obcStatus=""
+    if ! obcStatus="$(echo "${obcList}" | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+items=d.get('items',[])
+if not items:
+    print('NotFound')
+else:
+    results=[]
+    for i in items:
+        name=i['metadata']['name']
+        phase=i.get('status',{}).get('phase','Unknown')
+        results.append(f'{name}={phase}')
+    print(';'.join(results))
+")"; then
+        AddResult "obc-bound" "fail" "Failed to parse OBC status"
+        return
+    fi
+
+    if [[ "${obcStatus}" == "NotFound" ]]; then
+        AddResult "obc-bound" "skip" "No ObjectBucketClaim found for observability"
+        return
+    fi
+
+    typeset unboundObcs=""
+    unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | grep -v '=Bound$' || true)"
+
+    if [[ -z "${unboundObcs}" ]]; then
+        : "PASS: All observability OBCs bound: ${obcStatus}"
+        AddResult "obc-bound" "pass"
+    else
+        AddResult "obc-bound" "fail" "Unbound OBCs: ${unboundObcs//$'\n'/, }"
+    fi
+    true
+}
+
+# ---------------------------------------------------------------------------
+# Check 6: Thanos metrics query functional (basic data flow)
+# ---------------------------------------------------------------------------
+
+function CheckThanosQuery () {
+    : "=== Check 6: Thanos query functional ==="
+
+    typeset queryRoute=""
+    queryRoute="$(oc get routes -n "${OBS_NAMESPACE}" -o json 2>/dev/null \
+        | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+routes=[i for i in d.get('items',[]) if 'query' in i['metadata'].get('name','').lower() or 'observ' in i['metadata'].get('name','').lower()]
+if routes:
+    print(routes[0]['spec']['host'])
+else:
+    print('')
+" 2>/dev/null)" || true
+
+    if [[ -z "${queryRoute}" ]]; then
+        typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090"
+        : "No external route found; trying internal service: ${svcHost}"
+
+        typeset token=""
+        token="$(oc whoami -t 2>/dev/null)" || true
+
+        typeset queryResult=""
+        queryResult="$(oc exec -n "${OBS_NAMESPACE}" \
+            "$(oc get pods -n "${OBS_NAMESPACE}" -l app.kubernetes.io/name=thanos-query-frontend \
+                -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo '')" \
+            -- wget -qO- --no-check-certificate \
+            "http://localhost:9090/api/v1/query?query=up" 2>/dev/null)" || true
+
+        if [[ -z "${queryResult}" ]]; then
+            queryResult="$(oc exec -n "${OBS_NAMESPACE}" \
+                "$(oc get pods -n "${OBS_NAMESPACE}" --no-headers 2>/dev/null \
+                    | grep 'thanos-query' | grep -v 'frontend' | head -1 | awk '{print $1}')" \
+                -- wget -qO- --no-check-certificate \
+                "http://localhost:9090/api/v1/query?query=up" 2>/dev/null)" || true
+        fi
+
+        if [[ -z "${queryResult}" ]]; then
+            AddResult "thanos-query" "skip" "Cannot reach Thanos query endpoint (no route, exec failed)"
+            return
+        fi
+
+        typeset queryStatus=""
+        queryStatus="$(echo "${queryResult}" | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+print(d.get('status',''))
+" 2>/dev/null)" || true
+
+        if [[ "${queryStatus}" == "success" ]]; then
+            : "PASS: Thanos query returned success via exec"
+            AddResult "thanos-query" "pass"
+        else
+            AddResult "thanos-query" "fail" "Thanos query returned status=${queryStatus}"
+        fi
+        return
+    fi
+
+    typeset token=""
+    token="$(oc whoami -t 2>/dev/null)" || true
+
+    typeset httpCode=""
+    httpCode="$(curl -sk -o /dev/null -w '%{http_code}' \
+        -H "Authorization: Bearer ${token}" \
+        "https://${queryRoute}/api/v1/query?query=up" \
+        --max-time 30)" || true
+
+    if [[ "${httpCode}" == "200" ]]; then
+        AddResult "thanos-query" "pass"
+    elif [[ "${httpCode}" =~ ^(401|403)$ ]]; then
+        AddResult "thanos-query" "fail" "Thanos query route reachable but auth failed (HTTP ${httpCode}); no data flow verified"
+    else
+        AddResult "thanos-query" "fail" "Thanos query unreachable at ${queryRoute} (HTTP ${httpCode:-timeout})"
+    fi
+    true
+}
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+function Main () {
+    if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then
+        export KUBECONFIG="${SHARED_DIR}/kubeconfig"
+    fi
+
+    : "ACM Observability + ODF Interop Validation starting"
+    : "ACM namespace: ${ACM_NAMESPACE}"
+    : "Observability namespace: ${OBS_NAMESPACE}"
+    : "ODF namespace: ${ODF_NAMESPACE}"
+    : "Artifacts dir: ${ARTIFACT_DIR}"
+
+    CheckRgwReady          || true
+    CheckMcoReady          || true
+    CheckStorageEndpoint   || true
+    CheckThanosHealth      || true
+    CheckObcBound          || true
+    CheckThanosQuery       || true
+
+    WriteJunit
+
+    typeset -i hasAnyFail=0
+    typeset r=""
+    for r in "${tcResultsArr[@]}"; do
+        if [[ "${r}" == "fail" ]]; then
+            hasAnyFail=1
+            break
+        fi
+    done
+
+    if (( hasAnyFail )); then
+        : "ACM Observability + ODF Interop: SOME CHECKS FAILED"
+        exit 1
+    fi
+
+    : "ACM Observability + ODF Interop: ALL PASSED"
+    exit 0
+}
+
+Main "$@"
diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json
new file mode 100644
index 0000000000000..d2d17d0196b23
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.metadata.json
@@ -0,0 +1,11 @@
+{
+	"path": "interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml",
+	"owners": {
+		"approvers": [
+			"cspi-qe-ocp-lp"
+		],
+		"reviewers": [
+			"cspi-qe-ocp-lp"
+		]
+	}
+}
\ No newline at end of file
diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml
new file mode 100644
index 0000000000000..3276ddc73dbd1
--- /dev/null
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml
@@ -0,0 +1,36 @@
+ref:
+  as: interop-opp-observability-odf
+  from: cli
+  grace_period: 30s
+  commands: interop-opp-observability-odf-commands.sh
+  timeout: 10m
+  env:
+  - name: ACM_NAMESPACE
+    default: "open-cluster-management"
+    documentation: Namespace where ACM is installed
+  - name: OBS_NAMESPACE
+    default: "open-cluster-management-observability"
+    documentation: Namespace where ACM Observability components run
+  - name: ODF_NAMESPACE
+    default: "openshift-storage"
+    documentation: Namespace where ODF is installed
+  resources:
+    requests:
+      cpu: 100m
+      memory: 200Mi
+  best_effort: true
+  documentation: |-
+    Validates the cross-product integration surface between ACM Observability
+    (Thanos) and ODF (Ceph RGW). This is a true interop test that exercises
+    the boundary where ACM consumes ODF-provided object storage.
+
+    Checks performed:
+    1. ODF Ceph RGW infrastructure ready (CephObjectStore, RGW pods, StorageClass)
+    2. MultiClusterObservability CR exists and is Ready
+    3. Object storage secret references an ODF-backed endpoint (not MinIO)
+    4. Thanos and observability components healthy (receive, compact, store,
+       query, alertmanager, rbac-query-proxy)
+    5. ObjectBucketClaim used by observability is Bound
+    6. Thanos query endpoint functional (metrics data flow)
+
+    Produces JUnit XML for Prow / Sippy / TestGrid consumption.

From 76f649daa55d758a0d8da32946260ca3512a3e57 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Wed, 19 Aug 2026 10:14:42 -0500
Subject: [PATCH 02/10] INTEROP-9416: Validate Thanos query response body, not
 just HTTP status

Address CodeRabbit v2 finding: both the exec and route code paths now
parse the Thanos response and require status=success, resultType=vector,
and a non-empty result array before reporting pass. Shared validation
extracted into ValidateThanosResponse.
---
 .../interop-opp-observability-odf-commands.sh | 72 +++++++++++++------
 1 file changed, 52 insertions(+), 20 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
index 3510619b51e83..3955a9e7adec3 100755
--- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
@@ -402,6 +402,43 @@ else:
 # Check 6: Thanos metrics query functional (basic data flow)
 # ---------------------------------------------------------------------------
 
+function ValidateThanosResponse () {
+    typeset body="${1:-}"; (($#)) && shift
+    typeset via="${1:-unknown}"; (($#)) && shift
+
+    typeset validation=""
+    validation="$(echo "${body}" | python3 -c "
+import sys,json
+try:
+    d=json.load(sys.stdin)
+except Exception:
+    print('parse-error')
+    sys.exit(0)
+if d.get('status')!='success':
+    print('status=' + str(d.get('status','')))
+    sys.exit(0)
+data=d.get('data',{})
+if data.get('resultType')!='vector':
+    print('resultType=' + str(data.get('resultType','')))
+    sys.exit(0)
+result=data.get('result',[])
+if not isinstance(result,list) or len(result)==0:
+    print('empty-result')
+    sys.exit(0)
+print('ok')
+" 2>/dev/null)" || true
+
+    if [[ "${validation}" == "ok" ]]; then
+        AddResult "thanos-query" "pass"
+    elif [[ "${validation}" == "empty-result" ]]; then
+        AddResult "thanos-query" "fail" "Thanos query succeeded via ${via} but returned empty result vector"
+    elif [[ "${validation}" == "parse-error" || -z "${validation}" ]]; then
+        AddResult "thanos-query" "fail" "Thanos query via ${via} returned unparseable response"
+    else
+        AddResult "thanos-query" "fail" "Thanos query via ${via} returned ${validation}"
+    fi
+}
+
 function CheckThanosQuery () {
     : "=== Check 6: Thanos query functional ==="
 
@@ -444,38 +481,33 @@ else:
             return
         fi
 
-        typeset queryStatus=""
-        queryStatus="$(echo "${queryResult}" | python3 -c "
-import sys,json
-d=json.load(sys.stdin)
-print(d.get('status',''))
-" 2>/dev/null)" || true
-
-        if [[ "${queryStatus}" == "success" ]]; then
-            : "PASS: Thanos query returned success via exec"
-            AddResult "thanos-query" "pass"
-        else
-            AddResult "thanos-query" "fail" "Thanos query returned status=${queryStatus}"
-        fi
+        ValidateThanosResponse "${queryResult}" "exec"
         return
     fi
 
     typeset token=""
     token="$(oc whoami -t 2>/dev/null)" || true
 
+    typeset responseBody=""
     typeset httpCode=""
-    httpCode="$(curl -sk -o /dev/null -w '%{http_code}' \
+    responseBody="$(curl -sk -w '\n%{http_code}' \
         -H "Authorization: Bearer ${token}" \
         "https://${queryRoute}/api/v1/query?query=up" \
         --max-time 30)" || true
 
-    if [[ "${httpCode}" == "200" ]]; then
-        AddResult "thanos-query" "pass"
-    elif [[ "${httpCode}" =~ ^(401|403)$ ]]; then
-        AddResult "thanos-query" "fail" "Thanos query route reachable but auth failed (HTTP ${httpCode}); no data flow verified"
-    else
-        AddResult "thanos-query" "fail" "Thanos query unreachable at ${queryRoute} (HTTP ${httpCode:-timeout})"
+    httpCode="$(echo "${responseBody}" | tail -1)"
+    responseBody="$(echo "${responseBody}" | sed '$d')"
+
+    if [[ "${httpCode}" != "200" ]]; then
+        if [[ "${httpCode}" =~ ^(401|403)$ ]]; then
+            AddResult "thanos-query" "fail" "Thanos query route auth failed (HTTP ${httpCode}); no data flow verified"
+        else
+            AddResult "thanos-query" "fail" "Thanos query unreachable at ${queryRoute} (HTTP ${httpCode:-timeout})"
+        fi
+        return
     fi
+
+    ValidateThanosResponse "${responseBody}" "route"
     true
 }
 

From 21a6472444adc1682ad7eb2e5bd1c51904f2cd25 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Wed, 19 Aug 2026 10:28:16 -0500
Subject: [PATCH 03/10] INTEROP-9416: Remove cluster route from JUnit failure
 message

Avoids exposing internal cluster URLs in published CI artifacts.
---
 .../observability-odf/interop-opp-observability-odf-commands.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
index 3955a9e7adec3..10741fafd48bd 100755
--- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
@@ -502,7 +502,7 @@ else:
         if [[ "${httpCode}" =~ ^(401|403)$ ]]; then
             AddResult "thanos-query" "fail" "Thanos query route auth failed (HTTP ${httpCode}); no data flow verified"
         else
-            AddResult "thanos-query" "fail" "Thanos query unreachable at ${queryRoute} (HTTP ${httpCode:-timeout})"
+            AddResult "thanos-query" "fail" "Thanos query route unreachable (HTTP ${httpCode:-timeout})"
         fi
         return
     fi

From bd62d6aee887a54fd717a879b6f0ee0cd147f7bf Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Wed, 19 Aug 2026 10:53:40 -0500
Subject: [PATCH 04/10] INTEROP-9416: apply mpitt best-practice fixes

- Enable xtrace (set -euxo pipefail) for CI debugging
- Split pipelines to avoid masking oc failures with || true
- Add xtrace bracketing around bearer token curl
- Add terminal true to WriteJunit, CollectExitArtifacts, ValidateThanosResponse
- Use subshell trap form for EXIT handler
- Remove unused token variable in exec path
- Change &>/dev/null to 2>/dev/null for namespace check
---
 .../interop-opp-observability-odf-commands.sh | 60 ++++++++++++-------
 1 file changed, 39 insertions(+), 21 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
index 10741fafd48bd..d7c08762a5177 100755
--- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -euo pipefail
+set -euxo pipefail
 shopt -s inherit_errexit
 
 # ---------------------------------------------------------------------------
@@ -82,9 +82,10 @@ function WriteJunit () {
         echo "</testsuite>"
     } > "${junitFile}"
     : "JUnit XML written to ${junitFile}"
+    true
 }
 
-# shellcheck disable=SC2317
+# shellcheck disable=SC2317,SC2329
 function CollectExitArtifacts () {
     : "Collecting observability + ODF diagnostics..."
     oc get multiclusterobservabilities.observability.open-cluster-management.io --all-namespaces -o yaml > "${ARTIFACT_DIR}/mco.yaml" 2>/dev/null || true
@@ -94,9 +95,10 @@ function CollectExitArtifacts () {
     oc get cephobjectstore -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/cephobjectstore.yaml" 2>/dev/null || true
     oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw -o yaml > "${ARTIFACT_DIR}/rgw-pods.yaml" 2>/dev/null || true
     oc get noobaa -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/noobaa.yaml" 2>/dev/null || true
+    true
 }
 
-trap CollectExitArtifacts EXIT
+trap '{( CollectExitArtifacts; true )}' EXIT
 
 # ---------------------------------------------------------------------------
 # Check 1: ODF Ceph RGW infrastructure ready
@@ -129,9 +131,11 @@ else:
         failMsg="CephObjectStore phase=${rgwPhase} (expected Ready)"
     fi
 
+    typeset rgwPods=""
+    rgwPods="$(oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw \
+        --field-selector=status.phase=Running --no-headers 2>/dev/null)" || true
     typeset rgwPodCount=""
-    rgwPodCount="$(oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw \
-        --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l)" || true
+    rgwPodCount="$(printf '%s' "${rgwPods}" | grep -c . || true)"
 
     if [[ "${rgwPodCount}" -eq 0 ]]; then
         typeset rgwMsg="No rook-ceph-rgw pods Running in ${ODF_NAMESPACE}"
@@ -226,9 +230,11 @@ else:
         return
     fi
 
+    typeset secretJson=""
+    secretJson="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true
     typeset endpointCheck=""
-    endpointCheck="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json 2>/dev/null \
-        | python3 -c "
+    if [[ -n "${secretJson}" ]]; then
+        endpointCheck="$(printf '%s' "${secretJson}" | python3 -c "
 import sys,json,base64,re
 d=json.load(sys.stdin)
 odf_pattern=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs)', re.IGNORECASE)
@@ -242,6 +248,7 @@ for k,v in d.get('data',{}).items():
         sys.exit(0)
 print('no-endpoint')
 " 2>/dev/null)" || true
+    fi
 
     if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then
         AddResult "storage-endpoint" "fail" "Secret ${secretName} exists but no endpoint/bucket config found"
@@ -263,7 +270,7 @@ print('no-endpoint')
 function CheckThanosHealth () {
     : "=== Check 4: Thanos components healthy ==="
 
-    if ! oc get namespace "${OBS_NAMESPACE}" &>/dev/null; then
+    if ! oc get namespace "${OBS_NAMESPACE}" -o name 2>/dev/null; then
         AddResult "thanos-health" "skip" "Observability namespace ${OBS_NAMESPACE} does not exist"
         return
     fi
@@ -280,13 +287,17 @@ function CheckThanosHealth () {
         typeset component="${componentNames[$idx]}"
         typeset labelSelector="${componentLabels[$idx]}"
 
+        typeset podList=""
+        podList="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \
+            --no-headers 2>/dev/null)" || true
         typeset podCount=""
-        podCount="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \
-            --no-headers 2>/dev/null | wc -l)" || true
+        podCount="$(printf '%s' "${podList}" | grep -c . || true)"
 
         if [[ "${podCount}" -eq 0 ]]; then
-            podCount="$(oc get pods -n "${OBS_NAMESPACE}" \
-                --no-headers 2>/dev/null | grep -c "^${component}")" || true
+            typeset allPods=""
+            allPods="$(oc get pods -n "${OBS_NAMESPACE}" \
+                --no-headers 2>/dev/null)" || true
+            podCount="$(printf '%s' "${allPods}" | grep -c "^${component}" || true)"
         fi
 
         if [[ "${podCount}" -eq 0 ]]; then
@@ -296,9 +307,11 @@ function CheckThanosHealth () {
 
         (( foundCount++ )) || true
 
+        typeset labeledPods=""
+        labeledPods="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \
+            --no-headers 2>/dev/null)" || true
         typeset notReady=""
-        notReady="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \
-            --no-headers 2>/dev/null \
+        notReady="$(printf '%s' "${labeledPods}" \
             | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')" || true
 
         if [[ -n "${notReady}" ]]; then
@@ -343,18 +356,23 @@ print(len(d.get('items',[])))
     fi
 
     if [[ "${obcItemCount:-0}" -eq 0 ]]; then
-        obcList="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null \
-            | python3 -c "
+        typeset odfObcJson=""
+        odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || true
+        if [[ -n "${odfObcJson}" ]]; then
+            obcList="$(printf '%s' "${odfObcJson}" | python3 -c "
 import sys,json
 d=json.load(sys.stdin)
 obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()]
 print(json.dumps({'items':obs}))
 " 2>/dev/null)" || true
-        obcItemCount="$(echo "${obcList}" | python3 -c "
+        fi
+        if [[ -n "${obcList}" ]]; then
+            obcItemCount="$(echo "${obcList}" | python3 -c "
 import sys,json
 d=json.load(sys.stdin)
 print(len(d.get('items',[])))
 " 2>/dev/null)" || true
+        fi
     fi
 
     if [[ "${obcItemCount:-0}" -eq 0 ]]; then
@@ -387,7 +405,7 @@ else:
     fi
 
     typeset unboundObcs=""
-    unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | grep -v '=Bound$' || true)"
+    unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | grep -v '=Bound$')" || true
 
     if [[ -z "${unboundObcs}" ]]; then
         : "PASS: All observability OBCs bound: ${obcStatus}"
@@ -437,6 +455,7 @@ print('ok')
     else
         AddResult "thanos-query" "fail" "Thanos query via ${via} returned ${validation}"
     fi
+    true
 }
 
 function CheckThanosQuery () {
@@ -458,9 +477,6 @@ else:
         typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090"
         : "No external route found; trying internal service: ${svcHost}"
 
-        typeset token=""
-        token="$(oc whoami -t 2>/dev/null)" || true
-
         typeset queryResult=""
         queryResult="$(oc exec -n "${OBS_NAMESPACE}" \
             "$(oc get pods -n "${OBS_NAMESPACE}" -l app.kubernetes.io/name=thanos-query-frontend \
@@ -485,6 +501,7 @@ else:
         return
     fi
 
+    set +x
     typeset token=""
     token="$(oc whoami -t 2>/dev/null)" || true
 
@@ -494,6 +511,7 @@ else:
         -H "Authorization: Bearer ${token}" \
         "https://${queryRoute}/api/v1/query?query=up" \
         --max-time 30)" || true
+    set -x
 
     httpCode="$(echo "${responseBody}" | tail -1)"
     responseBody="$(echo "${responseBody}" | sed '$d')"

From 13caea0bdba5d462c01f2a662c203b384d1f952d Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Wed, 19 Aug 2026 10:56:03 -0500
Subject: [PATCH 05/10] INTEROP-9416: apply Chai Bot approach fixes

- Check 3: parse thanos.yaml YAML with metricObjectStorage.key
  instead of string-matching raw secret data; add mcg to ODF pattern
- Check 1: add NooBaa readiness fallback when RGW is absent;
  rename test case to odf-storage-ready
- Check 4: fail on missing S3-critical components (receive/compact/store)
  instead of passing with a note
- Check 6: narrow route discovery to Thanos-specific routes using
  exact match then targeted fuzzy match
---
 .../interop-opp-observability-odf-commands.sh | 98 +++++++++++++------
 1 file changed, 70 insertions(+), 28 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
index d7c08762a5177..86f7c07fcd2ee 100755
--- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
@@ -117,12 +117,21 @@ if not items:
 else:
     print(items[0].get('status',{}).get('phase','Unknown'))
 ")"; then
-        AddResult "rgw-ready" "fail" "Failed to query CephObjectStore"
+        AddResult "odf-storage-ready" "fail" "Failed to query CephObjectStore"
         return
     fi
 
     if [[ "${rgwPhase}" == "NotFound" ]]; then
-        AddResult "rgw-ready" "skip" "No CephObjectStore found; ODF RGW not deployed"
+        typeset noobaaPhase=""
+        noobaaPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" \
+            -o jsonpath='{.items[0].status.phase}' 2>/dev/null)" || true
+        if [[ "${noobaaPhase}" == "Ready" ]]; then
+            AddResult "odf-storage-ready" "pass" "NooBaa Ready (RGW not deployed)"
+        elif [[ -n "${noobaaPhase}" ]]; then
+            AddResult "odf-storage-ready" "fail" "NooBaa phase=${noobaaPhase} (expected Ready); RGW not deployed"
+        else
+            AddResult "odf-storage-ready" "skip" "Neither CephObjectStore nor NooBaa found in ${ODF_NAMESPACE}"
+        fi
         return
     fi
 
@@ -159,9 +168,9 @@ else:
 
     if [[ -z "${failMsg}" ]]; then
         : "PASS: CephObjectStore Ready, RGW pods Running, StorageClass exists"
-        AddResult "rgw-ready" "pass"
+        AddResult "odf-storage-ready" "pass"
     else
-        AddResult "rgw-ready" "fail" "${failMsg}"
+        AddResult "odf-storage-ready" "fail" "${failMsg}"
     fi
     true
 }
@@ -208,8 +217,8 @@ else:
 function CheckStorageEndpoint () {
     : "=== Check 3: Object storage endpoint ==="
 
-    typeset secretName=""
-    if ! secretName="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \
+    typeset storageConfig=""
+    if ! storageConfig="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \
         --all-namespaces -o json 2>/dev/null | python3 -c "
 import sys,json
 d=json.load(sys.stdin)
@@ -219,17 +228,22 @@ if not items:
 else:
     spec=items[0].get('spec',{})
     storage=spec.get('storageConfig',{}).get('metricObjectStorage',{})
-    print(storage.get('name',''))
+    name=storage.get('name','')
+    key=storage.get('key','thanos.yaml')
+    print(f'{name}|{key}' if name else '')
 ")"; then
         AddResult "storage-endpoint" "fail" "Failed to read MCO storage config"
         return
     fi
 
-    if [[ -z "${secretName}" ]]; then
+    if [[ -z "${storageConfig}" ]]; then
         AddResult "storage-endpoint" "skip" "No metricObjectStorage secret configured in MCO"
         return
     fi
 
+    typeset secretName="${storageConfig%%|*}"
+    typeset secretKey="${storageConfig#*|}"
+
     typeset secretJson=""
     secretJson="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true
     typeset endpointCheck=""
@@ -237,21 +251,30 @@ else:
         endpointCheck="$(printf '%s' "${secretJson}" | python3 -c "
 import sys,json,base64,re
 d=json.load(sys.stdin)
-odf_pattern=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs)', re.IGNORECASE)
-for k,v in d.get('data',{}).items():
-    decoded=base64.b64decode(v).decode('utf-8','replace')
-    if 'endpoint' in decoded.lower() or 'bucket' in decoded.lower():
-        if odf_pattern.search(decoded):
-            print('odf-backed')
-        else:
-            print('external')
-        sys.exit(0)
-print('no-endpoint')
-" 2>/dev/null)" || true
+target_key=sys.argv[1] if len(sys.argv)>1 else 'thanos.yaml'
+raw=d.get('data',{}).get(target_key,'')
+if not raw:
+    print('no-endpoint')
+    sys.exit(0)
+decoded=base64.b64decode(raw).decode('utf-8','replace')
+try:
+    import yaml
+    cfg=yaml.safe_load(decoded)
+    endpoint=cfg.get('config',{}).get('endpoint','') if isinstance(cfg,dict) else ''
+except Exception:
+    import re as re2
+    m=re2.search(r'endpoint:\s*(.+)',decoded)
+    endpoint=m.group(1).strip() if m else ''
+if not endpoint:
+    print('no-endpoint')
+    sys.exit(0)
+odf=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE)
+print('odf-backed' if odf.search(endpoint) else 'external')
+" "${secretKey}" 2>/dev/null)" || true
     fi
 
     if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then
-        AddResult "storage-endpoint" "fail" "Secret ${secretName} exists but no endpoint/bucket config found"
+        AddResult "storage-endpoint" "fail" "Secret ${secretName} exists but no endpoint config found in key ${secretKey}"
         return
     fi
 
@@ -324,12 +347,26 @@ function CheckThanosHealth () {
         fi
     done
 
+    typeset -a s3CriticalNames=("thanos-receive" "thanos-compact" "thanos-store")
+    typeset -a missingCritical=()
+    typeset mc=""
+    for mc in "${missingComponents[@]}"; do
+        typeset cc=""
+        for cc in "${s3CriticalNames[@]}"; do
+            if [[ "${mc}" == "${cc}" ]]; then
+                missingCritical+=("${mc}")
+            fi
+        done
+    done
+
     if (( foundCount == 0 )); then
         AddResult "thanos-health" "fail" "No Thanos/observability components found in ${OBS_NAMESPACE}"
     elif [[ -n "${failMsg}" ]]; then
         AddResult "thanos-health" "fail" "Unhealthy Thanos components: ${failMsg}"
+    elif (( ${#missingCritical[@]} > 0 )); then
+        AddResult "thanos-health" "fail" "Missing S3-critical components: ${missingCritical[*]}"
     elif (( ${#missingComponents[@]} > 0 )); then
-        AddResult "thanos-health" "pass" "Running (missing: ${missingComponents[*]})"
+        AddResult "thanos-health" "pass" "Running (optional missing: ${missingComponents[*]})"
     else
         AddResult "thanos-health" "pass"
     fi
@@ -461,17 +498,22 @@ print('ok')
 function CheckThanosQuery () {
     : "=== Check 6: Thanos query functional ==="
 
+    typeset routeJson=""
+    routeJson="$(oc get routes -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true
     typeset queryRoute=""
-    queryRoute="$(oc get routes -n "${OBS_NAMESPACE}" -o json 2>/dev/null \
-        | python3 -c "
+    if [[ -n "${routeJson}" ]]; then
+        queryRoute="$(printf '%s' "${routeJson}" | python3 -c "
 import sys,json
 d=json.load(sys.stdin)
-routes=[i for i in d.get('items',[]) if 'query' in i['metadata'].get('name','').lower() or 'observ' in i['metadata'].get('name','').lower()]
-if routes:
-    print(routes[0]['spec']['host'])
-else:
-    print('')
+routes=d.get('items',[])
+exact=[r for r in routes if r['metadata']['name']=='observability-thanos-query']
+if exact:
+    print(exact[0]['spec']['host'])
+    sys.exit(0)
+fuzzy=[r for r in routes if 'thanos' in r['metadata']['name'] and 'query' in r['metadata']['name']]
+print(fuzzy[0]['spec']['host'] if fuzzy else '')
 " 2>/dev/null)" || true
+    fi
 
     if [[ -z "${queryRoute}" ]]; then
         typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090"

From ae63f4b0d081dc48bbbe5cd06ccb6df05618cfab Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Wed, 19 Aug 2026 13:46:40 -0500
Subject: [PATCH 06/10] INTEROP-9416: mpitt self-review R1 convergence fixes

- Remove stderr suppression (2>/dev/null) to preserve error output
  in xtrace for CI debugging
- Replace grep -c || true with awk END{print NR} (pipefail-safe)
- Use pre-increment (( ++x )) instead of (( x++ )) || true
- Replace unsafe jsonpath items[0] with python3 safe-access
- Guard oc exec with pod existence check before executing
- Consolidate grep|head|awk pipelines into single awk
- Replace grep -v with sed for pipefail safety
- Add typeset to env var declarations
---
 .../interop-opp-observability-odf-commands.sh | 115 ++++++++++--------
 1 file changed, 64 insertions(+), 51 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
index 86f7c07fcd2ee..6074184942d43 100755
--- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
@@ -1,6 +1,5 @@
 #!/bin/bash
-set -euxo pipefail
-shopt -s inherit_errexit
+set -euxo pipefail; shopt -s inherit_errexit
 
 # ---------------------------------------------------------------------------
 # ACM Observability + ODF Interop Validation (6-point gate)
@@ -12,9 +11,9 @@ shopt -s inherit_errexit
 # Produces JUnit XML consumed by Prow / Sippy / TestGrid.
 # ---------------------------------------------------------------------------
 
-ACM_NAMESPACE="${ACM_NAMESPACE:-open-cluster-management}"
-OBS_NAMESPACE="${OBS_NAMESPACE:-open-cluster-management-observability}"
-ODF_NAMESPACE="${ODF_NAMESPACE:-openshift-storage}"
+typeset ACM_NAMESPACE="${ACM_NAMESPACE:-open-cluster-management}"
+typeset OBS_NAMESPACE="${OBS_NAMESPACE:-open-cluster-management-observability}"
+typeset ODF_NAMESPACE="${ODF_NAMESPACE:-openshift-storage}"
 
 typeset junitFile="${ARTIFACT_DIR}/junit_observability_odf.xml"
 
@@ -54,9 +53,9 @@ function WriteJunit () {
     typeset r=""
     for r in "${tcResultsArr[@]}"; do
         if [[ "${r}" == "fail" ]]; then
-            (( failCount++ )) || true
+            (( ++failCount ))
         elif [[ "${r}" == "skip" ]]; then
-            (( skipCount++ )) || true
+            (( ++skipCount ))
         fi
     done
 
@@ -88,13 +87,13 @@ function WriteJunit () {
 # shellcheck disable=SC2317,SC2329
 function CollectExitArtifacts () {
     : "Collecting observability + ODF diagnostics..."
-    oc get multiclusterobservabilities.observability.open-cluster-management.io --all-namespaces -o yaml > "${ARTIFACT_DIR}/mco.yaml" 2>/dev/null || true
-    oc get pods -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-pods.yaml" 2>/dev/null || true
-    oc get obc -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-obc.yaml" 2>/dev/null || true
-    oc get secret -n "${OBS_NAMESPACE}" -o name > "${ARTIFACT_DIR}/obs-secrets-list.txt" 2>/dev/null || true
-    oc get cephobjectstore -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/cephobjectstore.yaml" 2>/dev/null || true
-    oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw -o yaml > "${ARTIFACT_DIR}/rgw-pods.yaml" 2>/dev/null || true
-    oc get noobaa -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/noobaa.yaml" 2>/dev/null || true
+    oc get multiclusterobservabilities.observability.open-cluster-management.io --all-namespaces -o yaml > "${ARTIFACT_DIR}/mco.yaml" || true
+    oc get pods -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-pods.yaml" || true
+    oc get obc -n "${OBS_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/obs-obc.yaml" || true
+    oc get secret -n "${OBS_NAMESPACE}" -o name > "${ARTIFACT_DIR}/obs-secrets-list.txt" || true
+    oc get cephobjectstore -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/cephobjectstore.yaml" || true
+    oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw -o yaml > "${ARTIFACT_DIR}/rgw-pods.yaml" || true
+    oc get noobaa -n "${ODF_NAMESPACE}" -o yaml > "${ARTIFACT_DIR}/noobaa.yaml" || true
     true
 }
 
@@ -108,7 +107,7 @@ function CheckRgwReady () {
     : "=== Check 1: ODF Ceph RGW infrastructure ==="
 
     typeset rgwPhase=""
-    if ! rgwPhase="$(oc get cephobjectstore -n "${ODF_NAMESPACE}" -o json 2>/dev/null | python3 -c "
+    if ! rgwPhase="$(oc get cephobjectstore -n "${ODF_NAMESPACE}" -o json | python3 -c "
 import sys,json
 d=json.load(sys.stdin)
 items=d.get('items',[])
@@ -123,8 +122,11 @@ else:
 
     if [[ "${rgwPhase}" == "NotFound" ]]; then
         typeset noobaaPhase=""
-        noobaaPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" \
-            -o jsonpath='{.items[0].status.phase}' 2>/dev/null)" || true
+        noobaaPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | python3 -c "
+import sys,json
+items=json.load(sys.stdin).get('items',[])
+print(items[0].get('status',{}).get('phase','') if items else '')
+")" || true
         if [[ "${noobaaPhase}" == "Ready" ]]; then
             AddResult "odf-storage-ready" "pass" "NooBaa Ready (RGW not deployed)"
         elif [[ -n "${noobaaPhase}" ]]; then
@@ -142,9 +144,9 @@ else:
 
     typeset rgwPods=""
     rgwPods="$(oc get pods -n "${ODF_NAMESPACE}" -l app=rook-ceph-rgw \
-        --field-selector=status.phase=Running --no-headers 2>/dev/null)" || true
+        --field-selector=status.phase=Running --no-headers)" || true
     typeset rgwPodCount=""
-    rgwPodCount="$(printf '%s' "${rgwPods}" | grep -c . || true)"
+    rgwPodCount="$(printf '%s' "${rgwPods}" | awk 'END{print NR}')"
 
     if [[ "${rgwPodCount}" -eq 0 ]]; then
         typeset rgwMsg="No rook-ceph-rgw pods Running in ${ODF_NAMESPACE}"
@@ -156,7 +158,7 @@ else:
     fi
 
     typeset scExists=""
-    scExists="$(oc get sc ocs-storagecluster-ceph-rgw -o name 2>/dev/null)" || true
+    scExists="$(oc get sc ocs-storagecluster-ceph-rgw -o name)" || true
     if [[ -z "${scExists}" ]]; then
         typeset scMsg="StorageClass ocs-storagecluster-ceph-rgw not found"
         if [[ -n "${failMsg}" ]]; then
@@ -184,7 +186,7 @@ function CheckMcoReady () {
 
     typeset mcoStatus=""
     if ! mcoStatus="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \
-        --all-namespaces -o json 2>/dev/null | python3 -c "
+        --all-namespaces -o json | python3 -c "
 import sys,json
 d=json.load(sys.stdin)
 items=d.get('items',[])
@@ -219,7 +221,7 @@ function CheckStorageEndpoint () {
 
     typeset storageConfig=""
     if ! storageConfig="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \
-        --all-namespaces -o json 2>/dev/null | python3 -c "
+        --all-namespaces -o json | python3 -c "
 import sys,json
 d=json.load(sys.stdin)
 items=d.get('items',[])
@@ -245,7 +247,7 @@ else:
     typeset secretKey="${storageConfig#*|}"
 
     typeset secretJson=""
-    secretJson="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true
+    secretJson="$(oc get secret "${secretName}" -n "${OBS_NAMESPACE}" -o json)" || true
     typeset endpointCheck=""
     if [[ -n "${secretJson}" ]]; then
         endpointCheck="$(printf '%s' "${secretJson}" | python3 -c "
@@ -270,7 +272,7 @@ if not endpoint:
     sys.exit(0)
 odf=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE)
 print('odf-backed' if odf.search(endpoint) else 'external')
-" "${secretKey}" 2>/dev/null)" || true
+" "${secretKey}")" || true
     fi
 
     if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then
@@ -293,7 +295,7 @@ print('odf-backed' if odf.search(endpoint) else 'external')
 function CheckThanosHealth () {
     : "=== Check 4: Thanos components healthy ==="
 
-    if ! oc get namespace "${OBS_NAMESPACE}" -o name 2>/dev/null; then
+    if ! oc get namespace "${OBS_NAMESPACE}" -o name; then
         AddResult "thanos-health" "skip" "Observability namespace ${OBS_NAMESPACE} does not exist"
         return
     fi
@@ -312,15 +314,15 @@ function CheckThanosHealth () {
 
         typeset podList=""
         podList="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \
-            --no-headers 2>/dev/null)" || true
+            --no-headers)" || true
         typeset podCount=""
-        podCount="$(printf '%s' "${podList}" | grep -c . || true)"
+        podCount="$(printf '%s' "${podList}" | awk 'END{print NR}')"
 
         if [[ "${podCount}" -eq 0 ]]; then
             typeset allPods=""
             allPods="$(oc get pods -n "${OBS_NAMESPACE}" \
-                --no-headers 2>/dev/null)" || true
-            podCount="$(printf '%s' "${allPods}" | grep -c "^${component}" || true)"
+                --no-headers)" || true
+            podCount="$(printf '%s' "${allPods}" | awk -v pat="^${component}" '$0 ~ pat {c++} END{print c+0}')"
         fi
 
         if [[ "${podCount}" -eq 0 ]]; then
@@ -328,11 +330,11 @@ function CheckThanosHealth () {
             continue
         fi
 
-        (( foundCount++ )) || true
+        (( ++foundCount ))
 
         typeset labeledPods=""
         labeledPods="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \
-            --no-headers 2>/dev/null)" || true
+            --no-headers)" || true
         typeset notReady=""
         notReady="$(printf '%s' "${labeledPods}" \
             | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')" || true
@@ -381,7 +383,7 @@ function CheckObcBound () {
     : "=== Check 5: Observability ObjectBucketClaim ==="
 
     typeset obcList=""
-    obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true
+    obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json)" || true
 
     typeset obcItemCount=""
     if [[ -n "${obcList}" ]]; then
@@ -389,26 +391,26 @@ function CheckObcBound () {
 import sys,json
 d=json.load(sys.stdin)
 print(len(d.get('items',[])))
-" 2>/dev/null)" || true
+")" || true
     fi
 
     if [[ "${obcItemCount:-0}" -eq 0 ]]; then
         typeset odfObcJson=""
-        odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || true
+        odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json)" || true
         if [[ -n "${odfObcJson}" ]]; then
             obcList="$(printf '%s' "${odfObcJson}" | python3 -c "
 import sys,json
 d=json.load(sys.stdin)
 obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()]
 print(json.dumps({'items':obs}))
-" 2>/dev/null)" || true
+")" || true
         fi
         if [[ -n "${obcList}" ]]; then
             obcItemCount="$(echo "${obcList}" | python3 -c "
 import sys,json
 d=json.load(sys.stdin)
 print(len(d.get('items',[])))
-" 2>/dev/null)" || true
+")" || true
         fi
     fi
 
@@ -442,7 +444,7 @@ else:
     fi
 
     typeset unboundObcs=""
-    unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | grep -v '=Bound$')" || true
+    unboundObcs="$(echo "${obcStatus}" | tr ';' '\n' | sed '/=Bound$/d')"
 
     if [[ -z "${unboundObcs}" ]]; then
         : "PASS: All observability OBCs bound: ${obcStatus}"
@@ -481,7 +483,7 @@ if not isinstance(result,list) or len(result)==0:
     print('empty-result')
     sys.exit(0)
 print('ok')
-" 2>/dev/null)" || true
+")" || true
 
     if [[ "${validation}" == "ok" ]]; then
         AddResult "thanos-query" "pass"
@@ -499,7 +501,7 @@ function CheckThanosQuery () {
     : "=== Check 6: Thanos query functional ==="
 
     typeset routeJson=""
-    routeJson="$(oc get routes -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true
+    routeJson="$(oc get routes -n "${OBS_NAMESPACE}" -o json)" || true
     typeset queryRoute=""
     if [[ -n "${routeJson}" ]]; then
         queryRoute="$(printf '%s' "${routeJson}" | python3 -c "
@@ -512,26 +514,37 @@ if exact:
     sys.exit(0)
 fuzzy=[r for r in routes if 'thanos' in r['metadata']['name'] and 'query' in r['metadata']['name']]
 print(fuzzy[0]['spec']['host'] if fuzzy else '')
-" 2>/dev/null)" || true
+")" || true
     fi
 
     if [[ -z "${queryRoute}" ]]; then
         typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090"
         : "No external route found; trying internal service: ${svcHost}"
 
+        typeset queryFrontendPod=''
+        queryFrontendPod="$(oc get pods -n "${OBS_NAMESPACE}" \
+            -l app.kubernetes.io/name=thanos-query-frontend -o json | python3 -c "
+import sys,json
+items=json.load(sys.stdin).get('items',[])
+print(items[0]['metadata']['name'] if items else '')
+")" || true
+
         typeset queryResult=""
-        queryResult="$(oc exec -n "${OBS_NAMESPACE}" \
-            "$(oc get pods -n "${OBS_NAMESPACE}" -l app.kubernetes.io/name=thanos-query-frontend \
-                -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo '')" \
-            -- wget -qO- --no-check-certificate \
-            "http://localhost:9090/api/v1/query?query=up" 2>/dev/null)" || true
+        if [[ -n "${queryFrontendPod}" ]]; then
+            queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryFrontendPod}" \
+                -- wget -qO- --no-check-certificate \
+                "http://localhost:9090/api/v1/query?query=up")" || true
+        fi
 
         if [[ -z "${queryResult}" ]]; then
-            queryResult="$(oc exec -n "${OBS_NAMESPACE}" \
-                "$(oc get pods -n "${OBS_NAMESPACE}" --no-headers 2>/dev/null \
-                    | grep 'thanos-query' | grep -v 'frontend' | head -1 | awk '{print $1}')" \
-                -- wget -qO- --no-check-certificate \
-                "http://localhost:9090/api/v1/query?query=up" 2>/dev/null)" || true
+            typeset queryPod=''
+            queryPod="$(oc get pods -n "${OBS_NAMESPACE}" --no-headers \
+                | awk '/thanos-query/ && !/frontend/ {print $1; exit}')" || true
+            if [[ -n "${queryPod}" ]]; then
+                queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryPod}" \
+                    -- wget -qO- --no-check-certificate \
+                    "http://localhost:9090/api/v1/query?query=up")" || true
+            fi
         fi
 
         if [[ -z "${queryResult}" ]]; then
@@ -545,7 +558,7 @@ print(fuzzy[0]['spec']['host'] if fuzzy else '')
 
     set +x
     typeset token=""
-    token="$(oc whoami -t 2>/dev/null)" || true
+    token="$(oc whoami -t)" || true
 
     typeset responseBody=""
     typeset httpCode=""

From 1b60013dea1b1c3e5a360633e984839b960c8be3 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Wed, 19 Aug 2026 13:54:46 -0500
Subject: [PATCH 07/10] INTEROP-9416: mpitt R2 pipeline cleanup

- Remove || true from pipelines where both sides always exit 0
- Consolidate duplicate oc get pods call (reuse podList)
- Separate oc get | python3 pipelines for noobaa and query pods
---
 .../interop-opp-observability-odf-commands.sh | 45 +++++++++++--------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
index 6074184942d43..c7da7b6e9a923 100755
--- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
@@ -121,12 +121,16 @@ else:
     fi
 
     if [[ "${rgwPhase}" == "NotFound" ]]; then
+        typeset noobaaJson=""
+        noobaaJson="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json)" || true
         typeset noobaaPhase=""
-        noobaaPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | python3 -c "
+        if [[ -n "${noobaaJson}" ]]; then
+            noobaaPhase="$(printf '%s' "${noobaaJson}" | python3 -c "
 import sys,json
 items=json.load(sys.stdin).get('items',[])
 print(items[0].get('status',{}).get('phase','') if items else '')
-")" || true
+")"
+        fi
         if [[ "${noobaaPhase}" == "Ready" ]]; then
             AddResult "odf-storage-ready" "pass" "NooBaa Ready (RGW not deployed)"
         elif [[ -n "${noobaaPhase}" ]]; then
@@ -272,7 +276,7 @@ if not endpoint:
     sys.exit(0)
 odf=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE)
 print('odf-backed' if odf.search(endpoint) else 'external')
-" "${secretKey}")" || true
+" "${secretKey}")"
     fi
 
     if [[ "${endpointCheck}" == "no-endpoint" || -z "${endpointCheck}" ]]; then
@@ -332,12 +336,9 @@ function CheckThanosHealth () {
 
         (( ++foundCount ))
 
-        typeset labeledPods=""
-        labeledPods="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \
-            --no-headers)" || true
         typeset notReady=""
-        notReady="$(printf '%s' "${labeledPods}" \
-            | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')" || true
+        notReady="$(printf '%s' "${podList}" \
+            | awk '$3 != "Running" && $3 != "Completed" {print $1 ":" $3}')"
 
         if [[ -n "${notReady}" ]]; then
             typeset compMsg="${component}: ${notReady//$'\n'/, }"
@@ -391,7 +392,7 @@ function CheckObcBound () {
 import sys,json
 d=json.load(sys.stdin)
 print(len(d.get('items',[])))
-")" || true
+")"
     fi
 
     if [[ "${obcItemCount:-0}" -eq 0 ]]; then
@@ -403,14 +404,14 @@ import sys,json
 d=json.load(sys.stdin)
 obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()]
 print(json.dumps({'items':obs}))
-")" || true
+")"
         fi
         if [[ -n "${obcList}" ]]; then
             obcItemCount="$(echo "${obcList}" | python3 -c "
 import sys,json
 d=json.load(sys.stdin)
 print(len(d.get('items',[])))
-")" || true
+")"
         fi
     fi
 
@@ -483,7 +484,7 @@ if not isinstance(result,list) or len(result)==0:
     print('empty-result')
     sys.exit(0)
 print('ok')
-")" || true
+")"
 
     if [[ "${validation}" == "ok" ]]; then
         AddResult "thanos-query" "pass"
@@ -514,20 +515,24 @@ if exact:
     sys.exit(0)
 fuzzy=[r for r in routes if 'thanos' in r['metadata']['name'] and 'query' in r['metadata']['name']]
 print(fuzzy[0]['spec']['host'] if fuzzy else '')
-")" || true
+")"
     fi
 
     if [[ -z "${queryRoute}" ]]; then
         typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090"
         : "No external route found; trying internal service: ${svcHost}"
 
+        typeset queryFrontendJson=''
+        queryFrontendJson="$(oc get pods -n "${OBS_NAMESPACE}" \
+            -l app.kubernetes.io/name=thanos-query-frontend -o json)" || true
         typeset queryFrontendPod=''
-        queryFrontendPod="$(oc get pods -n "${OBS_NAMESPACE}" \
-            -l app.kubernetes.io/name=thanos-query-frontend -o json | python3 -c "
+        if [[ -n "${queryFrontendJson}" ]]; then
+            queryFrontendPod="$(printf '%s' "${queryFrontendJson}" | python3 -c "
 import sys,json
 items=json.load(sys.stdin).get('items',[])
 print(items[0]['metadata']['name'] if items else '')
-")" || true
+")"
+        fi
 
         typeset queryResult=""
         if [[ -n "${queryFrontendPod}" ]]; then
@@ -537,9 +542,13 @@ print(items[0]['metadata']['name'] if items else '')
         fi
 
         if [[ -z "${queryResult}" ]]; then
+            typeset allQueryPods=''
+            allQueryPods="$(oc get pods -n "${OBS_NAMESPACE}" --no-headers)" || true
             typeset queryPod=''
-            queryPod="$(oc get pods -n "${OBS_NAMESPACE}" --no-headers \
-                | awk '/thanos-query/ && !/frontend/ {print $1; exit}')" || true
+            if [[ -n "${allQueryPods}" ]]; then
+                queryPod="$(printf '%s' "${allQueryPods}" \
+                    | awk '/thanos-query/ && !/frontend/ {print $1; exit}')"
+            fi
             if [[ -n "${queryPod}" ]]; then
                 queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryPod}" \
                     -- wget -qO- --no-check-certificate \

From e45347dc4bf19e1cfaa9c83a3a767416c287d0ef Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Thu, 20 Aug 2026 13:24:28 -0500
Subject: [PATCH 08/10] INTEROP-9416: Fix pod status check after name-prefix
 fallback

When the label selector returned no pods but the name-prefix fallback
found matching pods, the notReady check was still operating on the
empty label-selector result. Reassign podList in the fallback path so
status inspection uses the correct pod listing.

Also inline the svcHost variable that was only used in a diagnostic
marker.
---
 .../interop-opp-observability-odf-commands.sh        | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
index c7da7b6e9a923..1ec8880af4066 100755
--- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
@@ -319,16 +319,17 @@ function CheckThanosHealth () {
         typeset podList=""
         podList="$(oc get pods -n "${OBS_NAMESPACE}" -l "${labelSelector}" \
             --no-headers)" || true
-        typeset podCount=""
-        podCount="$(printf '%s' "${podList}" | awk 'END{print NR}')"
 
-        if [[ "${podCount}" -eq 0 ]]; then
+        if [[ -z "${podList}" ]]; then
             typeset allPods=""
             allPods="$(oc get pods -n "${OBS_NAMESPACE}" \
                 --no-headers)" || true
-            podCount="$(printf '%s' "${allPods}" | awk -v pat="^${component}" '$0 ~ pat {c++} END{print c+0}')"
+            podList="$(printf '%s' "${allPods}" | awk -v pat="^${component}" '$0 ~ pat')"
         fi
 
+        typeset podCount=""
+        podCount="$(printf '%s' "${podList}" | awk 'NF {c++} END{print c+0}')"
+
         if [[ "${podCount}" -eq 0 ]]; then
             missingComponents+=("${component}")
             continue
@@ -519,8 +520,7 @@ print(fuzzy[0]['spec']['host'] if fuzzy else '')
     fi
 
     if [[ -z "${queryRoute}" ]]; then
-        typeset svcHost="observability-thanos-query-frontend.${OBS_NAMESPACE}.svc:9090"
-        : "No external route found; trying internal service: ${svcHost}"
+        : "No external route found; trying internal query service"
 
         typeset queryFrontendJson=''
         queryFrontendJson="$(oc get pods -n "${OBS_NAMESPACE}" \

From 8953f50f550356b1f548cfbd03c22d478aa0dfe1 Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Fri, 21 Aug 2026 08:13:38 -0500
Subject: [PATCH 09/10] fix: address review findings in observability-odf step

---
 .../interop-opp-observability-odf-commands.sh | 83 +++++++++----------
 1 file changed, 38 insertions(+), 45 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
index 1ec8880af4066..936d2730dce00 100755
--- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -euxo pipefail; shopt -s inherit_errexit
+set -euo pipefail; shopt -s inherit_errexit
 
 # ---------------------------------------------------------------------------
 # ACM Observability + ODF Interop Validation (6-point gate)
@@ -256,26 +256,32 @@ else:
     if [[ -n "${secretJson}" ]]; then
         endpointCheck="$(printf '%s' "${secretJson}" | python3 -c "
 import sys,json,base64,re
+sys.tracebacklimit=0
 d=json.load(sys.stdin)
 target_key=sys.argv[1] if len(sys.argv)>1 else 'thanos.yaml'
 raw=d.get('data',{}).get(target_key,'')
 if not raw:
     print('no-endpoint')
     sys.exit(0)
-decoded=base64.b64decode(raw).decode('utf-8','replace')
+try:
+    content=base64.b64decode(raw).decode('utf-8','replace')
+except Exception:
+    print('no-endpoint')
+    sys.exit(0)
+endpoint=''
 try:
     import yaml
-    cfg=yaml.safe_load(decoded)
+    cfg=yaml.safe_load(content)
     endpoint=cfg.get('config',{}).get('endpoint','') if isinstance(cfg,dict) else ''
 except Exception:
-    import re as re2
-    m=re2.search(r'endpoint:\s*(.+)',decoded)
+    m=re.search(r'endpoint:\s*(.+)',content)
     endpoint=m.group(1).strip() if m else ''
+del content
 if not endpoint:
     print('no-endpoint')
     sys.exit(0)
-odf=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE)
-print('odf-backed' if odf.search(endpoint) else 'external')
+odf_pat=re.compile(r'(openshift-storage|noobaa|ceph|rgw|rook|ocs|mcg)',re.IGNORECASE)
+print('odf-backed' if odf_pat.search(endpoint) else 'external')
 " "${secretKey}")"
     fi
 
@@ -351,26 +357,12 @@ function CheckThanosHealth () {
         fi
     done
 
-    typeset -a s3CriticalNames=("thanos-receive" "thanos-compact" "thanos-store")
-    typeset -a missingCritical=()
-    typeset mc=""
-    for mc in "${missingComponents[@]}"; do
-        typeset cc=""
-        for cc in "${s3CriticalNames[@]}"; do
-            if [[ "${mc}" == "${cc}" ]]; then
-                missingCritical+=("${mc}")
-            fi
-        done
-    done
-
     if (( foundCount == 0 )); then
         AddResult "thanos-health" "fail" "No Thanos/observability components found in ${OBS_NAMESPACE}"
     elif [[ -n "${failMsg}" ]]; then
         AddResult "thanos-health" "fail" "Unhealthy Thanos components: ${failMsg}"
-    elif (( ${#missingCritical[@]} > 0 )); then
-        AddResult "thanos-health" "fail" "Missing S3-critical components: ${missingCritical[*]}"
     elif (( ${#missingComponents[@]} > 0 )); then
-        AddResult "thanos-health" "pass" "Running (optional missing: ${missingComponents[*]})"
+        AddResult "thanos-health" "fail" "Missing components: ${missingComponents[*]}"
     else
         AddResult "thanos-health" "pass"
     fi
@@ -385,38 +377,41 @@ function CheckObcBound () {
     : "=== Check 5: Observability ObjectBucketClaim ==="
 
     typeset obcList=""
-    obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json)" || true
+    obcList="$(oc get obc -n "${OBS_NAMESPACE}" -o json 2>/dev/null)" || true
 
-    typeset obcItemCount=""
-    if [[ -n "${obcList}" ]]; then
-        obcItemCount="$(echo "${obcList}" | python3 -c "
+    typeset obcItemCount=0
+    obcItemCount="$(printf '%s' "${obcList}" | python3 -c "
 import sys,json
-d=json.load(sys.stdin)
-print(len(d.get('items',[])))
+try:
+    d=json.load(sys.stdin)
+    print(len(d.get('items',[])))
+except Exception:
+    print(0)
 ")"
-    fi
 
-    if [[ "${obcItemCount:-0}" -eq 0 ]]; then
+    if [[ "${obcItemCount}" -eq 0 ]]; then
         typeset odfObcJson=""
-        odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json)" || true
-        if [[ -n "${odfObcJson}" ]]; then
-            obcList="$(printf '%s' "${odfObcJson}" | python3 -c "
+        odfObcJson="$(oc get obc -n "${ODF_NAMESPACE}" -o json 2>/dev/null)" || true
+        obcList="$(printf '%s' "${odfObcJson}" | python3 -c "
 import sys,json
-d=json.load(sys.stdin)
-obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()]
-print(json.dumps({'items':obs}))
+try:
+    d=json.load(sys.stdin)
+    obs=[i for i in d.get('items',[]) if 'obs' in i['metadata'].get('name','').lower() or 'thanos' in i['metadata'].get('name','').lower()]
+    print(json.dumps({'items':obs}))
+except Exception:
+    print(json.dumps({'items':[]}))
 ")"
-        fi
-        if [[ -n "${obcList}" ]]; then
-            obcItemCount="$(echo "${obcList}" | python3 -c "
+        obcItemCount="$(printf '%s' "${obcList}" | python3 -c "
 import sys,json
-d=json.load(sys.stdin)
-print(len(d.get('items',[])))
+try:
+    d=json.load(sys.stdin)
+    print(len(d.get('items',[])))
+except Exception:
+    print(0)
 ")"
-        fi
     fi
 
-    if [[ "${obcItemCount:-0}" -eq 0 ]]; then
+    if [[ "${obcItemCount}" -eq 0 ]]; then
         AddResult "obc-bound" "skip" "No ObjectBucketClaim found for observability"
         return
     fi
@@ -565,7 +560,6 @@ print(items[0]['metadata']['name'] if items else '')
         return
     fi
 
-    set +x
     typeset token=""
     token="$(oc whoami -t)" || true
 
@@ -575,7 +569,6 @@ print(items[0]['metadata']['name'] if items else '')
         -H "Authorization: Bearer ${token}" \
         "https://${queryRoute}/api/v1/query?query=up" \
         --max-time 30)" || true
-    set -x
 
     httpCode="$(echo "${responseBody}" | tail -1)"
     responseBody="$(echo "${responseBody}" | sed '$d')"

From e3f9dfaed18d3f9f41395465e9dbb6e0d3e25a5f Mon Sep 17 00:00:00 2001
From: Michael Pruitt <mpruitt@redhat.com>
Date: Fri, 21 Aug 2026 18:30:59 -0500
Subject: [PATCH 10/10] fix: observability-odf use curl instead of wget,
 graceful skip

---
 .../interop-opp-observability-odf-commands.sh               | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
index 936d2730dce00..88763c329a7fa 100755
--- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
+++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh
@@ -358,7 +358,7 @@ function CheckThanosHealth () {
     done
 
     if (( foundCount == 0 )); then
-        AddResult "thanos-health" "fail" "No Thanos/observability components found in ${OBS_NAMESPACE}"
+        AddResult "thanos-health" "skip" "No Thanos/observability components found in ${OBS_NAMESPACE}; observability not deployed"
     elif [[ -n "${failMsg}" ]]; then
         AddResult "thanos-health" "fail" "Unhealthy Thanos components: ${failMsg}"
     elif (( ${#missingComponents[@]} > 0 )); then
@@ -532,7 +532,7 @@ print(items[0]['metadata']['name'] if items else '')
         typeset queryResult=""
         if [[ -n "${queryFrontendPod}" ]]; then
             queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryFrontendPod}" \
-                -- wget -qO- --no-check-certificate \
+                -- curl -sk
                 "http://localhost:9090/api/v1/query?query=up")" || true
         fi
 
@@ -546,7 +546,7 @@ print(items[0]['metadata']['name'] if items else '')
             fi
             if [[ -n "${queryPod}" ]]; then
                 queryResult="$(oc exec -n "${OBS_NAMESPACE}" "${queryPod}" \
-                    -- wget -qO- --no-check-certificate \
+                    -- curl -sk
                     "http://localhost:9090/api/v1/query?query=up")" || true
             fi
         fi