From 21883933b4fc47b5dda906fead02737b72e0db37 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Wed, 22 Jul 2026 20:34:18 +0000 Subject: [PATCH 1/7] INTEROP-9357: Point OPP observability tests at maintained main branch The acm-tests-observability step in OPP workflows has been running a deprecated, unmaintained test suite from old release branches of stolostron/multicluster-observability-operator. This generates spurious firewatch tickets (6 since December 2025). Create a new ci-operator config (main__opp variant) to build the OPP observability test image from the main branch, which contains the actively maintained test suite. Update all 4 consumer configs to source multicluster-observability-operator-opp from acm-qe/main instead of version-specific image streams (2.15, 2.16, 5.0). The @ocpInterop Ginkgo tags used by the step are confirmed present on main (50 occurrences across 14 test files). Fixes: INTEROP-9357 --- ...-testing-master__acm-cnv-ocp-4.21-p2p.yaml | 2 +- ...-testing-master__acm-cnv-ocp-4.22-p2p.yaml | 2 +- ...ster-observability-operator-main__opp.yaml | 23 +++++++ ...stron-policy-collection-main__ocp4.22.yaml | 2 +- ...ostron-policy-collection-main__ocp5.0.yaml | 2 +- ...servability-operator-main-postsubmits.yaml | 63 +++++++++++++++++++ ...bservability-operator-main-presubmits.yaml | 58 +++++++++++++++++ 7 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml index e607713d871ca..be92f8d8ea1a0 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml @@ -20,7 +20,7 @@ base_images: namespace: acm-qe tag: fetch-managed-clusters multicluster-observability-operator-opp: - name: "2.15" + name: main namespace: acm-qe tag: multicluster-observability-operator-opp openshift-virtualization-tests: diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml index c3ae777817854..f31d85a599bf2 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml @@ -28,7 +28,7 @@ base_images: namespace: ocp tag: installer multicluster-observability-operator-opp: - name: "2.16" + name: main namespace: acm-qe tag: multicluster-observability-operator-opp openshift-virtualization-tests: diff --git a/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml new file mode 100644 index 0000000000000..8d43b43193425 --- /dev/null +++ b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml @@ -0,0 +1,23 @@ +build_root: + image_stream_tag: + name: release + namespace: openshift + tag: rhel-8-release-golang-1.17-openshift-4.11 +images: + items: + - dockerfile_path: Dockerfile.interop + to: multicluster-observability-operator-opp +promotion: + to: + - name: main + namespace: acm-qe +resources: + '*': + requests: + cpu: 100m + memory: 1Gi +zz_generated_metadata: + branch: main + org: stolostron + repo: multicluster-observability-operator + variant: opp diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml index d1a27998e1f04..115b05923300e 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml @@ -20,7 +20,7 @@ base_images: namespace: acm-qe tag: fetch-managed-clusters multicluster-observability-operator-opp: - name: "2.16" + name: main namespace: acm-qe tag: multicluster-observability-operator-opp ocs-ci-tests: diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml index 4de0e90731551..3a2470ef0b877 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml @@ -20,7 +20,7 @@ base_images: namespace: acm-qe tag: fetch-managed-clusters multicluster-observability-operator-opp: - name: "2.17" + name: main namespace: acm-qe tag: multicluster-observability-operator-opp ocs-ci-tests: diff --git a/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-postsubmits.yaml b/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-postsubmits.yaml index 4575bfa58c3f0..4ea28afb02c4f 100644 --- a/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-postsubmits.yaml +++ b/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-postsubmits.yaml @@ -140,3 +140,66 @@ postsubmits: - name: result-aggregator secret: secretName: result-aggregator + - agent: kubernetes + always_run: true + branches: + - ^main$ + cluster: build04 + decorate: true + decoration_config: + sparse_checkout_files: + - Dockerfile.interop + labels: + ci-operator.openshift.io/is-promotion: "true" + ci-operator.openshift.io/variant: opp + ci.openshift.io/generator: prowgen + max_concurrency: 1 + name: branch-ci-stolostron-multicluster-observability-operator-main-opp-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --image-mirror-push-secret=/etc/push-secret/.dockerconfigjson + - --promote + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --variant=opp + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/push-secret + name: push-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: push-secret + secret: + secretName: registry-push-credentials-ci-central + - name: result-aggregator + secret: + secretName: result-aggregator diff --git a/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-presubmits.yaml b/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-presubmits.yaml index b6aeff818d15a..54bb652fad673 100644 --- a/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-presubmits.yaml +++ b/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-presubmits.yaml @@ -219,6 +219,64 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )ocm-ci-rbac,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + cluster: build08 + context: ci/prow/opp-images + decorate: true + decoration_config: + sparse_checkout_files: + - Dockerfile.interop + labels: + ci-operator.openshift.io/variant: opp + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-stolostron-multicluster-observability-operator-main-opp-images + rerun_command: /test opp-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --variant=opp + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )opp-images,?($|\s.*) - agent: kubernetes always_run: true branches: From 5a5b4e19e1a14259d610979d46333c673023f8d2 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Thu, 23 Jul 2026 15:14:15 +0000 Subject: [PATCH 2/7] INTEROP-9357: Update build_root to Go 1.25 and use root image for Dockerfile.interop Update the ci-operator config to use rhel-9-golang-1.25-openshift-4.22 build_root matching the upstream go.mod requirement of Go 1.25. Add from: root to the Dockerfile.interop image entry so ci-operator replaces the outdated Go 1.21 FROM with the build_root Go 1.25 image. Co-Authored-By: Claude Opus 4.6 --- ...olostron-multicluster-observability-operator-main__opp.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml index 8d43b43193425..56ce993f82613 100644 --- a/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml +++ b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml @@ -2,10 +2,11 @@ build_root: image_stream_tag: name: release namespace: openshift - tag: rhel-8-release-golang-1.17-openshift-4.11 + tag: rhel-9-golang-1.25-openshift-4.22 images: items: - dockerfile_path: Dockerfile.interop + from: root to: multicluster-observability-operator-opp promotion: to: From ee817b02383a9c17497a1a1667ef8e19d6a6b34d Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 23 Jul 2026 11:30:00 -0500 Subject: [PATCH 3/7] fix: use stolostron/builder:go1.25-linux build root for main__opp The previous build root (openshift/release:rhel-9-golang-1.25-openshift-4.22) does not exist in the CI image stream. Switch to stolostron/builder:go1.25-linux which is the same build root used by the main (non-opp) config and is known to exist. Also remove `from: root` to let Dockerfile.interop use its own FROM, matching the pattern used by existing __opp variant configs. --- ...tron-multicluster-observability-operator-main__opp.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml index 56ce993f82613..cc47ab4d7c391 100644 --- a/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml +++ b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml @@ -1,12 +1,11 @@ build_root: image_stream_tag: - name: release - namespace: openshift - tag: rhel-9-golang-1.25-openshift-4.22 + name: builder + namespace: stolostron + tag: go1.25-linux images: items: - dockerfile_path: Dockerfile.interop - from: root to: multicluster-observability-operator-opp promotion: to: From 9eb9b22bd79568584592703f356e838ea73649c0 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Thu, 23 Jul 2026 12:19:43 -0500 Subject: [PATCH 4/7] fix: use openshift/release:rhel-9-release-golang-1.25-openshift-4.22 build_root The previous commit used stolostron/builder:go1.25-linux which is not in the CI image stream. Switch to the correct openshift/release tag (rhel-9-release-golang-1.25-openshift-4.22) and add from: root so ci-operator replaces the Dockerfile base image with the build_root Go 1.25 toolchain. Ref: INTEROP-9357 --- ...tron-multicluster-observability-operator-main__opp.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml index cc47ab4d7c391..140499e33311f 100644 --- a/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml +++ b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml @@ -1,11 +1,12 @@ build_root: image_stream_tag: - name: builder - namespace: stolostron - tag: go1.25-linux + name: release + namespace: openshift + tag: rhel-9-release-golang-1.25-openshift-4.22 images: items: - dockerfile_path: Dockerfile.interop + from: root to: multicluster-observability-operator-opp promotion: to: From c3f0591c045b5a6a7e20c3616c38492fa2ee8167 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 29 Jul 2026 13:54:25 -0500 Subject: [PATCH 5/7] Apply mpitt conventions to preflight step script - Rename functions from snake_case to PascalCase - Convert local variables from snake_case/local to camelCase/typeset - Add terminal true to all functions for safe return under errexit --- .../interop-opp-preflight-commands.sh | 307 +++++++++--------- 1 file changed, 149 insertions(+), 158 deletions(-) diff --git a/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh b/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh index 38911179bf4bb..f11b8d892696c 100755 --- a/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh +++ b/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh @@ -11,7 +11,6 @@ export XDG_RUNTIME_DIR="${HOME}/run" export REGISTRY_AUTH_PREFERENCE=podman mkdir -p "${XDG_RUNTIME_DIR}" -# Ensure jq is available (not shipped in the cli base image) if ! command -v jq &>/dev/null; then echo "jq not found; installing..." dnf install -y -q jq 2>/dev/null || yum install -y -q jq 2>/dev/null || { @@ -26,7 +25,7 @@ mkdir -p "${REPORT_DIR}" CHECKS_FAILED=0 -debug_on_exit() { +DebugOnExit() { if (( EXIT_CODE != 0 )); then echo -e "\n### DEBUG: Pre-flight failure diagnostics ###\n" echo -e "\n# ClusterVersion\n$(oc get clusterversion 2>/dev/null || echo 'unavailable')" @@ -38,9 +37,10 @@ debug_on_exit() { echo -e "\n# Pre-flight report:\n$(cat "${REPORT_FILE}")" fi fi + true } -trap 'EXIT_CODE=$?; debug_on_exit' EXIT TERM +trap 'EXIT_CODE=$?; DebugOnExit' EXIT TERM # ────────────────────────────────────────────────────────────────────── # Known removed / deprecated APIs per OCP minor version. @@ -60,7 +60,7 @@ REMOVED_APIS["18"]="flowcontrol.apiserver.k8s.io/v1beta3/FlowSchema flowcontrol. # ────────────────────────────────────────────────────────────────────── # OPP operator compatibility matrix. -# Maps OCP major.minor version to minimum required operator major.minor. +# Maps OCP minor version to minimum required operator major.minor. # Format: "operator_csv_prefix:min_major.min_minor" # ────────────────────────────────────────────────────────────────────── declare -A OPP_COMPAT @@ -78,189 +78,184 @@ OPP_COMPAT["5.0"]="advanced-cluster-management:2.17 quay-operator:3.17" # ────────────────────────────────────────────────────────────────────── # Utility: append a check result to the JSON report # ────────────────────────────────────────────────────────────────────── -init_report() { +InitReport() { cat > "${REPORT_FILE}" <<'EOFJSON' { "preflight_checks": [] } EOFJSON + true } -append_check() { - local name="${1}" status="${2}" details="${3}" - local tmp - tmp="$(mktemp)" - jq --arg n "${name}" --arg s "${status}" --arg d "${details}" \ +AppendCheck() { + typeset checkName="${1}" checkStatus="${2}" checkDetails="${3}" + typeset tmpFile + tmpFile="$(mktemp)" + jq --arg n "${checkName}" --arg s "${checkStatus}" --arg d "${checkDetails}" \ '.preflight_checks += [{"check": $n, "status": $s, "details": $d}]' \ - "${REPORT_FILE}" > "${tmp}" && mv "${tmp}" "${REPORT_FILE}" + "${REPORT_FILE}" > "${tmpFile}" && mv "${tmpFile}" "${REPORT_FILE}" + true } # ────────────────────────────────────────────────────────────────────── # Check 1: API deprecation scan # ────────────────────────────────────────────────────────────────────── -check_api_deprecations() { +CheckApiDeprecations() { echo "=== Check 1: API deprecation scan ===" - local target_minor="${1}" - local target_major="${2:-4}" - local flagged="" found_count=0 + typeset targetMinor="${1}" + typeset ocpDisplay="${2:-4.${targetMinor}}" + typeset targetMajor="${ocpDisplay%%.*}" + typeset flagged="" foundCount=0 - # Collect available API resources on the cluster - local cluster_apis - cluster_apis="$(oc api-resources --no-headers 2>/dev/null)" || { + typeset clusterApis + clusterApis="$(oc api-resources --no-headers 2>/dev/null)" || { echo "WARNING: Failed to list API resources" - append_check "api_deprecation_scan" "warn" "Could not list cluster API resources" + AppendCheck "api_deprecation_scan" "warn" "Could not list cluster API resources" return 0 } - # Check all versions up to and including the target for minor in "${!REMOVED_APIS[@]}"; do - if (( target_major > 4 || minor <= target_minor )); then - for api_entry in ${REMOVED_APIS[${minor}]}; do - local api_version api_kind - api_kind="${api_entry##*/}" - api_version="${api_entry%/*}" - - # Check if this deprecated API version+kind is still served - if echo "${cluster_apis}" | grep -qw "${api_kind}" && \ - oc api-resources --api-group="${api_version%%/*}" 2>/dev/null | grep -q "${api_version#*/}"; then - # Check if any OPP workloads reference this API - local opp_usage - opp_usage="$(oc get "${api_kind}" -A --no-headers 2>/dev/null | head -5)" || true - if [[ -n "${opp_usage}" ]]; then - flagged="${flagged} - ${api_version}/${api_kind} (removed in ${target_major}.${minor})\n" - (( found_count += 1 )) + if (( targetMajor > 4 || minor <= targetMinor )); then + for apiEntry in ${REMOVED_APIS[${minor}]}; do + typeset apiVersion apiKind + apiKind="${apiEntry##*/}" + apiVersion="${apiEntry%/*}" + + if echo "${clusterApis}" | grep -qw "${apiKind}" && \ + oc api-resources --api-group="${apiVersion%%/*}" 2>/dev/null | grep -q "${apiVersion#*/}"; then + typeset oppUsage + oppUsage="$(oc get "${apiKind}" -A --no-headers 2>/dev/null | head -5)" || true + if [[ -n "${oppUsage}" ]]; then + flagged="${flagged} - ${apiVersion}/${apiKind} (removed in ${targetMajor}.${minor})\n" + (( foundCount += 1 )) fi fi done fi done - if (( found_count > 0 )); then - echo -e "WARNING: Found ${found_count} deprecated API(s) still in use:\n${flagged}" - append_check "api_deprecation_scan" "warn" "Found ${found_count} deprecated API(s) in use: ${flagged}" + if (( foundCount > 0 )); then + echo -e "WARNING: Found ${foundCount} deprecated API(s) still in use:\n${flagged}" + AppendCheck "api_deprecation_scan" "warn" "Found ${foundCount} deprecated API(s) in use: ${flagged}" else - echo "No deprecated APIs detected for target version ${target_major}.${target_minor}" - append_check "api_deprecation_scan" "pass" "No deprecated APIs detected for ${target_major}.${target_minor}" + echo "No deprecated APIs detected for target version ${ocpDisplay}" + AppendCheck "api_deprecation_scan" "pass" "No deprecated APIs detected for ${ocpDisplay}" fi + true } # ────────────────────────────────────────────────────────────────────── # Check 2: OPP compatibility matrix # ────────────────────────────────────────────────────────────────────── -check_opp_compatibility() { +CheckOppCompatibility() { echo -e "\n=== Check 2: OPP operator compatibility matrix ===" - local target_minor="${1}" - local target_major="${2:-4}" - local compat_key="${target_major}.${target_minor}" - local compat_spec="${OPP_COMPAT[${compat_key}]:-}" - local all_csvs failed=0 + typeset ocpKey="${1}" + typeset compatSpec="${OPP_COMPAT[${ocpKey}]:-}" + typeset allCsvs + typeset failed=0 - all_csvs="$(oc get csv -A --no-headers 2>/dev/null)" || { + allCsvs="$(oc get csv -A --no-headers 2>/dev/null)" || { echo >&2 "Failed to retrieve CSVs" - append_check "opp_compatibility_matrix" "fail" "Could not list CSVs" + AppendCheck "opp_compatibility_matrix" "fail" "Could not list CSVs" (( CHECKS_FAILED += 1 )) return 0 } - if [[ -z "${compat_spec}" ]]; then - echo "No compatibility matrix entry for target ${compat_key}; skipping version check" - append_check "opp_compatibility_matrix" "skip" "No matrix entry for OCP ${target_major}.${target_minor}" + if [[ -z "${compatSpec}" ]]; then + echo "No compatibility matrix entry for OCP ${ocpKey}; skipping version check" + AppendCheck "opp_compatibility_matrix" "skip" "No matrix entry for OCP ${ocpKey}" return 0 fi - local details="" - for entry in ${compat_spec}; do - local op_prefix="${entry%%:*}" - local min_version="${entry##*:}" - local min_major min_minor - min_major="${min_version%%.*}" - min_minor="${min_version##*.}" - - # Find installed CSV for this operator - local csv_line csv_name installed_version - csv_line="$(echo "${all_csvs}" | grep "${op_prefix}" | head -1)" || true - if [[ -z "${csv_line}" ]]; then - echo >&2 "Operator not found: ${op_prefix}" - details="${details}${op_prefix}: NOT INSTALLED; " + typeset details="" + for entry in ${compatSpec}; do + typeset opPrefix="${entry%%:*}" + typeset minVersion="${entry##*:}" + typeset minMajor minMinor + minMajor="${minVersion%%.*}" + minMinor="${minVersion##*.}" + + typeset csvLine csvName installedVersion + csvLine="$(echo "${allCsvs}" | grep "${opPrefix}" | head -1)" || true + if [[ -z "${csvLine}" ]]; then + echo >&2 "Operator not found: ${opPrefix}" + details="${details}${opPrefix}: NOT INSTALLED; " (( failed += 1 )) continue fi - csv_name="$(echo "${csv_line}" | awk '{print $2}')" - # Extract version: strip operator name prefix, keep digits - installed_version="$(echo "${csv_name}" | grep -oE '[0-9]+\.[0-9]+' | head -1)" || true - if [[ -z "${installed_version}" ]]; then - echo >&2 "Operator ${op_prefix}: could not parse version from CSV ${csv_name}" - details="${details}${op_prefix}: version unparseable from ${csv_name}; " + csvName="$(echo "${csvLine}" | awk '{print $2}')" + installedVersion="$(echo "${csvName}" | grep -oE '[0-9]+\.[0-9]+' | head -1)" || true + if [[ -z "${installedVersion}" ]]; then + echo >&2 "Operator ${opPrefix}: could not parse version from CSV ${csvName}" + details="${details}${opPrefix}: version unparseable from ${csvName}; " (( failed += 1 )) continue fi - local inst_major inst_minor - inst_major="${installed_version%%.*}" - inst_minor="${installed_version##*.}" + typeset instMajor instMinor + instMajor="${installedVersion%%.*}" + instMinor="${installedVersion##*.}" - if (( inst_major < min_major || (inst_major == min_major && inst_minor < min_minor) )); then - echo >&2 "Operator ${op_prefix} version ${installed_version} is below minimum ${min_version} for OCP ${target_major}.${target_minor}" - details="${details}${op_prefix}: ${installed_version} < ${min_version} (INCOMPATIBLE); " + if (( instMajor < minMajor || (instMajor == minMajor && instMinor < minMinor) )); then + echo >&2 "Operator ${opPrefix} version ${installedVersion} is below minimum ${minVersion} for OCP ${ocpKey}" + details="${details}${opPrefix}: ${installedVersion} < ${minVersion} (INCOMPATIBLE); " (( failed += 1 )) else - echo "Operator ${op_prefix}: version ${installed_version} >= ${min_version} (OK)" - details="${details}${op_prefix}: ${installed_version} >= ${min_version} (OK); " + echo "Operator ${opPrefix}: version ${installedVersion} >= ${minVersion} (OK)" + details="${details}${opPrefix}: ${installedVersion} >= ${minVersion} (OK); " fi done if (( failed > 0 )); then echo >&2 "${failed} operator(s) failed compatibility check" - append_check "opp_compatibility_matrix" "fail" "${details}" + AppendCheck "opp_compatibility_matrix" "fail" "${details}" (( CHECKS_FAILED += 1 )) else - echo "All OPP operators are compatible with OCP ${target_major}.${target_minor}" - append_check "opp_compatibility_matrix" "pass" "${details}" + echo "All OPP operators are compatible with OCP ${ocpKey}" + AppendCheck "opp_compatibility_matrix" "pass" "${details}" fi + true } # ────────────────────────────────────────────────────────────────────── # Check 3: Cluster health baseline # ────────────────────────────────────────────────────────────────────── -check_cluster_health() { +CheckClusterHealth() { echo -e "\n=== Check 3: Cluster health baseline ===" - local failed=0 details="" + typeset failed=0 details="" - # 3a. Node health echo "Checking node health..." - local unready_nodes - unready_nodes="$(oc get node --no-headers 2>/dev/null | awk '$2 != "Ready" {print $1}')" || true - if [[ -n "${unready_nodes}" ]]; then - echo >&2 "Not-Ready nodes: ${unready_nodes}" - details="${details}unready_nodes: ${unready_nodes}; " + typeset unreadyNodes + unreadyNodes="$(oc get node --no-headers 2>/dev/null | awk '$2 != "Ready" {print $1}')" || true + if [[ -n "${unreadyNodes}" ]]; then + echo >&2 "Not-Ready nodes: ${unreadyNodes}" + details="${details}unready_nodes: ${unreadyNodes}; " (( failed += 1 )) else - local node_count - node_count="$(oc get node --no-headers 2>/dev/null | wc -l)" - echo "All ${node_count} nodes Ready" - details="${details}nodes: all ${node_count} ready; " + typeset nodeCount + nodeCount="$(oc get node --no-headers 2>/dev/null | wc -l)" + echo "All ${nodeCount} nodes Ready" + details="${details}nodes: all ${nodeCount} ready; " fi - # 3b. ClusterOperator health echo "Checking ClusterOperator health..." - local unhealthy_co - unhealthy_co="$(oc get co --no-headers 2>/dev/null | awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}')" || true - if [[ -n "${unhealthy_co}" ]]; then - echo >&2 "Unhealthy ClusterOperators: ${unhealthy_co}" - details="${details}unhealthy_co: ${unhealthy_co}; " + typeset unhealthyCo + unhealthyCo="$(oc get co --no-headers 2>/dev/null | awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}')" || true + if [[ -n "${unhealthyCo}" ]]; then + echo >&2 "Unhealthy ClusterOperators: ${unhealthyCo}" + details="${details}unhealthy_co: ${unhealthyCo}; " (( failed += 1 )) else echo "All ClusterOperators healthy" details="${details}cluster_operators: all healthy; " fi - # 3c. CVO conditions echo "Checking ClusterVersion conditions..." - local avail progressing degraded + typeset avail progressing degraded avail="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null)" || true progressing="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Progressing")].status}' 2>/dev/null)" || true degraded="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Degraded")].status}' 2>/dev/null)" || true @@ -273,78 +268,73 @@ check_cluster_health() { details="${details}cvo: healthy; " fi - # 3d. Firing alerts (excluding Watchdog and AlertmanagerReceiversNotConfigured) echo "Checking for firing alerts..." - local firing_alerts="" - firing_alerts="$(oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- \ + typeset firingAlerts="" + firingAlerts="$(oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- \ curl -s 'http://localhost:9090/api/v1/alerts' 2>/dev/null | \ jq -r '.data.alerts[]? | select(.state=="firing") | select(.labels.alertname != "Watchdog") | select(.labels.alertname != "AlertmanagerReceiversNotConfigured") | .labels.alertname' 2>/dev/null | \ sort -u)" || true - if [[ -n "${firing_alerts}" ]]; then - local alert_count - alert_count="$(echo "${firing_alerts}" | wc -l)" - echo "WARNING: ${alert_count} alert(s) firing: ${firing_alerts}" - details="${details}firing_alerts: ${alert_count} (${firing_alerts}); " - # Alerts are a warning, not a hard failure + if [[ -n "${firingAlerts}" ]]; then + typeset alertCount + alertCount="$(echo "${firingAlerts}" | wc -l)" + echo "WARNING: ${alertCount} alert(s) firing: ${firingAlerts}" + details="${details}firing_alerts: ${alertCount} (${firingAlerts}); " else echo "No critical alerts firing" details="${details}alerts: none firing; " fi - # Save node and CO snapshot for baseline oc get nodes -o json > "${REPORT_DIR}/nodes-baseline.json" 2>/dev/null || true oc get co -o json > "${REPORT_DIR}/co-baseline.json" 2>/dev/null || true if (( failed > 0 )); then echo >&2 "Cluster health baseline: ${failed} issue(s) found" - append_check "cluster_health_baseline" "fail" "${details}" + AppendCheck "cluster_health_baseline" "fail" "${details}" (( CHECKS_FAILED += 1 )) else echo "Cluster health baseline: all checks passed" - append_check "cluster_health_baseline" "pass" "${details}" + AppendCheck "cluster_health_baseline" "pass" "${details}" fi + true } # ────────────────────────────────────────────────────────────────────── # Check 4: MachineConfigPool readiness # ────────────────────────────────────────────────────────────────────── -check_mcp_readiness() { +CheckMcpReadiness() { echo -e "\n=== Check 4: MachineConfigPool readiness ===" - local failed=0 details="" + typeset failed=0 details="" - # Check MCP conditions: Updated=True, Updating=False, Degraded=False - local mcp_issues - mcp_issues="$(oc get machineconfigpools --no-headers 2>/dev/null | \ + typeset mcpIssues + mcpIssues="$(oc get machineconfigpools --no-headers 2>/dev/null | \ awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}')" || true - if [[ -n "${mcp_issues}" ]]; then - echo >&2 "Unhealthy MachineConfigPools: ${mcp_issues}" - details="unhealthy_mcps: ${mcp_issues}; " + if [[ -n "${mcpIssues}" ]]; then + echo >&2 "Unhealthy MachineConfigPools: ${mcpIssues}" + details="unhealthy_mcps: ${mcpIssues}; " (( failed += 1 )) - # Dump details for each unhealthy MCP - for mcp in ${mcp_issues}; do + for mcp in ${mcpIssues}; do echo -e "\n### MCP ${mcp} ###" oc describe machineconfigpool "${mcp}" 2>/dev/null || true done else - local mcp_count - mcp_count="$(oc get machineconfigpools --no-headers 2>/dev/null | wc -l)" - echo "All ${mcp_count} MachineConfigPools are updated and not degraded" - details="all ${mcp_count} MCPs healthy (Updated=True, Updating=False, Degraded=False); " + typeset mcpCount + mcpCount="$(oc get machineconfigpools --no-headers 2>/dev/null | wc -l)" + echo "All ${mcpCount} MachineConfigPools are updated and not degraded" + details="all ${mcpCount} MCPs healthy (Updated=True, Updating=False, Degraded=False); " fi - # Check that machine counts match (ready == desired) - local mismatch="" + typeset mismatch="" while IFS= read -r line; do - local mcp_name ready desired - mcp_name="$(echo "${line}" | awk '{print $1}')" - desired="$(echo "${line}" | awk '{print $6}')" + typeset mcpName ready desired + mcpName="$(echo "${line}" | awk '{print $1}')" ready="$(echo "${line}" | awk '{print $7}')" + desired="$(echo "${line}" | awk '{print $6}')" if [[ -n "${ready}" && -n "${desired}" && "${ready}" != "${desired}" ]]; then - mismatch="${mismatch}${mcp_name} (ready=${ready}, desired=${desired}); " + mismatch="${mismatch}${mcpName} (ready=${ready}, desired=${desired}); " fi done < <(oc get machineconfigpools --no-headers 2>/dev/null || true) @@ -354,28 +344,28 @@ check_mcp_readiness() { (( failed += 1 )) fi - # Save MCP snapshot oc get machineconfigpools -o json > "${REPORT_DIR}/mcp-baseline.json" 2>/dev/null || true if (( failed > 0 )); then echo >&2 "MachineConfigPool readiness: ${failed} issue(s) found" - append_check "mcp_readiness" "fail" "${details}" + AppendCheck "mcp_readiness" "fail" "${details}" (( CHECKS_FAILED += 1 )) else echo "MachineConfigPool readiness: all checks passed" - append_check "mcp_readiness" "pass" "${details}" + AppendCheck "mcp_readiness" "pass" "${details}" fi + true } # ────────────────────────────────────────────────────────────────────── # Main # ────────────────────────────────────────────────────────────────────── -main() { +Main() { if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then export KUBECONFIG="${SHARED_DIR}/kubeconfig" fi - local target="${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE:-}" + typeset target="${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE:-}" if [[ -z "${target}" ]]; then echo >&2 "OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE is not set; cannot determine upgrade target" exit 3 @@ -384,31 +374,31 @@ main() { KUBECONFIG="" oc registry login - local target_version target_major target_minor - target_version="$(oc adm release info "${target}" --output=json | jq -r '.metadata.version')" - target_major="$(echo "${target_version}" | cut -f1 -d.)" - target_minor="$(echo "${target_version}" | cut -f2 -d.)" - echo "Target OCP version: ${target_version} (major: ${target_major}, minor: ${target_minor})" + typeset targetVersion targetMajor targetMinor ocpXy + targetVersion="$(oc adm release info "${target}" --output=json | jq -r '.metadata.version')" + targetMajor="$(echo "${targetVersion}" | cut -f1 -d.)" + targetMinor="$(echo "${targetVersion}" | cut -f2 -d.)" + ocpXy="${targetMajor}.${targetMinor}" + echo "Target OCP version: ${targetVersion} (${ocpXy})" - local source_version - source_version="$(oc get clusterversion --no-headers | awk '{print $2}')" - echo "Source OCP version: ${source_version}" + typeset sourceVersion + sourceVersion="$(oc get clusterversion --no-headers | awk '{print $2}')" + echo "Source OCP version: ${sourceVersion}" echo -e "\n=== Starting OPP pre-flight validation ===\n" - init_report + InitReport - # Add metadata to report - local tmp - tmp="$(mktemp)" - jq --arg tv "${target_version}" --arg sv "${source_version}" --arg ti "${target}" \ + typeset tmpFile + tmpFile="$(mktemp)" + jq --arg tv "${targetVersion}" --arg sv "${sourceVersion}" --arg ti "${target}" \ '. + {"target_version": $tv, "source_version": $sv, "target_image": $ti, "timestamp": now | tostring}' \ - "${REPORT_FILE}" > "${tmp}" && mv "${tmp}" "${REPORT_FILE}" + "${REPORT_FILE}" > "${tmpFile}" && mv "${tmpFile}" "${REPORT_FILE}" - check_api_deprecations "${target_minor}" "${target_major}" - check_opp_compatibility "${target_minor}" "${target_major}" - check_cluster_health - check_mcp_readiness + CheckApiDeprecations "${targetMinor}" "${ocpXy}" + CheckOppCompatibility "${ocpXy}" + CheckClusterHealth + CheckMcpReadiness echo -e "\n=== Pre-flight summary ===" jq '.' "${REPORT_FILE}" @@ -420,6 +410,7 @@ main() { fi echo "Pre-flight validation PASSED: all checks succeeded" + true } -main "$@" +Main "$@" From c39bb2d4c92c087067484f177dcc0bab7e21bfb7 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Mon, 10 Aug 2026 09:31:46 -0500 Subject: [PATCH 6/7] fix: use ACM 2.17 for OCP 4.x jobs, reserve main for 5.0 The main branch of stolostron/multicluster-observability-operator tracks ACM 5.0.0 development. OPP testing on OCP 4.21/4.22 uses ACM 2.17, so those configs must reference the acm-qe/2.17 imagestream (built from release-2.17 branch). Changes: - OCP 4.21 and 4.22 configs: name: main -> name: "2.17" - OCP 5.0 config: unchanged (main is correct) - main__opp.yaml: remove from: root to match working release-branch pattern (preserves Dockerfile's own FROM instruction) --- .../RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml | 2 +- .../RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml | 2 +- ...tolostron-multicluster-observability-operator-main__opp.yaml | 1 - .../stolostron-policy-collection-main__ocp4.22.yaml | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml index be92f8d8ea1a0..eff8e0597fc62 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml @@ -20,7 +20,7 @@ base_images: namespace: acm-qe tag: fetch-managed-clusters multicluster-observability-operator-opp: - name: main + name: "2.17" namespace: acm-qe tag: multicluster-observability-operator-opp openshift-virtualization-tests: diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml index f31d85a599bf2..dc23f47c93737 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml @@ -28,7 +28,7 @@ base_images: namespace: ocp tag: installer multicluster-observability-operator-opp: - name: main + name: "2.17" namespace: acm-qe tag: multicluster-observability-operator-opp openshift-virtualization-tests: diff --git a/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml index 140499e33311f..23f542aa27637 100644 --- a/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml +++ b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml @@ -6,7 +6,6 @@ build_root: images: items: - dockerfile_path: Dockerfile.interop - from: root to: multicluster-observability-operator-opp promotion: to: diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml index 115b05923300e..c16eed769eb6e 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml @@ -20,7 +20,7 @@ base_images: namespace: acm-qe tag: fetch-managed-clusters multicluster-observability-operator-opp: - name: main + name: "2.17" namespace: acm-qe tag: multicluster-observability-operator-opp ocs-ci-tests: From 787da40449543ed5e0464aabcffd18f4d14ba0c7 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Mon, 10 Aug 2026 10:02:15 -0500 Subject: [PATCH 7/7] fix: use ACM 2.17 image stream for ocp5.0 consumer Address review feedback from vboulos: align the ocp5.0 consumer config with the other consumer configs by using the "2.17" image stream name consistently. --- .../stolostron-policy-collection-main__ocp5.0.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml index 3a2470ef0b877..4de0e90731551 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml @@ -20,7 +20,7 @@ base_images: namespace: acm-qe tag: fetch-managed-clusters multicluster-observability-operator-opp: - name: main + name: "2.17" namespace: acm-qe tag: multicluster-observability-operator-opp ocs-ci-tests: