diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml index e607713d871ca..eff8e0597fc62 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.21-p2p.yaml @@ -20,7 +20,7 @@ base_images: namespace: acm-qe tag: fetch-managed-clusters multicluster-observability-operator-opp: - name: "2.15" + name: "2.17" namespace: acm-qe tag: multicluster-observability-operator-opp openshift-virtualization-tests: diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml index c3ae777817854..dc23f47c93737 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__acm-cnv-ocp-4.22-p2p.yaml @@ -28,7 +28,7 @@ base_images: namespace: ocp tag: installer multicluster-observability-operator-opp: - name: "2.16" + name: "2.17" namespace: acm-qe tag: multicluster-observability-operator-opp openshift-virtualization-tests: diff --git a/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml new file mode 100644 index 0000000000000..23f542aa27637 --- /dev/null +++ b/ci-operator/config/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main__opp.yaml @@ -0,0 +1,23 @@ +build_root: + image_stream_tag: + name: release + namespace: openshift + tag: rhel-9-release-golang-1.25-openshift-4.22 +images: + items: + - dockerfile_path: Dockerfile.interop + to: multicluster-observability-operator-opp +promotion: + to: + - name: main + namespace: acm-qe +resources: + '*': + requests: + cpu: 100m + memory: 1Gi +zz_generated_metadata: + branch: main + org: stolostron + repo: multicluster-observability-operator + variant: opp diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml index d1a27998e1f04..c16eed769eb6e 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml @@ -20,7 +20,7 @@ base_images: namespace: acm-qe tag: fetch-managed-clusters multicluster-observability-operator-opp: - name: "2.16" + name: "2.17" namespace: acm-qe tag: multicluster-observability-operator-opp ocs-ci-tests: diff --git a/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-postsubmits.yaml b/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-postsubmits.yaml index 4575bfa58c3f0..4ea28afb02c4f 100644 --- a/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-postsubmits.yaml +++ b/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-postsubmits.yaml @@ -140,3 +140,66 @@ postsubmits: - name: result-aggregator secret: secretName: result-aggregator + - agent: kubernetes + always_run: true + branches: + - ^main$ + cluster: build04 + decorate: true + decoration_config: + sparse_checkout_files: + - Dockerfile.interop + labels: + ci-operator.openshift.io/is-promotion: "true" + ci-operator.openshift.io/variant: opp + ci.openshift.io/generator: prowgen + max_concurrency: 1 + name: branch-ci-stolostron-multicluster-observability-operator-main-opp-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --image-mirror-push-secret=/etc/push-secret/.dockerconfigjson + - --promote + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --variant=opp + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/push-secret + name: push-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: push-secret + secret: + secretName: registry-push-credentials-ci-central + - name: result-aggregator + secret: + secretName: result-aggregator diff --git a/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-presubmits.yaml b/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-presubmits.yaml index b6aeff818d15a..54bb652fad673 100644 --- a/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-presubmits.yaml +++ b/ci-operator/jobs/stolostron/multicluster-observability-operator/stolostron-multicluster-observability-operator-main-presubmits.yaml @@ -219,6 +219,64 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )ocm-ci-rbac,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + cluster: build08 + context: ci/prow/opp-images + decorate: true + decoration_config: + sparse_checkout_files: + - Dockerfile.interop + labels: + ci-operator.openshift.io/variant: opp + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-stolostron-multicluster-observability-operator-main-opp-images + rerun_command: /test opp-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --variant=opp + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )opp-images,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh b/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh index 38911179bf4bb..f11b8d892696c 100755 --- a/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh +++ b/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh @@ -11,7 +11,6 @@ export XDG_RUNTIME_DIR="${HOME}/run" export REGISTRY_AUTH_PREFERENCE=podman mkdir -p "${XDG_RUNTIME_DIR}" -# Ensure jq is available (not shipped in the cli base image) if ! command -v jq &>/dev/null; then echo "jq not found; installing..." dnf install -y -q jq 2>/dev/null || yum install -y -q jq 2>/dev/null || { @@ -26,7 +25,7 @@ mkdir -p "${REPORT_DIR}" CHECKS_FAILED=0 -debug_on_exit() { +DebugOnExit() { if (( EXIT_CODE != 0 )); then echo -e "\n### DEBUG: Pre-flight failure diagnostics ###\n" echo -e "\n# ClusterVersion\n$(oc get clusterversion 2>/dev/null || echo 'unavailable')" @@ -38,9 +37,10 @@ debug_on_exit() { echo -e "\n# Pre-flight report:\n$(cat "${REPORT_FILE}")" fi fi + true } -trap 'EXIT_CODE=$?; debug_on_exit' EXIT TERM +trap 'EXIT_CODE=$?; DebugOnExit' EXIT TERM # ────────────────────────────────────────────────────────────────────── # Known removed / deprecated APIs per OCP minor version. @@ -60,7 +60,7 @@ REMOVED_APIS["18"]="flowcontrol.apiserver.k8s.io/v1beta3/FlowSchema flowcontrol. # ────────────────────────────────────────────────────────────────────── # OPP operator compatibility matrix. -# Maps OCP major.minor version to minimum required operator major.minor. +# Maps OCP minor version to minimum required operator major.minor. # Format: "operator_csv_prefix:min_major.min_minor" # ────────────────────────────────────────────────────────────────────── declare -A OPP_COMPAT @@ -78,189 +78,184 @@ OPP_COMPAT["5.0"]="advanced-cluster-management:2.17 quay-operator:3.17" # ────────────────────────────────────────────────────────────────────── # Utility: append a check result to the JSON report # ────────────────────────────────────────────────────────────────────── -init_report() { +InitReport() { cat > "${REPORT_FILE}" <<'EOFJSON' { "preflight_checks": [] } EOFJSON + true } -append_check() { - local name="${1}" status="${2}" details="${3}" - local tmp - tmp="$(mktemp)" - jq --arg n "${name}" --arg s "${status}" --arg d "${details}" \ +AppendCheck() { + typeset checkName="${1}" checkStatus="${2}" checkDetails="${3}" + typeset tmpFile + tmpFile="$(mktemp)" + jq --arg n "${checkName}" --arg s "${checkStatus}" --arg d "${checkDetails}" \ '.preflight_checks += [{"check": $n, "status": $s, "details": $d}]' \ - "${REPORT_FILE}" > "${tmp}" && mv "${tmp}" "${REPORT_FILE}" + "${REPORT_FILE}" > "${tmpFile}" && mv "${tmpFile}" "${REPORT_FILE}" + true } # ────────────────────────────────────────────────────────────────────── # Check 1: API deprecation scan # ────────────────────────────────────────────────────────────────────── -check_api_deprecations() { +CheckApiDeprecations() { echo "=== Check 1: API deprecation scan ===" - local target_minor="${1}" - local target_major="${2:-4}" - local flagged="" found_count=0 + typeset targetMinor="${1}" + typeset ocpDisplay="${2:-4.${targetMinor}}" + typeset targetMajor="${ocpDisplay%%.*}" + typeset flagged="" foundCount=0 - # Collect available API resources on the cluster - local cluster_apis - cluster_apis="$(oc api-resources --no-headers 2>/dev/null)" || { + typeset clusterApis + clusterApis="$(oc api-resources --no-headers 2>/dev/null)" || { echo "WARNING: Failed to list API resources" - append_check "api_deprecation_scan" "warn" "Could not list cluster API resources" + AppendCheck "api_deprecation_scan" "warn" "Could not list cluster API resources" return 0 } - # Check all versions up to and including the target for minor in "${!REMOVED_APIS[@]}"; do - if (( target_major > 4 || minor <= target_minor )); then - for api_entry in ${REMOVED_APIS[${minor}]}; do - local api_version api_kind - api_kind="${api_entry##*/}" - api_version="${api_entry%/*}" - - # Check if this deprecated API version+kind is still served - if echo "${cluster_apis}" | grep -qw "${api_kind}" && \ - oc api-resources --api-group="${api_version%%/*}" 2>/dev/null | grep -q "${api_version#*/}"; then - # Check if any OPP workloads reference this API - local opp_usage - opp_usage="$(oc get "${api_kind}" -A --no-headers 2>/dev/null | head -5)" || true - if [[ -n "${opp_usage}" ]]; then - flagged="${flagged} - ${api_version}/${api_kind} (removed in ${target_major}.${minor})\n" - (( found_count += 1 )) + if (( targetMajor > 4 || minor <= targetMinor )); then + for apiEntry in ${REMOVED_APIS[${minor}]}; do + typeset apiVersion apiKind + apiKind="${apiEntry##*/}" + apiVersion="${apiEntry%/*}" + + if echo "${clusterApis}" | grep -qw "${apiKind}" && \ + oc api-resources --api-group="${apiVersion%%/*}" 2>/dev/null | grep -q "${apiVersion#*/}"; then + typeset oppUsage + oppUsage="$(oc get "${apiKind}" -A --no-headers 2>/dev/null | head -5)" || true + if [[ -n "${oppUsage}" ]]; then + flagged="${flagged} - ${apiVersion}/${apiKind} (removed in ${targetMajor}.${minor})\n" + (( foundCount += 1 )) fi fi done fi done - if (( found_count > 0 )); then - echo -e "WARNING: Found ${found_count} deprecated API(s) still in use:\n${flagged}" - append_check "api_deprecation_scan" "warn" "Found ${found_count} deprecated API(s) in use: ${flagged}" + if (( foundCount > 0 )); then + echo -e "WARNING: Found ${foundCount} deprecated API(s) still in use:\n${flagged}" + AppendCheck "api_deprecation_scan" "warn" "Found ${foundCount} deprecated API(s) in use: ${flagged}" else - echo "No deprecated APIs detected for target version ${target_major}.${target_minor}" - append_check "api_deprecation_scan" "pass" "No deprecated APIs detected for ${target_major}.${target_minor}" + echo "No deprecated APIs detected for target version ${ocpDisplay}" + AppendCheck "api_deprecation_scan" "pass" "No deprecated APIs detected for ${ocpDisplay}" fi + true } # ────────────────────────────────────────────────────────────────────── # Check 2: OPP compatibility matrix # ────────────────────────────────────────────────────────────────────── -check_opp_compatibility() { +CheckOppCompatibility() { echo -e "\n=== Check 2: OPP operator compatibility matrix ===" - local target_minor="${1}" - local target_major="${2:-4}" - local compat_key="${target_major}.${target_minor}" - local compat_spec="${OPP_COMPAT[${compat_key}]:-}" - local all_csvs failed=0 + typeset ocpKey="${1}" + typeset compatSpec="${OPP_COMPAT[${ocpKey}]:-}" + typeset allCsvs + typeset failed=0 - all_csvs="$(oc get csv -A --no-headers 2>/dev/null)" || { + allCsvs="$(oc get csv -A --no-headers 2>/dev/null)" || { echo >&2 "Failed to retrieve CSVs" - append_check "opp_compatibility_matrix" "fail" "Could not list CSVs" + AppendCheck "opp_compatibility_matrix" "fail" "Could not list CSVs" (( CHECKS_FAILED += 1 )) return 0 } - if [[ -z "${compat_spec}" ]]; then - echo "No compatibility matrix entry for target ${compat_key}; skipping version check" - append_check "opp_compatibility_matrix" "skip" "No matrix entry for OCP ${target_major}.${target_minor}" + if [[ -z "${compatSpec}" ]]; then + echo "No compatibility matrix entry for OCP ${ocpKey}; skipping version check" + AppendCheck "opp_compatibility_matrix" "skip" "No matrix entry for OCP ${ocpKey}" return 0 fi - local details="" - for entry in ${compat_spec}; do - local op_prefix="${entry%%:*}" - local min_version="${entry##*:}" - local min_major min_minor - min_major="${min_version%%.*}" - min_minor="${min_version##*.}" - - # Find installed CSV for this operator - local csv_line csv_name installed_version - csv_line="$(echo "${all_csvs}" | grep "${op_prefix}" | head -1)" || true - if [[ -z "${csv_line}" ]]; then - echo >&2 "Operator not found: ${op_prefix}" - details="${details}${op_prefix}: NOT INSTALLED; " + typeset details="" + for entry in ${compatSpec}; do + typeset opPrefix="${entry%%:*}" + typeset minVersion="${entry##*:}" + typeset minMajor minMinor + minMajor="${minVersion%%.*}" + minMinor="${minVersion##*.}" + + typeset csvLine csvName installedVersion + csvLine="$(echo "${allCsvs}" | grep "${opPrefix}" | head -1)" || true + if [[ -z "${csvLine}" ]]; then + echo >&2 "Operator not found: ${opPrefix}" + details="${details}${opPrefix}: NOT INSTALLED; " (( failed += 1 )) continue fi - csv_name="$(echo "${csv_line}" | awk '{print $2}')" - # Extract version: strip operator name prefix, keep digits - installed_version="$(echo "${csv_name}" | grep -oE '[0-9]+\.[0-9]+' | head -1)" || true - if [[ -z "${installed_version}" ]]; then - echo >&2 "Operator ${op_prefix}: could not parse version from CSV ${csv_name}" - details="${details}${op_prefix}: version unparseable from ${csv_name}; " + csvName="$(echo "${csvLine}" | awk '{print $2}')" + installedVersion="$(echo "${csvName}" | grep -oE '[0-9]+\.[0-9]+' | head -1)" || true + if [[ -z "${installedVersion}" ]]; then + echo >&2 "Operator ${opPrefix}: could not parse version from CSV ${csvName}" + details="${details}${opPrefix}: version unparseable from ${csvName}; " (( failed += 1 )) continue fi - local inst_major inst_minor - inst_major="${installed_version%%.*}" - inst_minor="${installed_version##*.}" + typeset instMajor instMinor + instMajor="${installedVersion%%.*}" + instMinor="${installedVersion##*.}" - if (( inst_major < min_major || (inst_major == min_major && inst_minor < min_minor) )); then - echo >&2 "Operator ${op_prefix} version ${installed_version} is below minimum ${min_version} for OCP ${target_major}.${target_minor}" - details="${details}${op_prefix}: ${installed_version} < ${min_version} (INCOMPATIBLE); " + if (( instMajor < minMajor || (instMajor == minMajor && instMinor < minMinor) )); then + echo >&2 "Operator ${opPrefix} version ${installedVersion} is below minimum ${minVersion} for OCP ${ocpKey}" + details="${details}${opPrefix}: ${installedVersion} < ${minVersion} (INCOMPATIBLE); " (( failed += 1 )) else - echo "Operator ${op_prefix}: version ${installed_version} >= ${min_version} (OK)" - details="${details}${op_prefix}: ${installed_version} >= ${min_version} (OK); " + echo "Operator ${opPrefix}: version ${installedVersion} >= ${minVersion} (OK)" + details="${details}${opPrefix}: ${installedVersion} >= ${minVersion} (OK); " fi done if (( failed > 0 )); then echo >&2 "${failed} operator(s) failed compatibility check" - append_check "opp_compatibility_matrix" "fail" "${details}" + AppendCheck "opp_compatibility_matrix" "fail" "${details}" (( CHECKS_FAILED += 1 )) else - echo "All OPP operators are compatible with OCP ${target_major}.${target_minor}" - append_check "opp_compatibility_matrix" "pass" "${details}" + echo "All OPP operators are compatible with OCP ${ocpKey}" + AppendCheck "opp_compatibility_matrix" "pass" "${details}" fi + true } # ────────────────────────────────────────────────────────────────────── # Check 3: Cluster health baseline # ────────────────────────────────────────────────────────────────────── -check_cluster_health() { +CheckClusterHealth() { echo -e "\n=== Check 3: Cluster health baseline ===" - local failed=0 details="" + typeset failed=0 details="" - # 3a. Node health echo "Checking node health..." - local unready_nodes - unready_nodes="$(oc get node --no-headers 2>/dev/null | awk '$2 != "Ready" {print $1}')" || true - if [[ -n "${unready_nodes}" ]]; then - echo >&2 "Not-Ready nodes: ${unready_nodes}" - details="${details}unready_nodes: ${unready_nodes}; " + typeset unreadyNodes + unreadyNodes="$(oc get node --no-headers 2>/dev/null | awk '$2 != "Ready" {print $1}')" || true + if [[ -n "${unreadyNodes}" ]]; then + echo >&2 "Not-Ready nodes: ${unreadyNodes}" + details="${details}unready_nodes: ${unreadyNodes}; " (( failed += 1 )) else - local node_count - node_count="$(oc get node --no-headers 2>/dev/null | wc -l)" - echo "All ${node_count} nodes Ready" - details="${details}nodes: all ${node_count} ready; " + typeset nodeCount + nodeCount="$(oc get node --no-headers 2>/dev/null | wc -l)" + echo "All ${nodeCount} nodes Ready" + details="${details}nodes: all ${nodeCount} ready; " fi - # 3b. ClusterOperator health echo "Checking ClusterOperator health..." - local unhealthy_co - unhealthy_co="$(oc get co --no-headers 2>/dev/null | awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}')" || true - if [[ -n "${unhealthy_co}" ]]; then - echo >&2 "Unhealthy ClusterOperators: ${unhealthy_co}" - details="${details}unhealthy_co: ${unhealthy_co}; " + typeset unhealthyCo + unhealthyCo="$(oc get co --no-headers 2>/dev/null | awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}')" || true + if [[ -n "${unhealthyCo}" ]]; then + echo >&2 "Unhealthy ClusterOperators: ${unhealthyCo}" + details="${details}unhealthy_co: ${unhealthyCo}; " (( failed += 1 )) else echo "All ClusterOperators healthy" details="${details}cluster_operators: all healthy; " fi - # 3c. CVO conditions echo "Checking ClusterVersion conditions..." - local avail progressing degraded + typeset avail progressing degraded avail="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null)" || true progressing="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Progressing")].status}' 2>/dev/null)" || true degraded="$(oc get clusterversion version -o jsonpath='{.status.conditions[?(@.type=="Degraded")].status}' 2>/dev/null)" || true @@ -273,78 +268,73 @@ check_cluster_health() { details="${details}cvo: healthy; " fi - # 3d. Firing alerts (excluding Watchdog and AlertmanagerReceiversNotConfigured) echo "Checking for firing alerts..." - local firing_alerts="" - firing_alerts="$(oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- \ + typeset firingAlerts="" + firingAlerts="$(oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- \ curl -s 'http://localhost:9090/api/v1/alerts' 2>/dev/null | \ jq -r '.data.alerts[]? | select(.state=="firing") | select(.labels.alertname != "Watchdog") | select(.labels.alertname != "AlertmanagerReceiversNotConfigured") | .labels.alertname' 2>/dev/null | \ sort -u)" || true - if [[ -n "${firing_alerts}" ]]; then - local alert_count - alert_count="$(echo "${firing_alerts}" | wc -l)" - echo "WARNING: ${alert_count} alert(s) firing: ${firing_alerts}" - details="${details}firing_alerts: ${alert_count} (${firing_alerts}); " - # Alerts are a warning, not a hard failure + if [[ -n "${firingAlerts}" ]]; then + typeset alertCount + alertCount="$(echo "${firingAlerts}" | wc -l)" + echo "WARNING: ${alertCount} alert(s) firing: ${firingAlerts}" + details="${details}firing_alerts: ${alertCount} (${firingAlerts}); " else echo "No critical alerts firing" details="${details}alerts: none firing; " fi - # Save node and CO snapshot for baseline oc get nodes -o json > "${REPORT_DIR}/nodes-baseline.json" 2>/dev/null || true oc get co -o json > "${REPORT_DIR}/co-baseline.json" 2>/dev/null || true if (( failed > 0 )); then echo >&2 "Cluster health baseline: ${failed} issue(s) found" - append_check "cluster_health_baseline" "fail" "${details}" + AppendCheck "cluster_health_baseline" "fail" "${details}" (( CHECKS_FAILED += 1 )) else echo "Cluster health baseline: all checks passed" - append_check "cluster_health_baseline" "pass" "${details}" + AppendCheck "cluster_health_baseline" "pass" "${details}" fi + true } # ────────────────────────────────────────────────────────────────────── # Check 4: MachineConfigPool readiness # ────────────────────────────────────────────────────────────────────── -check_mcp_readiness() { +CheckMcpReadiness() { echo -e "\n=== Check 4: MachineConfigPool readiness ===" - local failed=0 details="" + typeset failed=0 details="" - # Check MCP conditions: Updated=True, Updating=False, Degraded=False - local mcp_issues - mcp_issues="$(oc get machineconfigpools --no-headers 2>/dev/null | \ + typeset mcpIssues + mcpIssues="$(oc get machineconfigpools --no-headers 2>/dev/null | \ awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}')" || true - if [[ -n "${mcp_issues}" ]]; then - echo >&2 "Unhealthy MachineConfigPools: ${mcp_issues}" - details="unhealthy_mcps: ${mcp_issues}; " + if [[ -n "${mcpIssues}" ]]; then + echo >&2 "Unhealthy MachineConfigPools: ${mcpIssues}" + details="unhealthy_mcps: ${mcpIssues}; " (( failed += 1 )) - # Dump details for each unhealthy MCP - for mcp in ${mcp_issues}; do + for mcp in ${mcpIssues}; do echo -e "\n### MCP ${mcp} ###" oc describe machineconfigpool "${mcp}" 2>/dev/null || true done else - local mcp_count - mcp_count="$(oc get machineconfigpools --no-headers 2>/dev/null | wc -l)" - echo "All ${mcp_count} MachineConfigPools are updated and not degraded" - details="all ${mcp_count} MCPs healthy (Updated=True, Updating=False, Degraded=False); " + typeset mcpCount + mcpCount="$(oc get machineconfigpools --no-headers 2>/dev/null | wc -l)" + echo "All ${mcpCount} MachineConfigPools are updated and not degraded" + details="all ${mcpCount} MCPs healthy (Updated=True, Updating=False, Degraded=False); " fi - # Check that machine counts match (ready == desired) - local mismatch="" + typeset mismatch="" while IFS= read -r line; do - local mcp_name ready desired - mcp_name="$(echo "${line}" | awk '{print $1}')" - desired="$(echo "${line}" | awk '{print $6}')" + typeset mcpName ready desired + mcpName="$(echo "${line}" | awk '{print $1}')" ready="$(echo "${line}" | awk '{print $7}')" + desired="$(echo "${line}" | awk '{print $6}')" if [[ -n "${ready}" && -n "${desired}" && "${ready}" != "${desired}" ]]; then - mismatch="${mismatch}${mcp_name} (ready=${ready}, desired=${desired}); " + mismatch="${mismatch}${mcpName} (ready=${ready}, desired=${desired}); " fi done < <(oc get machineconfigpools --no-headers 2>/dev/null || true) @@ -354,28 +344,28 @@ check_mcp_readiness() { (( failed += 1 )) fi - # Save MCP snapshot oc get machineconfigpools -o json > "${REPORT_DIR}/mcp-baseline.json" 2>/dev/null || true if (( failed > 0 )); then echo >&2 "MachineConfigPool readiness: ${failed} issue(s) found" - append_check "mcp_readiness" "fail" "${details}" + AppendCheck "mcp_readiness" "fail" "${details}" (( CHECKS_FAILED += 1 )) else echo "MachineConfigPool readiness: all checks passed" - append_check "mcp_readiness" "pass" "${details}" + AppendCheck "mcp_readiness" "pass" "${details}" fi + true } # ────────────────────────────────────────────────────────────────────── # Main # ────────────────────────────────────────────────────────────────────── -main() { +Main() { if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then export KUBECONFIG="${SHARED_DIR}/kubeconfig" fi - local target="${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE:-}" + typeset target="${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE:-}" if [[ -z "${target}" ]]; then echo >&2 "OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE is not set; cannot determine upgrade target" exit 3 @@ -384,31 +374,31 @@ main() { KUBECONFIG="" oc registry login - local target_version target_major target_minor - target_version="$(oc adm release info "${target}" --output=json | jq -r '.metadata.version')" - target_major="$(echo "${target_version}" | cut -f1 -d.)" - target_minor="$(echo "${target_version}" | cut -f2 -d.)" - echo "Target OCP version: ${target_version} (major: ${target_major}, minor: ${target_minor})" + typeset targetVersion targetMajor targetMinor ocpXy + targetVersion="$(oc adm release info "${target}" --output=json | jq -r '.metadata.version')" + targetMajor="$(echo "${targetVersion}" | cut -f1 -d.)" + targetMinor="$(echo "${targetVersion}" | cut -f2 -d.)" + ocpXy="${targetMajor}.${targetMinor}" + echo "Target OCP version: ${targetVersion} (${ocpXy})" - local source_version - source_version="$(oc get clusterversion --no-headers | awk '{print $2}')" - echo "Source OCP version: ${source_version}" + typeset sourceVersion + sourceVersion="$(oc get clusterversion --no-headers | awk '{print $2}')" + echo "Source OCP version: ${sourceVersion}" echo -e "\n=== Starting OPP pre-flight validation ===\n" - init_report + InitReport - # Add metadata to report - local tmp - tmp="$(mktemp)" - jq --arg tv "${target_version}" --arg sv "${source_version}" --arg ti "${target}" \ + typeset tmpFile + tmpFile="$(mktemp)" + jq --arg tv "${targetVersion}" --arg sv "${sourceVersion}" --arg ti "${target}" \ '. + {"target_version": $tv, "source_version": $sv, "target_image": $ti, "timestamp": now | tostring}' \ - "${REPORT_FILE}" > "${tmp}" && mv "${tmp}" "${REPORT_FILE}" + "${REPORT_FILE}" > "${tmpFile}" && mv "${tmpFile}" "${REPORT_FILE}" - check_api_deprecations "${target_minor}" "${target_major}" - check_opp_compatibility "${target_minor}" "${target_major}" - check_cluster_health - check_mcp_readiness + CheckApiDeprecations "${targetMinor}" "${ocpXy}" + CheckOppCompatibility "${ocpXy}" + CheckClusterHealth + CheckMcpReadiness echo -e "\n=== Pre-flight summary ===" jq '.' "${REPORT_FILE}" @@ -420,6 +410,7 @@ main() { fi echo "Pre-flight validation PASSED: all checks succeeded" + true } -main "$@" +Main "$@"