Allow repeated MachineConfigNodeFailed events on single-node upgrades

pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go

@@ -675,6 +675,13 @@ func NewUpgradePathologicalEventMatchers(kubeConfig *rest.Config, finalIntervals
 		repeatThresholdOverride: 100,
 	})
 
+	// During a single-node (SNO) upgrade the cluster's only node reboots, taking the kube-apiserver
+	// down with it, so the machine-config operator repeatedly reports MachineConfigNodeFailed until
+	// the API recovers. This burst is expected only during an upgrade; outside an upgrade a flood of
+	// these events would be genuinely pathological, so the matcher is registered here (upgrade-only)
+	// rather than in the universal set.
+	registry.AddPathologicalEventMatcherOrDie(newSingleNodeMachineConfigNodeFailedEventMatcher())
+
 	return registry
 }
 
@@ -1218,6 +1225,34 @@ func newSingleNodeKubeAPIProgressingEventMatcher(finalIntervals monitorapi.Inter
 	}
 }
 
+// newSingleNodeMachineConfigNodeFailedEventMatcher tolerates the machine-config
+// ClusterOperator going Degraded with reason MachineConfigNodeFailed on single-node (SNO)
+// clusters. During an SNO upgrade the lone node reboots, taking the kube-apiserver down with it.
+// While the API is unreachable the machine-config-operator cannot resync machineconfignodes, and
+// library-go surfaces every failed resync as a repeated "OperatorDegraded: MachineConfigNodeFailed"
+// event on the machine-config ClusterOperator in ns/openshift-machine-config-operator, e.g.:
+//
+//	Failed to resync <payload> because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes":
+//	dial tcp 172.30.0.1:443: connect: connection refused
+//
+// The API server recovers once the reboot completes, so these repeats are expected on SNO. Across
+// recent 4.21-4.23 SNO upgrade jobs the event was observed 23-29 times per run over a ~30 minute
+// window, so we allow up to 40 to leave headroom while still failing runs where the event repeats
+// egregiously (a sign the API never came back). Multi-node topologies keep the default threshold
+// because they retain a control-plane quorum during upgrades and should not see this repeat.
+func newSingleNodeMachineConfigNodeFailedEventMatcher() EventMatcher {
+	snoTopology := v1.SingleReplicaTopologyMode
+	return &SimplePathologicalEventMatcher{
+		name: "MachineConfigNodeFailedDuringSingleNodeUpgrade",
+		locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{
+			monitorapi.LocatorNamespaceKey: regexp.MustCompile(`^openshift-machine-config-operator$`),
+		},
+		messageReasonRegex:      regexp.MustCompile(`^OperatorDegraded: MachineConfigNodeFailed$`),
+		repeatThresholdOverride: 40,
+		topology:                &snoTopology,
+	}
+}
+
 // kmsEncryptionTestsDetected returns true if OCP KMS encryption tests are
 // present in the given intervals. It matches the [OCPFeatureGate:KMSEncryption]
 // tag to avoid catching upstream KMS tests that don't trigger the same

pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go

@@ -249,6 +249,70 @@ func TestAllowedRepeatedEvents(t *testing.T) {
 				Reason("NetworkNotReady").Build(),
 			expectedAllowName: "NetworkNotReady",
 		},
+		{
+			// On SNO the only node reboots during upgrade, so the machine-config operator cannot
+			// reach the API server and repeatedly reports MachineConfigNodeFailed. Expected on SNO.
+			name: "machine config node failed during single node upgrade",
+			locator: monitorapi.Locator{
+				Keys: map[monitorapi.LocatorKey]string{
+					monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator",
+				},
+			},
+			msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`).
+				Reason("OperatorDegraded: MachineConfigNodeFailed").
+				WithAnnotation(monitorapi.AnnotationCount, "29").Build(),
+			topology:          v1.SingleReplicaTopologyMode,
+			expectedAllowName: "MachineConfigNodeFailedDuringSingleNodeUpgrade",
+		},
+		{
+			// The same event on a multi-node cluster is not expected (control-plane quorum is
+			// retained during upgrade), so the topology-gated matcher must not allow it.
+			name: "machine config node failed is not allowed on multi-node",
+			locator: monitorapi.Locator{
+				Keys: map[monitorapi.LocatorKey]string{
+					monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator",
+				},
+			},
+			msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`).
+				Reason("OperatorDegraded: MachineConfigNodeFailed").
+				WithAnnotation(monitorapi.AnnotationCount, "29").Build(),
+			topology:          v1.HighlyAvailableTopologyMode,
+			expectedAllowName: "",
+		},
+		{
+			// The override threshold is 40, so exactly 40 repeats is still allowed on SNO. This
+			// boundary case pins the threshold: 40 is allowed, 41 (next case) is not.
+			name: "machine config node failed at threshold is allowed on single node",
+			locator: monitorapi.Locator{
+				Keys: map[monitorapi.LocatorKey]string{
+					monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator",
+				},
+			},
+			msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`).
+				Reason("OperatorDegraded: MachineConfigNodeFailed").
+				WithAnnotation(monitorapi.AnnotationCount, "40").Build(),
+			topology:          v1.SingleReplicaTopologyMode,
+			expectedAllowName: "MachineConfigNodeFailedDuringSingleNodeUpgrade",
+		},
+		{
+			// Even on SNO, repeats past the override threshold indicate the API never recovered and
+			// must still fail: the matcher matches but does not allow the pathological repeat. This
+			// message also satisfies the SNO connection-refused matcher (both key off the same
+			// "connection refused" text), and MatchesAny iterates the matcher map in
+			// non-deterministic order, so either matcher name is an acceptable match here.
+			name: "machine config node failed over threshold is not allowed on single node",
+			locator: monitorapi.Locator{
+				Keys: map[monitorapi.LocatorKey]string{
+					monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator",
+				},
+			},
+			msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`).
+				Reason("OperatorDegraded: MachineConfigNodeFailed").
+				WithAnnotation(monitorapi.AnnotationCount, "41").Build(),
+			topology:          v1.SingleReplicaTopologyMode,
+			expectedAllowName: "",
+			expectedMatchName: "MachineConfigNodeFailedDuringSingleNodeUpgrade,ConnectionErrorDuringSingleNodeAPIServerTargetDown",
+		},
 	}
 	for _, test := range tests {
 		registry := NewUpgradePathologicalEventMatchers(nil, nil)