diff --git a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go index c8106ccbbeca..ce5cd021928e 100644 --- a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go +++ b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go @@ -675,6 +675,13 @@ func NewUpgradePathologicalEventMatchers(kubeConfig *rest.Config, finalIntervals repeatThresholdOverride: 100, }) + // During a single-node (SNO) upgrade the cluster's only node reboots, taking the kube-apiserver + // down with it, so the machine-config operator repeatedly reports MachineConfigNodeFailed until + // the API recovers. This burst is expected only during an upgrade; outside an upgrade a flood of + // these events would be genuinely pathological, so the matcher is registered here (upgrade-only) + // rather than in the universal set. + registry.AddPathologicalEventMatcherOrDie(newSingleNodeMachineConfigNodeFailedEventMatcher()) + return registry } @@ -1218,6 +1225,34 @@ func newSingleNodeKubeAPIProgressingEventMatcher(finalIntervals monitorapi.Inter } } +// newSingleNodeMachineConfigNodeFailedEventMatcher tolerates the machine-config +// ClusterOperator going Degraded with reason MachineConfigNodeFailed on single-node (SNO) +// clusters. During an SNO upgrade the lone node reboots, taking the kube-apiserver down with it. +// While the API is unreachable the machine-config-operator cannot resync machineconfignodes, and +// library-go surfaces every failed resync as a repeated "OperatorDegraded: MachineConfigNodeFailed" +// event on the machine-config ClusterOperator in ns/openshift-machine-config-operator, e.g.: +// +// Failed to resync because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": +// dial tcp 172.30.0.1:443: connect: connection refused +// +// The API server recovers once the reboot completes, so these repeats are expected on SNO. Across +// recent 4.21-4.23 SNO upgrade jobs the event was observed 23-29 times per run over a ~30 minute +// window, so we allow up to 40 to leave headroom while still failing runs where the event repeats +// egregiously (a sign the API never came back). Multi-node topologies keep the default threshold +// because they retain a control-plane quorum during upgrades and should not see this repeat. +func newSingleNodeMachineConfigNodeFailedEventMatcher() EventMatcher { + snoTopology := v1.SingleReplicaTopologyMode + return &SimplePathologicalEventMatcher{ + name: "MachineConfigNodeFailedDuringSingleNodeUpgrade", + locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{ + monitorapi.LocatorNamespaceKey: regexp.MustCompile(`^openshift-machine-config-operator$`), + }, + messageReasonRegex: regexp.MustCompile(`^OperatorDegraded: MachineConfigNodeFailed$`), + repeatThresholdOverride: 40, + topology: &snoTopology, + } +} + // kmsEncryptionTestsDetected returns true if OCP KMS encryption tests are // present in the given intervals. It matches the [OCPFeatureGate:KMSEncryption] // tag to avoid catching upstream KMS tests that don't trigger the same diff --git a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go index 75ad4115159f..b7dc14f9cd70 100644 --- a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go +++ b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go @@ -249,6 +249,70 @@ func TestAllowedRepeatedEvents(t *testing.T) { Reason("NetworkNotReady").Build(), expectedAllowName: "NetworkNotReady", }, + { + // On SNO the only node reboots during upgrade, so the machine-config operator cannot + // reach the API server and repeatedly reports MachineConfigNodeFailed. Expected on SNO. + name: "machine config node failed during single node upgrade", + locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator", + }, + }, + msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`). + Reason("OperatorDegraded: MachineConfigNodeFailed"). + WithAnnotation(monitorapi.AnnotationCount, "29").Build(), + topology: v1.SingleReplicaTopologyMode, + expectedAllowName: "MachineConfigNodeFailedDuringSingleNodeUpgrade", + }, + { + // The same event on a multi-node cluster is not expected (control-plane quorum is + // retained during upgrade), so the topology-gated matcher must not allow it. + name: "machine config node failed is not allowed on multi-node", + locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator", + }, + }, + msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`). + Reason("OperatorDegraded: MachineConfigNodeFailed"). + WithAnnotation(monitorapi.AnnotationCount, "29").Build(), + topology: v1.HighlyAvailableTopologyMode, + expectedAllowName: "", + }, + { + // The override threshold is 40, so exactly 40 repeats is still allowed on SNO. This + // boundary case pins the threshold: 40 is allowed, 41 (next case) is not. + name: "machine config node failed at threshold is allowed on single node", + locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator", + }, + }, + msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`). + Reason("OperatorDegraded: MachineConfigNodeFailed"). + WithAnnotation(monitorapi.AnnotationCount, "40").Build(), + topology: v1.SingleReplicaTopologyMode, + expectedAllowName: "MachineConfigNodeFailedDuringSingleNodeUpgrade", + }, + { + // Even on SNO, repeats past the override threshold indicate the API never recovered and + // must still fail: the matcher matches but does not allow the pathological repeat. This + // message also satisfies the SNO connection-refused matcher (both key off the same + // "connection refused" text), and MatchesAny iterates the matcher map in + // non-deterministic order, so either matcher name is an acceptable match here. + name: "machine config node failed over threshold is not allowed on single node", + locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator", + }, + }, + msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`). + Reason("OperatorDegraded: MachineConfigNodeFailed"). + WithAnnotation(monitorapi.AnnotationCount, "41").Build(), + topology: v1.SingleReplicaTopologyMode, + expectedAllowName: "", + expectedMatchName: "MachineConfigNodeFailedDuringSingleNodeUpgrade,ConnectionErrorDuringSingleNodeAPIServerTargetDown", + }, } for _, test := range tests { registry := NewUpgradePathologicalEventMatchers(nil, nil)