From f20272fde593e9768344866e70a90f6a4cc1fb67 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Mon, 24 Aug 2026 13:00:28 +0000 Subject: [PATCH] Allow repeated MachineConfigNodeFailed events on single-node upgrades During a single-node (SNO) upgrade the cluster's only node reboots, taking the kube-apiserver down with it. While the API is unreachable the machine-config-operator cannot resync machineconfignodes, so library-go repeatedly surfaces an "OperatorDegraded: MachineConfigNodeFailed" event on the machine-config ClusterOperator in ns/openshift-machine-config-operator (e.g. `dial tcp 172.30.0.1:443: connect: connection refused`). The API server recovers once the reboot completes, so these repeats are expected behavior on SNO rather than a real fault. The pathological-events monitor only has the default threshold (~20) for this event and no SNO-specific override, so the repeats (observed 23-29 times per run across recent 4.21-4.23 SNO upgrade jobs) fail the "events should not repeat" test. Add a SingleReplica topology-gated matcher that raises the allowed threshold to 40 for this event, matching the approach used by the other SNO-specific overrides (ConnectionErrorDuringSingleNodeAPIServerTargetDown, KubeAPIServerProgressingDuringSingleNodeUpgrade). Multi-node topologies keep the default threshold since they retain a control-plane quorum during upgrades, and the override still fails runs where the event repeats past 40 (a sign the API never recovered). Register the matcher only in the upgrade matcher set. The node reboot that triggers these repeats only occurs during an upgrade, so outside an upgrade a burst of MachineConfigNodeFailed events is genuinely pathological and should continue to fail. Unit tests cover the 40-repeat threshold boundary (allowed) and 41 (rejected); because the event text also satisfies the SNO connection-refused matcher and the matcher registry iterates a map in non-deterministic order, the over-threshold test accepts either matcher name for the match assertion. Co-Authored-By: Claude Opus 4.8 --- .../duplicated_event_patterns.go | 35 ++++++++++ .../duplicated_events_test.go | 64 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go index c8106ccbbeca..ce5cd021928e 100644 --- a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go +++ b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go @@ -675,6 +675,13 @@ func NewUpgradePathologicalEventMatchers(kubeConfig *rest.Config, finalIntervals repeatThresholdOverride: 100, }) + // During a single-node (SNO) upgrade the cluster's only node reboots, taking the kube-apiserver + // down with it, so the machine-config operator repeatedly reports MachineConfigNodeFailed until + // the API recovers. This burst is expected only during an upgrade; outside an upgrade a flood of + // these events would be genuinely pathological, so the matcher is registered here (upgrade-only) + // rather than in the universal set. + registry.AddPathologicalEventMatcherOrDie(newSingleNodeMachineConfigNodeFailedEventMatcher()) + return registry } @@ -1218,6 +1225,34 @@ func newSingleNodeKubeAPIProgressingEventMatcher(finalIntervals monitorapi.Inter } } +// newSingleNodeMachineConfigNodeFailedEventMatcher tolerates the machine-config +// ClusterOperator going Degraded with reason MachineConfigNodeFailed on single-node (SNO) +// clusters. During an SNO upgrade the lone node reboots, taking the kube-apiserver down with it. +// While the API is unreachable the machine-config-operator cannot resync machineconfignodes, and +// library-go surfaces every failed resync as a repeated "OperatorDegraded: MachineConfigNodeFailed" +// event on the machine-config ClusterOperator in ns/openshift-machine-config-operator, e.g.: +// +// Failed to resync because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": +// dial tcp 172.30.0.1:443: connect: connection refused +// +// The API server recovers once the reboot completes, so these repeats are expected on SNO. Across +// recent 4.21-4.23 SNO upgrade jobs the event was observed 23-29 times per run over a ~30 minute +// window, so we allow up to 40 to leave headroom while still failing runs where the event repeats +// egregiously (a sign the API never came back). Multi-node topologies keep the default threshold +// because they retain a control-plane quorum during upgrades and should not see this repeat. +func newSingleNodeMachineConfigNodeFailedEventMatcher() EventMatcher { + snoTopology := v1.SingleReplicaTopologyMode + return &SimplePathologicalEventMatcher{ + name: "MachineConfigNodeFailedDuringSingleNodeUpgrade", + locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{ + monitorapi.LocatorNamespaceKey: regexp.MustCompile(`^openshift-machine-config-operator$`), + }, + messageReasonRegex: regexp.MustCompile(`^OperatorDegraded: MachineConfigNodeFailed$`), + repeatThresholdOverride: 40, + topology: &snoTopology, + } +} + // kmsEncryptionTestsDetected returns true if OCP KMS encryption tests are // present in the given intervals. It matches the [OCPFeatureGate:KMSEncryption] // tag to avoid catching upstream KMS tests that don't trigger the same diff --git a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go index 75ad4115159f..b7dc14f9cd70 100644 --- a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go +++ b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_events_test.go @@ -249,6 +249,70 @@ func TestAllowedRepeatedEvents(t *testing.T) { Reason("NetworkNotReady").Build(), expectedAllowName: "NetworkNotReady", }, + { + // On SNO the only node reboots during upgrade, so the machine-config operator cannot + // reach the API server and repeatedly reports MachineConfigNodeFailed. Expected on SNO. + name: "machine config node failed during single node upgrade", + locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator", + }, + }, + msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`). + Reason("OperatorDegraded: MachineConfigNodeFailed"). + WithAnnotation(monitorapi.AnnotationCount, "29").Build(), + topology: v1.SingleReplicaTopologyMode, + expectedAllowName: "MachineConfigNodeFailedDuringSingleNodeUpgrade", + }, + { + // The same event on a multi-node cluster is not expected (control-plane quorum is + // retained during upgrade), so the topology-gated matcher must not allow it. + name: "machine config node failed is not allowed on multi-node", + locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator", + }, + }, + msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`). + Reason("OperatorDegraded: MachineConfigNodeFailed"). + WithAnnotation(monitorapi.AnnotationCount, "29").Build(), + topology: v1.HighlyAvailableTopologyMode, + expectedAllowName: "", + }, + { + // The override threshold is 40, so exactly 40 repeats is still allowed on SNO. This + // boundary case pins the threshold: 40 is allowed, 41 (next case) is not. + name: "machine config node failed at threshold is allowed on single node", + locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator", + }, + }, + msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`). + Reason("OperatorDegraded: MachineConfigNodeFailed"). + WithAnnotation(monitorapi.AnnotationCount, "40").Build(), + topology: v1.SingleReplicaTopologyMode, + expectedAllowName: "MachineConfigNodeFailedDuringSingleNodeUpgrade", + }, + { + // Even on SNO, repeats past the override threshold indicate the API never recovered and + // must still fail: the matcher matches but does not allow the pathological repeat. This + // message also satisfies the SNO connection-refused matcher (both key off the same + // "connection refused" text), and MatchesAny iterates the matcher map in + // non-deterministic order, so either matcher name is an acceptable match here. + name: "machine config node failed over threshold is not allowed on single node", + locator: monitorapi.Locator{ + Keys: map[monitorapi.LocatorKey]string{ + monitorapi.LocatorNamespaceKey: "openshift-machine-config-operator", + }, + }, + msg: monitorapi.NewMessage().HumanMessage(`Failed to resync 4.22.0-0.nightly-2026-08-23-134143 because: Get "https://172.30.0.1:443/apis/machineconfiguration.openshift.io/v1/machineconfignodes": dial tcp 172.30.0.1:443: connect: connection refused`). + Reason("OperatorDegraded: MachineConfigNodeFailed"). + WithAnnotation(monitorapi.AnnotationCount, "41").Build(), + topology: v1.SingleReplicaTopologyMode, + expectedAllowName: "", + expectedMatchName: "MachineConfigNodeFailedDuringSingleNodeUpgrade,ConnectionErrorDuringSingleNodeAPIServerTargetDown", + }, } for _, test := range tests { registry := NewUpgradePathologicalEventMatchers(nil, nil)