diff --git a/test/extended/edge_topologies/tnf_etcd_disruption.go b/test/extended/edge_topologies/tnf_etcd_disruption.go index f720df620f8c..c728ebbf3153 100644 --- a/test/extended/edge_topologies/tnf_etcd_disruption.go +++ b/test/extended/edge_topologies/tnf_etcd_disruption.go @@ -12,6 +12,7 @@ import ( o "github.com/onsi/gomega" v1 "github.com/openshift/api/config/v1" "github.com/openshift/origin/test/extended/edge_topologies/utils" + "github.com/openshift/origin/test/extended/edge_topologies/utils/apis" "github.com/openshift/origin/test/extended/edge_topologies/utils/core" "github.com/openshift/origin/test/extended/edge_topologies/utils/services" "github.com/openshift/origin/test/extended/etcd/helpers" @@ -582,6 +583,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual g.By(fmt.Sprintf("Killing etcd container on %s via SSH", targetNode.Name)) killEtcdViaSSH(&targetNode) + g.By("Waiting for PacemakerHealthCheckDegraded=True after etcd container kill") + o.Expect(apis.WaitForPacemakerHealthCheckDegraded(oc, "", pacemakerDegradedDetectionTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should become True after etcd container kill") + // Wait for the cluster to self-heal. g.By("Waiting for etcd cluster to self-heal after container kill") o.Eventually(func() error { @@ -611,6 +616,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual o.Expect(failedSection).To(o.ContainSubstring("etcd"), "Expected Failed Resource Actions to reference etcd") framework.Logf("Failed Resource Actions section:\n%s", failedSection) + + g.By("Waiting for PacemakerHealthCheckDegraded to clear after coordinated recovery") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, longRecoveryTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after coordinated recovery") }) // This test verifies that Pacemaker detects an etcd process crash and automatically @@ -627,11 +636,19 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual g.By(fmt.Sprintf("Killing etcd process/container on %s via SSH", targetNode.Name)) killEtcdViaSSH(&targetNode) + g.By("Waiting for PacemakerHealthCheckDegraded=True after etcd process kill") + o.Expect(apis.WaitForPacemakerHealthCheckDegraded(oc, "", pacemakerDegradedDetectionTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should become True after etcd process kill") + g.By("Waiting for cluster to recover - both nodes become started voting members") validateEtcdRecoveryState(oc, etcdClientFactory, &execNode, &targetNode, true, false, 6*time.Minute, 45*time.Second) + + g.By("Waiting for PacemakerHealthCheckDegraded to clear after etcd process crash recovery") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, 6*time.Minute)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after etcd process crash recovery") }) // This test verifies that the podman-etcd resource agent retries setting @@ -662,6 +679,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual fmt.Sprintf("Expected pcs node standby to succeed, output: %s", output)) framework.Logf("PCS node standby output: %s", output) + g.By("Waiting for PacemakerHealthCheckDegraded=True after node standby") + o.Expect(apis.WaitForPacemakerHealthCheckDegraded(oc, "", pacemakerDegradedDetectionTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should become True after node standby") + // Wait for force-new-cluster recovery to complete. g.By(fmt.Sprintf("Waiting for %s to appear as learner in etcd member list", standbyNode.Name)) o.Eventually(func() error { @@ -760,5 +781,9 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual return verifyEtcdCloneStartedOnAllNodes(oc, execNode.Name, nodes) }, longRecoveryTimeout, utils.FiveSecondPollInterval).ShouldNot( o.HaveOccurred(), "etcd-clone should be Started on both nodes after recovery") + + g.By("Waiting for PacemakerHealthCheckDegraded to clear after attribute retry recovery") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, longRecoveryTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after attribute retry recovery") }) }) diff --git a/test/extended/edge_topologies/tnf_fencing_credentials.go b/test/extended/edge_topologies/tnf_fencing_credentials.go index 6f0409e5033e..37b147c4f350 100644 --- a/test/extended/edge_topologies/tnf_fencing_credentials.go +++ b/test/extended/edge_topologies/tnf_fencing_credentials.go @@ -106,7 +106,8 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual creds.Username, currentPw, newPw) } - hasPacemakerCR := apis.IsPacemakerClusterAvailable(oc) + hasPacemakerCR, availErr := apis.IsPacemakerClusterAvailable(oc) + o.Expect(availErr).ToNot(o.HaveOccurred(), "expected to check PacemakerCluster availability without error") if hasPacemakerCR { g.By("Verifying PacemakerCluster CR is healthy before credential change") pc, pcErr := apis.GetPacemakerCluster(oc) @@ -252,5 +253,201 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual }, fencingHealthTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), "expected PacemakerCluster to remain healthy after credential update") } + g.By("Verifying PacemakerHealthCheckDegraded is not set after credential update") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, fencingHealthTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should not be set after credential update") + }) + + g.It("should not degrade when fencing is at risk but still available", func() { + g.By("Finding a fencing agent to unmanage on the target node") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + pcsOutput, err := services.PcsStatusViaDebug(ctx, oc, peerNode.Name) + o.Expect(err).ToNot(o.HaveOccurred(), "expected pcs status to succeed") + + var stonithResourceName string + for _, line := range strings.Split(pcsOutput, "\n") { + trimmed := strings.TrimSpace(line) + if !strings.Contains(trimmed, "fence_") || !strings.Contains(trimmed, "Started") { + continue + } + // Resource lines are bullet-prefixed (e.g. "* master-1_redfish\t(stonith:fence_redfish):\t Started master-1"), + // so fields[0] is the "*" bullet and the resource name is fields[1]. Stonith resources are + // named "_redfish" — the node they fence, NOT the node they currently run + // on (pcs may run either node's fencing agent on either surviving node). Matching on the + // resource name itself avoids picking the wrong agent when both agents happen to be Started + // on the same node (a valid pacemaker placement, not tied to which node they fence). + fields := strings.Fields(trimmed) + if len(fields) < 2 { + continue + } + resourceName := strings.TrimSuffix(fields[1], ":") + if strings.HasPrefix(resourceName, targetNode.Name) { + stonithResourceName = resourceName + break + } + } + if stonithResourceName == "" { + g.Skip("Could not identify a started fencing agent for the target node — skipping negative test") + } + framework.Logf("Selected fencing agent to unmanage: %s", stonithResourceName) + + g.By(fmt.Sprintf("Unmanaging fencing agent %s to create FencingHealthy=False, FencingAvailable=True state", stonithResourceName)) + unmanageCmd := fmt.Sprintf("sudo pcs stonith meta %s is-managed=false", stonithResourceName) + _, err = exutil.DebugNodeRetryWithOptionsAndChroot(oc, peerNode.Name, "default", "bash", "-c", unmanageCmd) + o.Expect(err).ToNot(o.HaveOccurred(), "expected to unmanage fencing agent") + + g.DeferCleanup(func() { + framework.Logf("Restoring management of fencing agent %s", stonithResourceName) + manageCmd := fmt.Sprintf("sudo pcs stonith meta %s is-managed=true 2>/dev/null; true", stonithResourceName) + if _, restoreErr := exutil.DebugNodeRetryWithOptionsAndChroot(oc, peerNode.Name, "default", "bash", "-c", manageCmd); restoreErr != nil { + fmt.Fprintf(g.GinkgoWriter, "Warning: failed to re-manage fencing agent: %v\n", restoreErr) + } + }) + + pcAvailable, availErr := apis.IsPacemakerClusterAvailable(oc) + o.Expect(availErr).ToNot(o.HaveOccurred(), "expected to check PacemakerCluster availability without error") + if pcAvailable { + g.By("Waiting for PacemakerCluster to report FencingHealthy=False, FencingAvailable=True for target node") + o.Eventually(func() error { + pc, pcErr := apis.GetPacemakerCluster(oc) + if pcErr != nil { + return pcErr + } + if err := apis.ExpectNodeFencingUnhealthy(pc, targetNode.Name); err != nil { + return err + } + return apis.ExpectNodeFencingAvailable(pc, targetNode.Name) + }, 2*time.Minute, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), + "expected fencing to be at-risk (FencingHealthy=False) but still available (FencingAvailable=True) for target node") + } else { + framework.Logf("PacemakerCluster CRD not available, skipping CR fencing-state checks") + } + + g.By("Verifying PacemakerHealthCheckDegraded stays False during fencing warning state") + o.Consistently(func() error { + return apis.ExpectPacemakerHealthCheckNotDegraded(oc) + }, 3*time.Minute, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), + "PacemakerHealthCheckDegraded should stay False when fencing is at risk but still available") + + g.By(fmt.Sprintf("Re-managing fencing agent %s", stonithResourceName)) + manageCmd := fmt.Sprintf("sudo pcs stonith meta %s is-managed=true", stonithResourceName) + _, err = exutil.DebugNodeRetryWithOptionsAndChroot(oc, peerNode.Name, "default", "bash", "-c", manageCmd) + o.Expect(err).ToNot(o.HaveOccurred(), "expected to re-manage fencing agent") + + g.By("Verifying cluster returns to fully healthy state") + pcAvailable, availErr = apis.IsPacemakerClusterAvailable(oc) + o.Expect(availErr).ToNot(o.HaveOccurred(), "expected to check PacemakerCluster availability without error") + if pcAvailable { + o.Eventually(func() error { + pc, pcErr := apis.GetPacemakerCluster(oc) + if pcErr != nil { + return pcErr + } + return apis.ExpectClusterHealthy(pc) + }, fencingHealthTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), + "expected PacemakerCluster to be healthy after re-managing fencing agent") + } + }) + + g.It("should degrade when a node's fencing agent is completely unavailable", func() { + g.By("Finding a fencing agent to disable on the target node") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + pcsOutput, err := services.PcsStatusViaDebug(ctx, oc, peerNode.Name) + o.Expect(err).ToNot(o.HaveOccurred(), "expected pcs status to succeed") + + var stonithResourceName string + for _, line := range strings.Split(pcsOutput, "\n") { + trimmed := strings.TrimSpace(line) + if !strings.Contains(trimmed, "fence_") || !strings.Contains(trimmed, "Started") { + continue + } + // Resource lines are bullet-prefixed (e.g. "* master-1_redfish\t(stonith:fence_redfish):\t Started master-1"), + // so fields[0] is the "*" bullet and the resource name is fields[1]. Stonith resources are + // named "_redfish" — the node they fence, NOT the node they currently run + // on (pcs may run either node's fencing agent on either surviving node). Matching on the + // resource name itself avoids picking the wrong agent when both agents happen to be Started + // on the same node (a valid pacemaker placement, not tied to which node they fence). + fields := strings.Fields(trimmed) + if len(fields) < 2 { + continue + } + resourceName := strings.TrimSuffix(fields[1], ":") + if strings.HasPrefix(resourceName, targetNode.Name) { + stonithResourceName = resourceName + break + } + } + if stonithResourceName == "" { + g.Skip("Could not identify a started fencing agent for the target node — skipping fencing disable test") + } + framework.Logf("Selected fencing agent to disable: %s", stonithResourceName) + + g.By(fmt.Sprintf("Disabling fencing agent %s to make fencing completely unavailable for %s", stonithResourceName, targetNode.Name)) + // pcs rejects `pcs resource disable/enable` for stonith resources ("This command + // does not accept stonith resources") — must use `pcs stonith disable/enable` instead. + disableCmd := fmt.Sprintf("sudo pcs stonith disable %s", stonithResourceName) + _, err = exutil.DebugNodeRetryWithOptionsAndChroot(oc, peerNode.Name, "default", "bash", "-c", disableCmd) + o.Expect(err).ToNot(o.HaveOccurred(), "expected to disable fencing agent") + + g.DeferCleanup(func() { + framework.Logf("Re-enabling fencing agent %s", stonithResourceName) + enableCmd := fmt.Sprintf("sudo pcs stonith enable %s 2>/dev/null; true", stonithResourceName) + if _, enableErr := exutil.DebugNodeRetryWithOptionsAndChroot(oc, peerNode.Name, "default", "bash", "-c", enableCmd); enableErr != nil { + fmt.Fprintf(g.GinkgoWriter, "Warning: failed to re-enable fencing agent: %v\n", enableErr) + } + }) + + g.By("Waiting for PacemakerHealthCheckDegraded=True due to fencing unavailable") + o.Expect(apis.WaitForPacemakerHealthCheckDegraded(oc, "fencing unavailable", healthCheckRecoveryTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should become True when fencing is completely unavailable") + + pcAvailable, availErr := apis.IsPacemakerClusterAvailable(oc) + o.Expect(availErr).ToNot(o.HaveOccurred(), "expected to check PacemakerCluster availability without error") + if pcAvailable { + g.By("Verifying PacemakerCluster CR shows FencingAvailable=False for target node") + o.Eventually(func() error { + pc, pcErr := apis.GetPacemakerCluster(oc) + if pcErr != nil { + return pcErr + } + return apis.ExpectNodeFencingUnavailable(pc, targetNode.Name) + }, 2*time.Minute, 10*time.Second).ShouldNot(o.HaveOccurred(), + "expected FencingAvailable=False on PacemakerCluster CR for target node") + } else { + framework.Logf("PacemakerCluster CRD not available, skipping CR FencingAvailable=False check") + } + + g.By(fmt.Sprintf("Re-enabling fencing agent %s", stonithResourceName)) + enableCmd := fmt.Sprintf("sudo pcs stonith enable %s", stonithResourceName) + _, err = exutil.DebugNodeRetryWithOptionsAndChroot(oc, peerNode.Name, "default", "bash", "-c", enableCmd) + o.Expect(err).ToNot(o.HaveOccurred(), "expected to re-enable fencing agent") + + g.By("Waiting for PacemakerHealthCheckDegraded to clear after re-enabling fencing") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, healthCheckRecoveryTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after fencing is re-enabled") + + pcAvailable, availErr = apis.IsPacemakerClusterAvailable(oc) + o.Expect(availErr).ToNot(o.HaveOccurred(), "expected to check PacemakerCluster availability without error") + if pcAvailable { + g.By("Verifying cluster returns to fully healthy state") + o.Eventually(func() error { + pc, pcErr := apis.GetPacemakerCluster(oc) + if pcErr != nil { + return pcErr + } + if err := apis.ExpectClusterHealthy(pc); err != nil { + return err + } + if err := apis.ExpectNodeFencingHealthy(pc, targetNode.Name); err != nil { + return err + } + return apis.ExpectNodeFencingAvailable(pc, targetNode.Name) + }, healthCheckRecoveryTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), + "expected PacemakerCluster to be healthy with FencingHealthy=True and FencingAvailable=True after re-enabling agent") + } else { + framework.Logf("PacemakerCluster CRD not available, skipping final CR health check") + } }) }) diff --git a/test/extended/edge_topologies/tnf_kubelet_disruption.go b/test/extended/edge_topologies/tnf_kubelet_disruption.go index 779fcafcf9c0..e98d1e1a7a9f 100644 --- a/test/extended/edge_topologies/tnf_kubelet_disruption.go +++ b/test/extended/edge_topologies/tnf_kubelet_disruption.go @@ -9,6 +9,7 @@ import ( o "github.com/onsi/gomega" v1 "github.com/openshift/api/config/v1" "github.com/openshift/origin/test/extended/edge_topologies/utils" + "github.com/openshift/origin/test/extended/edge_topologies/utils/apis" "github.com/openshift/origin/test/extended/etcd/helpers" exutil "github.com/openshift/origin/test/extended/util" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -136,6 +137,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual return !nodeutil.IsNodeReady(nodeObj) }, kubeletDisruptionTimeout, utils.FiveSecondPollInterval).Should(o.BeTrue(), fmt.Sprintf("Node %s is not in state Ready after kubelet resource ban is applied", targetNode.Name)) + g.By("Waiting for PacemakerHealthCheckDegraded to become True") + o.Expect(apis.WaitForPacemakerHealthCheckDegraded(oc, "node is unhealthy", kubeletDisruptionTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should become True after kubelet disruption") + g.By("Validating etcd cluster remains healthy with surviving node") o.Consistently(func() error { return helpers.EnsureHealthyMember(g.GinkgoT(), etcdClientFactory, survivingNode.Name) @@ -165,6 +170,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual o.Eventually(func() error { return utils.ValidateEssentialOperatorsAvailable(oc) }, kubeletRestoreTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), "Essential operators should be available") + + g.By("Waiting for PacemakerHealthCheckDegraded to clear") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, kubeletRestoreTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after kubelet recovery") }) g.It("should properly stop kubelet service and verify automatic restart on target node", func() { @@ -202,6 +211,41 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual err = utils.StopKubeletService(oc, targetNode.Name) o.Expect(err).To(o.BeNil(), fmt.Sprintf("Expected to stop kubelet service on node %s without errors", targetNode.Name)) + // Assert degradation before waiting on recovery. This is racy by nature: + // Pacemaker auto-restarts kubelet, which clears the degraded condition once + // healthy again. The PacemakerCluster CR backing this condition is only + // refreshed by a once-per-minute CronJob, and Pacemaker's own out-of-band + // failure detection plus recovery for the kubelet resource can complete in + // less than one CronJob interval — so a single stop can be fully recovered + // within a gap between snapshots and never be observed as degraded. + // + // To observe the condition reliably, poll it directly and, whenever we find + // that Pacemaker has already restarted kubelet before degradation was seen, + // stop kubelet again. This keeps the resource failing across at least one + // status snapshot without suppressing Pacemaker's real recovery behavior. + g.By("Waiting for PacemakerHealthCheckDegraded=True after kubelet stop (re-stopping kubelet as needed)") + o.Eventually(func() (bool, error) { + degraded, msg, err := apis.IsPacemakerHealthCheckDegraded(oc) + if err != nil { + framework.Logf("Error checking PacemakerHealthCheckDegraded: %v", err) + return false, nil + } + if degraded { + framework.Logf("PacemakerHealthCheckDegraded=True observed (message: %q)", msg) + return true, nil + } + + // Not degraded yet. If Pacemaker has already restarted kubelet, re-stop + // it so the failure persists until the next status snapshot catches it. + if utils.IsServiceRunning(oc, survivingNode.Name, targetNode.Name, "kubelet") { + framework.Logf("Kubelet was restarted by Pacemaker before degradation was observed; stopping it again on %s", targetNode.Name) + if stopErr := utils.StopKubeletService(oc, targetNode.Name); stopErr != nil { + framework.Logf("Warning: failed to re-stop kubelet on %s: %v", targetNode.Name, stopErr) + } + } + return false, nil + }, kubeletDisruptionTimeout, utils.FiveSecondPollInterval).Should(o.BeTrue(), "PacemakerHealthCheckDegraded should become True after kubelet stop") + g.By("Waiting for Pacemaker to auto-recover and restart kubelet-clone service") o.Eventually(func() bool { isRunning := utils.IsServiceRunning(oc, survivingNode.Name, targetNode.Name, "kubelet") @@ -237,6 +281,9 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual o.Eventually(func() error { return utils.ValidateEssentialOperatorsAvailable(oc) }, kubeletRestoreTimeout, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), "Essential operators should be available") - }) + g.By("Waiting for PacemakerHealthCheckDegraded to clear") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, kubeletRestoreTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after kubelet recovery") + }) }) diff --git a/test/extended/edge_topologies/tnf_node_replacement.go b/test/extended/edge_topologies/tnf_node_replacement.go index d5d08de2b377..2c3024d76af3 100644 --- a/test/extended/edge_topologies/tnf_node_replacement.go +++ b/test/extended/edge_topologies/tnf_node_replacement.go @@ -141,6 +141,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][Suite:openshift/two e2e.Logf("[stage timing] Restoring etcd quorum: %v (phase1 etcd start cap: %v, phase2 %d×%v)", time.Since(stageStart), etcdPhase1StartAfterStonithTimeout, stonithCleanupMaxAttempts, stonithCleanupRoundTimeout) stageStart = time.Now() + g.By("Verifying PacemakerHealthCheckDegraded=True after node destruction") + o.Expect(apis.WaitForPacemakerHealthCheckDegraded(oc, "", pacemakerDegradedDetectionTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should be True after node destruction and quorum restore") + g.By("Deleting OpenShift node references") deleteNodeReferences(&testConfig, oc) e2e.Logf("[stage timing] Deleting node references (BMH/Machine/Node + OVN SB chassis-del + etcd/KAS nodeStatus + installer pods): %v (BMH/Machine delete wait: %v, poll: %v)", time.Since(stageStart), bmhMachineDeleteWaitTimeout, bmhMachineDeletePollInterval) @@ -215,6 +219,23 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][Suite:openshift/two e2e.Logf("[stage timing] Restoring pacemaker cluster (total): %v (see sub-lines from restorePacemakerCluster for CEO job vs pcs online caps)", time.Since(stageStart)) stageStart = time.Now() + g.By("Waiting for PacemakerHealthCheckDegraded to clear after full node replacement") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, 5*time.Minute)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after pacemaker cluster is restored") + + g.By("Verifying PacemakerCluster CR shows replacement node as member with fencing available") + o.Eventually(func() error { + pc, pcErr := apis.GetPacemakerCluster(oc) + if pcErr != nil { + return pcErr + } + if err := apis.ExpectNodeMember(pc, testConfig.TargetNode.Name); err != nil { + return err + } + return apis.ExpectNodeFencingAvailable(pc, testConfig.TargetNode.Name) + }, 5*time.Minute, 10*time.Second).ShouldNot(o.HaveOccurred(), + "Replacement node should be a member with fencing available in PacemakerCluster CR") + g.By("Verifying the cluster is fully restored") verifyRestoredCluster(&testConfig, oc) e2e.Logf("[stage timing] Verify cluster restored (total): %v (see sub-lines for CO monitor cap)", time.Since(stageStart)) diff --git a/test/extended/edge_topologies/tnf_pacemaker_healthcheck.go b/test/extended/edge_topologies/tnf_pacemaker_healthcheck.go new file mode 100644 index 000000000000..6f3d2e86292f --- /dev/null +++ b/test/extended/edge_topologies/tnf_pacemaker_healthcheck.go @@ -0,0 +1,181 @@ +package edge_topologies + +import ( + "fmt" + "math/rand" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + v1 "github.com/openshift/api/config/v1" + "github.com/openshift/origin/test/extended/edge_topologies/utils" + "github.com/openshift/origin/test/extended/edge_topologies/utils/apis" + "github.com/openshift/origin/test/extended/etcd/helpers" + exutil "github.com/openshift/origin/test/extended/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + healthCheckRecoveryTimeout = 10 * time.Minute + + // pacemakerDegradedDetectionTimeout must exceed the operator's worst-case + // detection latency. When a node drops, the etcd/API/CronJob pipeline is + // disrupted, so degraded is often reached via the staleness path: + // StatusStalenessThreshold (5m) -> status Unknown, then + // StatusUnknownDegradedThreshold (5m) -> PacemakerHealthCheckDegraded=True + // (see cluster-etcd-operator pkg/tnf/pkg/pacemaker/constants.go). That is a + // 10m minimum even with a healthy controller; the extra margin covers + // status-collector CronJob scheduling jitter. + pacemakerDegradedDetectionTimeout = 15 * time.Minute +) + +var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:DualReplica][Suite:openshift/two-node][Serial][Disruptive] PacemakerHealthCheck degraded condition", func() { + defer g.GinkgoRecover() + + var ( + oc = exutil.NewCLIWithoutNamespace("").AsAdmin() + etcdClientFactory *helpers.EtcdClientFactoryImpl + execNode corev1.Node + targetNode corev1.Node + nodes []corev1.Node + ) + + g.BeforeEach(func() { + utils.SkipIfNotTopology(oc, v1.DualReplicaTopologyMode) + + etcdClientFactory = helpers.NewEtcdClientFactory(oc.KubeClient()) + + utils.SkipIfClusterIsNotHealthy(oc, etcdClientFactory) + + nodeList, err := utils.GetNodes(oc, utils.AllNodes) + o.Expect(err).To(o.BeNil(), "Expected to retrieve nodes without error") + o.Expect(len(nodeList.Items)).To(o.Equal(2), "Expected exactly 2 nodes for two-node cluster") + + randomIndex := rand.Intn(len(nodeList.Items)) + execNode = nodeList.Items[randomIndex] + targetNode = nodeList.Items[(randomIndex+1)%len(nodeList.Items)] + nodes = nodeList.Items + + g.DeferCleanup(func() { + logFinalClusterStatus(nodes) + }) + }) + + g.It("should detect and recover from cluster maintenance mode", func() { + // Capture an event baseline before the disruptive action so the event + // assertions below only accept freshly-emitted events, not stale ones + // left over from a prior reconcile or test run. + maintenanceBaseline := time.Now() + + g.By("Enabling cluster maintenance mode") + _, err := exutil.DebugNodeRetryWithOptionsAndChroot( + oc, execNode.Name, "default", "bash", "-c", + "sudo pcs property set maintenance-mode=true") + o.Expect(err).To(o.BeNil(), "Expected to enable maintenance mode") + + g.DeferCleanup(func() { + framework.Logf("DeferCleanup: Ensuring maintenance mode is disabled") + if _, cleanupErr := exutil.DebugNodeRetryWithOptionsAndChroot( + oc, execNode.Name, "default", "bash", "-c", + "sudo pcs property set maintenance-mode=false 2>/dev/null; true"); cleanupErr != nil { + framework.Logf("Warning: Failed to disable maintenance mode: %v", cleanupErr) + } + }) + + g.By("Waiting for PacemakerHealthCheckDegraded=True due to maintenance mode") + o.Expect(apis.WaitForPacemakerHealthCheckDegraded(oc, "maintenance mode", healthCheckRecoveryTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should become True when cluster is in maintenance mode") + + g.By("Verifying PacemakerClusterInMaintenance event was emitted") + o.Expect(apis.WaitForPacemakerEvent(oc, "PacemakerClusterInMaintenance", maintenanceBaseline, 2*time.Minute)). + ShouldNot(o.HaveOccurred(), "Expected PacemakerClusterInMaintenance event in openshift-etcd namespace") + + // Baseline for the post-recovery PacemakerHealthy event: the cluster was + // healthy before this test, so an earlier PacemakerHealthy event may + // exist. Require one emitted after recovery begins. + recoveryBaseline := time.Now() + + g.By("Disabling cluster maintenance mode") + _, err = exutil.DebugNodeRetryWithOptionsAndChroot( + oc, execNode.Name, "default", "bash", "-c", + "sudo pcs property set maintenance-mode=false") + o.Expect(err).To(o.BeNil(), "Expected to disable maintenance mode") + + g.By("Waiting for PacemakerHealthCheckDegraded to clear") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, healthCheckRecoveryTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after maintenance mode is disabled") + + g.By("Verifying PacemakerHealthy event was emitted after recovery") + o.Expect(apis.WaitForPacemakerEvent(oc, "PacemakerHealthy", recoveryBaseline, 2*time.Minute)). + ShouldNot(o.HaveOccurred(), "Expected PacemakerHealthy event in openshift-etcd namespace after recovery") + + g.By("Validating cluster health after maintenance mode recovery") + o.Eventually(func() error { + return utils.ValidateEssentialOperatorsAvailable(oc) + }, healthCheckRecoveryTimeout, utils.FiveSecondPollInterval).ShouldNot( + o.HaveOccurred(), "Essential operators should be available after maintenance mode recovery") + }) + + g.It("should detect and recover from a node going offline via pcs cluster stop", func() { + g.By(fmt.Sprintf("Stopping pacemaker cluster on %s from peer node %s", targetNode.Name, execNode.Name)) + _, err := exutil.DebugNodeRetryWithOptionsAndChroot( + oc, execNode.Name, "default", "bash", "-c", + fmt.Sprintf("sudo pcs cluster stop %s", targetNode.Name)) + o.Expect(err).To(o.BeNil(), "Expected pcs cluster stop to succeed") + + g.DeferCleanup(func() { + framework.Logf("DeferCleanup: Ensuring pacemaker cluster is started on %s", targetNode.Name) + if _, cleanupErr := exutil.DebugNodeRetryWithOptionsAndChroot( + oc, execNode.Name, "default", "bash", "-c", + fmt.Sprintf("sudo pcs cluster start %s 2>/dev/null; true", targetNode.Name)); cleanupErr != nil { + framework.Logf("Warning: Failed to restart pacemaker on %s: %v", targetNode.Name, cleanupErr) + } + }) + + // Accept any degraded message rather than requiring "is offline". Even in + // this single-node-down scenario the controller can reach degraded via the + // staleness path (message "is stale") if the status-collector CronJob pod is + // scheduled onto the corosync-stopped node before the surviving node's + // collector writes NodeOnline=False. Both paths correctly signal degradation. + g.By("Waiting for PacemakerHealthCheckDegraded=True due to node offline") + o.Expect(apis.WaitForPacemakerHealthCheckDegraded(oc, "", pacemakerDegradedDetectionTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should become True when a node is offline") + + // NodeCountAsExpected is derived from the CIB (`pcs cluster config`), which + // still lists both nodes after `pcs cluster stop` — stopping corosync on a + // node does not remove it from the configured node count. The condition must + // therefore remain True while the node is offline. + g.By("Verifying PacemakerCluster CR keeps NodeCountAsExpected=True while node is offline") + o.Eventually(func() error { + pc, pcErr := apis.GetPacemakerCluster(oc) + if pcErr != nil { + return pcErr + } + return apis.ExpectClusterNodeCountAsExpected(pc) + }, 2*time.Minute, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), + "NodeCountAsExpected should remain True while node is offline (pcs cluster stop does not change the CIB node count)") + + g.By(fmt.Sprintf("Starting pacemaker cluster on %s", targetNode.Name)) + _, err = exutil.DebugNodeRetryWithOptionsAndChroot( + oc, execNode.Name, "default", "bash", "-c", + fmt.Sprintf("sudo pcs cluster start %s", targetNode.Name)) + o.Expect(err).To(o.BeNil(), "Expected pcs cluster start to succeed") + + g.By("Waiting for PacemakerHealthCheckDegraded to clear") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, healthCheckRecoveryTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after node comes back online") + + g.By("Validating cluster health after node restart") + o.Eventually(func() error { + return utils.ValidateEssentialOperatorsAvailable(oc) + }, healthCheckRecoveryTimeout, utils.FiveSecondPollInterval).ShouldNot( + o.HaveOccurred(), "Essential operators should be available after node restart") + + g.By("Validating etcd cluster recovered") + o.Eventually(func() error { + return utils.LogEtcdClusterStatus(oc, "after pcs cluster start", etcdClientFactory) + }, healthCheckRecoveryTimeout, utils.FiveSecondPollInterval).ShouldNot( + o.HaveOccurred(), "Etcd cluster should be healthy after node restart") + }) +}) diff --git a/test/extended/edge_topologies/tnf_recovery.go b/test/extended/edge_topologies/tnf_recovery.go index af226d0502c4..39f8e3bf7ab3 100644 --- a/test/extended/edge_topologies/tnf_recovery.go +++ b/test/extended/edge_topologies/tnf_recovery.go @@ -118,6 +118,16 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual o.Expect(err).To(o.BeNil(), "Expected to gracefully shutdown the node without errors") time.Sleep(time.Minute) + // Informational: these recovery tests validate etcd recovery, not the health + // monitoring pipeline. A missed degraded transition signals a monitoring gap + // (directly covered by the dedicated tnf_pacemaker_healthcheck.go tests), so + // log and continue rather than block the recovery validation that follows. + g.By("Waiting for PacemakerHealthCheckDegraded=True while target node is down") + if err := apis.WaitForPacemakerHealthCheckDegraded(oc, "", pacemakerDegradedDetectionTimeout); err != nil { + framework.Logf("WARNING: PacemakerHealthCheckDegraded did not fire while target node was down "+ + "(health monitoring gap — recovery validation continues): %v", err) + } + g.By(fmt.Sprintf("Ensuring that %s is a healthy voting member and adds %s back as learner (timeout: %v)", peerNode.Name, targetNode.Name, memberIsLeaderTimeout)) validateEtcdRecoveryState(oc, etcdClientFactory, &survivedNode, @@ -138,6 +148,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual g.By(fmt.Sprintf("Checking recovery path for %s from %s journal", targetNode.Name, survivedNode.Name)) logRecoveryPath(oc, &survivedNode, &targetNode) + + g.By("Waiting for PacemakerHealthCheckDegraded to clear after graceful recovery") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, memberPromotedVotingTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after etcd recovery") }) g.It("should recover from ungraceful node shutdown with etcd member re-addition", func() { @@ -151,6 +165,16 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual o.Expect(err).To(o.BeNil(), "Expected to ungracefully shutdown the node without errors", targetNode.Name, err) time.Sleep(1 * time.Minute) + // Informational: these recovery tests validate etcd recovery, not the health + // monitoring pipeline. A missed degraded transition signals a monitoring gap + // (directly covered by the dedicated tnf_pacemaker_healthcheck.go tests), so + // log and continue rather than block the recovery validation that follows. + g.By("Waiting for PacemakerHealthCheckDegraded=True while target node is down") + if err := apis.WaitForPacemakerHealthCheckDegraded(oc, "", pacemakerDegradedDetectionTimeout); err != nil { + framework.Logf("WARNING: PacemakerHealthCheckDegraded did not fire while target node was down "+ + "(health monitoring gap — recovery validation continues): %v", err) + } + g.By(fmt.Sprintf("Ensuring that %s added %s back as learner (timeout: %v)", peerNode.Name, targetNode.Name, memberIsLeaderTimeout)) validateEtcdRecoveryState(oc, etcdClientFactory, &survivedNode, @@ -168,6 +192,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual &survivedNode, &targetNode, true, false, // targetNode expected started == true, learner == false memberPromotedVotingTimeout, utils.FiveSecondPollInterval) + + g.By("Waiting for PacemakerHealthCheckDegraded to clear after ungraceful recovery") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, memberPromotedVotingTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after etcd recovery") }) g.It("should recover from network disruption with etcd member re-addition", func() { @@ -202,6 +230,13 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual leaderNode, learnerNode, true, false, // targetNode expected started == true, learner == false memberPromotedVotingTimeout, utils.FiveSecondPollInterval) + + g.By("Checking PacemakerHealthCheckDegraded after short network disruption (informational)") + if checkErr := apis.ExpectPacemakerHealthCheckNotDegraded(oc); checkErr != nil { + framework.Logf("PacemakerHealthCheckDegraded was True after network disruption (may be expected): %v", checkErr) + } else { + framework.Logf("PacemakerHealthCheckDegraded remained False after short network disruption (fault window shorter than healthcheck resync)") + } }) g.It("should recover from a double node failure (cold-boot) [Requires:HypervisorSSHConfig]", func() { @@ -241,6 +276,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual &nodeA, &nodeB, true, false, membersHealthyAfterDoubleReboot, utils.FiveSecondPollInterval) + + g.By("Waiting for PacemakerHealthCheckDegraded to clear after double node failure recovery") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, membersHealthyAfterDoubleReboot)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after double node failure recovery") }) g.It("should recover from double graceful node shutdown (cold-boot) [Requires:HypervisorSSHConfig]", func() { @@ -280,6 +319,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual &nodeA, &nodeB, true, false, membersHealthyAfterDoubleReboot, utils.FiveSecondPollInterval) + + g.By("Waiting for PacemakerHealthCheckDegraded to clear after double graceful shutdown recovery") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, membersHealthyAfterDoubleReboot)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after double graceful shutdown recovery") }) g.It("should recover from sequential graceful node shutdowns (cold-boot) [Requires:HypervisorSSHConfig]", func() { @@ -305,6 +348,16 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual err = vmShutdownAndWait(VMShutdownModeGraceful, vmFirstToShutdown, c) o.Expect(err).To(o.BeNil(), fmt.Sprintf("Expected VM %s to reach shut off state", vmFirstToShutdown)) + // Informational: these recovery tests validate etcd recovery, not the health + // monitoring pipeline. A missed degraded transition signals a monitoring gap + // (directly covered by the dedicated tnf_pacemaker_healthcheck.go tests), so + // log and continue rather than block the recovery validation that follows. + g.By("Waiting for PacemakerHealthCheckDegraded=True after first node shutdown") + if err := apis.WaitForPacemakerHealthCheckDegraded(oc, "", pacemakerDegradedDetectionTimeout); err != nil { + framework.Logf("WARNING: PacemakerHealthCheckDegraded did not fire while first node was down "+ + "(health monitoring gap — recovery validation continues): %v", err) + } + g.By(fmt.Sprintf("Gracefully shutting down second node: %s", secondToShutdown.Name)) err = vmShutdownAndWait(VMShutdownModeGraceful, vmSecondToShutdown, c) o.Expect(err).To(o.BeNil(), fmt.Sprintf("Expected VM %s to reach shut off state", vmSecondToShutdown)) @@ -318,6 +371,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual &firstToShutdown, &secondToShutdown, true, false, membersHealthyAfterDoubleReboot, utils.FiveSecondPollInterval) + + g.By("Waiting for PacemakerHealthCheckDegraded to clear after sequential shutdown recovery") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, membersHealthyAfterDoubleReboot)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after sequential shutdown recovery") }) g.It("should recover from graceful shutdown followed by ungraceful node failure (cold-boot) [Requires:HypervisorSSHConfig]", func() { @@ -342,6 +399,16 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual err = vmShutdownAndWait(VMShutdownModeGraceful, vmFirstToShutdown, c) o.Expect(err).To(o.BeNil(), fmt.Sprintf("Expected VM %s to reach shut off state", vmFirstToShutdown)) + // Informational: these recovery tests validate etcd recovery, not the health + // monitoring pipeline. A missed degraded transition signals a monitoring gap + // (directly covered by the dedicated tnf_pacemaker_healthcheck.go tests), so + // log and continue rather than block the recovery validation that follows. + g.By("Waiting for PacemakerHealthCheckDegraded=True after first node shutdown") + if err := apis.WaitForPacemakerHealthCheckDegraded(oc, "", pacemakerDegradedDetectionTimeout); err != nil { + framework.Logf("WARNING: PacemakerHealthCheckDegraded did not fire while first node was down "+ + "(health monitoring gap — recovery validation continues): %v", err) + } + g.By(fmt.Sprintf("Waiting for %s to recover the etcd cluster standalone (timeout: %v)", secondToShutdown.Name, memberIsLeaderTimeout)) validateEtcdRecoveryState(oc, etcdClientFactory, &secondToShutdown, @@ -361,6 +428,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual &firstToShutdown, &secondToShutdown, true, false, membersHealthyAfterDoubleReboot, utils.FiveSecondPollInterval) + + g.By("Waiting for PacemakerHealthCheckDegraded to clear after mixed cold-boot recovery") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, membersHealthyAfterDoubleReboot)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after mixed cold-boot recovery") }) g.It("should recover from BMC credential rotation with fencing", func() { @@ -448,6 +519,16 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual err = exutil.TriggerKernelPanic(oc.KubeClient(), targetNode.Name) o.Expect(err).To(o.BeNil(), "Expected to trigger kernel panic without error") + // Informational: this test validates etcd recovery and the revision bump, not + // the health monitoring pipeline. A missed degraded transition signals a + // monitoring gap (directly covered by the dedicated + // tnf_pacemaker_healthcheck.go tests), so log and continue. + g.By("Waiting for PacemakerHealthCheckDegraded=True while target node is down after kernel panic") + if err := apis.WaitForPacemakerHealthCheckDegraded(oc, "", pacemakerDegradedDetectionTimeout); err != nil { + framework.Logf("WARNING: PacemakerHealthCheckDegraded did not fire after kernel panic "+ + "(health monitoring gap — recovery validation continues): %v", err) + } + g.By(fmt.Sprintf("Ensuring that %s added %s back as learner (timeout: %v)", survivedNode.Name, targetNode.Name, memberIsLeaderTimeout)) validateEtcdRecoveryState(oc, etcdClientFactory, &survivedNode, @@ -548,6 +629,9 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual }, 5*time.Minute, utils.FiveSecondPollInterval).ShouldNot(o.HaveOccurred(), fmt.Sprintf("expected etcd-previous container to exist on %s", targetNode.Name)) + g.By("Waiting for PacemakerHealthCheckDegraded to clear after kernel panic recovery") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, memberPromotedVotingTimeout)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after kernel panic recovery") }) g.It("should recover after simultaneous graceful shutdown of both nodes", func() { @@ -578,6 +662,10 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual o.Expect(err).To(o.BeNil(), fmt.Sprintf("Expected no error checking etcd on %s", node.Name)) o.Expect(got).To(o.Equal("'true'"), fmt.Sprintf("Expected etcd container running on %s", node.Name)) } + + g.By("Waiting for PacemakerHealthCheckDegraded to clear after simultaneous graceful shutdown recovery") + o.Expect(apis.WaitForPacemakerHealthCheckCleared(oc, membersHealthyAfterDoubleReboot)). + ShouldNot(o.HaveOccurred(), "PacemakerHealthCheckDegraded should clear after simultaneous graceful shutdown recovery") }) }) diff --git a/test/extended/edge_topologies/tnf_taint.go b/test/extended/edge_topologies/tnf_taint.go index 2ca15c4808bf..a12a7c1b8126 100644 --- a/test/extended/edge_topologies/tnf_taint.go +++ b/test/extended/edge_topologies/tnf_taint.go @@ -126,8 +126,8 @@ var _ = g.Describe("[sig-node][apigroup:config.openshift.io][OCPFeatureGate:Dual g.BeforeEach(func() { utils.SkipIfNotTopology(oc, v1.DualReplicaTopologyMode) - // Skip the test if cluster version is below 5.0, as fencing taint was introduced in that version - utils.SkipIfVersionBelow(oc, 5, 0) + // Skip the test if cluster version is below 4.22, as fencing taint was introduced in that version + utils.SkipIfVersionBelow(oc, 4, 22) }) g.It("should have pacemaker taint and untaint alerts registered", func() { @@ -171,8 +171,8 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual g.BeforeEach(func() { utils.SkipIfNotTopology(oc, v1.DualReplicaTopologyMode) - // Skip the test if cluster version is below 5.0, as fencing taint was introduced in that version - utils.SkipIfVersionBelow(oc, 5, 0) + // Skip the test if cluster version is below 4.22, as fencing taint was introduced in that version + utils.SkipIfVersionBelow(oc, 4, 22) etcdClientFactory = helpers.NewEtcdClientFactory(oc.KubeClient()) @@ -330,8 +330,8 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual services.TaintScriptLogTag, services.TaintSuccessLog, baseTimestamp) }, journalCheckTimeout, utils.FiveSecondPollInterval).Should(o.BeTrue(), "taint-fenced-node should log successful taint and annotation application") - // --- Recovery Wait --- + // --- Recovery Wait --- if !learnerStarted { g.By(fmt.Sprintf("Ensuring %s rejoins as learner (timeout: %v)", learnerNode.Name, memberRejoinedLearnerTimeout)) diff --git a/test/extended/edge_topologies/utils/apis/pacemakercluster.go b/test/extended/edge_topologies/utils/apis/pacemakercluster.go index 1d59a04c6170..6b1718e71336 100644 --- a/test/extended/edge_topologies/utils/apis/pacemakercluster.go +++ b/test/extended/edge_topologies/utils/apis/pacemakercluster.go @@ -5,31 +5,42 @@ import ( "fmt" "time" - etcdv1alpha1 "github.com/openshift/api/etcd/v1alpha1" + etcdv1 "github.com/openshift/api/etcd/v1" exutil "github.com/openshift/origin/test/extended/util" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ) var PacemakerClusterGVR = schema.GroupVersionResource{ - Group: etcdv1alpha1.GroupName, Version: "v1alpha1", Resource: "pacemakerclusters", + Group: etcdv1.GroupName, Version: "v1", Resource: "pacemakerclusters", } -func IsPacemakerClusterAvailable(oc *exutil.CLI) bool { +// IsPacemakerClusterAvailable reports whether the PacemakerCluster CRD is served +// by the API. Only a NotFound error means the CRD is genuinely absent; any other +// error (authorization, transient API failures) is returned so callers fail +// rather than silently skip checks by mistaking a real error for CRD absence. +func IsPacemakerClusterAvailable(oc *exutil.CLI) (bool, error) { _, err := oc.AdminDynamicClient().Resource(PacemakerClusterGVR).List( context.Background(), metav1.ListOptions{Limit: 1}) - return err == nil + if err == nil { + return true, nil + } + if apierrors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("check PacemakerCluster availability: %w", err) } -func GetPacemakerCluster(oc *exutil.CLI) (*etcdv1alpha1.PacemakerCluster, error) { +func GetPacemakerCluster(oc *exutil.CLI) (*etcdv1.PacemakerCluster, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() u, err := oc.AdminDynamicClient().Resource(PacemakerClusterGVR).Get(ctx, "cluster", metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("get PacemakerCluster: %w", err) } - var pc etcdv1alpha1.PacemakerCluster + var pc etcdv1.PacemakerCluster if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.UnstructuredContent(), &pc); err != nil { return nil, fmt.Errorf("convert PacemakerCluster: %w", err) } @@ -45,19 +56,124 @@ func findCondition(conditions []metav1.Condition, condType string) *metav1.Condi return nil } -func ExpectClusterHealthy(pc *etcdv1alpha1.PacemakerCluster) error { - c := findCondition(pc.Status.Conditions, etcdv1alpha1.ClusterHealthyConditionType) +func ExpectClusterHealthy(pc *etcdv1.PacemakerCluster) error { + c := findCondition(pc.Status.Conditions, etcdv1.ClusterHealthyConditionType) + if c == nil { + return fmt.Errorf("PacemakerCluster missing %s condition", etcdv1.ClusterHealthyConditionType) + } + if c.Status != metav1.ConditionTrue { + return fmt.Errorf("PacemakerCluster %s=%s (reason: %s, message: %s)", + etcdv1.ClusterHealthyConditionType, c.Status, c.Reason, c.Message) + } + return nil +} + +func ExpectNodeFencingAvailable(pc *etcdv1.PacemakerCluster, nodeName string) error { + if pc.Status.Nodes == nil { + return fmt.Errorf("PacemakerCluster has no nodes in status") + } + for _, node := range *pc.Status.Nodes { + if node.NodeName != nodeName { + continue + } + c := findCondition(node.Conditions, etcdv1.NodeFencingAvailableConditionType) + if c == nil { + return fmt.Errorf("node %s missing %s condition", nodeName, etcdv1.NodeFencingAvailableConditionType) + } + if c.Status != metav1.ConditionTrue { + return fmt.Errorf("node %s %s=%s (reason: %s, message: %s)", + nodeName, etcdv1.NodeFencingAvailableConditionType, c.Status, c.Reason, c.Message) + } + return nil + } + return fmt.Errorf("node %s not found in PacemakerCluster status", nodeName) +} + +// ExpectNodeFencingUnavailable returns nil only when the node's +// NodeFencingAvailable condition is explicitly False. Missing nodes, missing +// conditions, and other schema errors are propagated so a definitively-False +// state is the only success — a lookup failure must never be mistaken for +// "fencing is unavailable". +func ExpectNodeFencingUnavailable(pc *etcdv1.PacemakerCluster, nodeName string) error { + if pc.Status.Nodes == nil { + return fmt.Errorf("PacemakerCluster has no nodes in status") + } + for _, node := range *pc.Status.Nodes { + if node.NodeName != nodeName { + continue + } + c := findCondition(node.Conditions, etcdv1.NodeFencingAvailableConditionType) + if c == nil { + return fmt.Errorf("node %s missing %s condition", nodeName, etcdv1.NodeFencingAvailableConditionType) + } + if c.Status != metav1.ConditionFalse { + return fmt.Errorf("node %s %s=%s, expected False (reason: %s, message: %s)", + nodeName, etcdv1.NodeFencingAvailableConditionType, c.Status, c.Reason, c.Message) + } + return nil + } + return fmt.Errorf("node %s not found in PacemakerCluster status", nodeName) +} + +// ExpectNodeFencingUnhealthy returns nil only when the node's +// NodeFencingHealthy condition is explicitly False (e.g. a fencing agent is +// unmanaged but still running). Missing nodes, missing conditions, and other +// schema errors are propagated so they cannot be mistaken for the desired state. +func ExpectNodeFencingUnhealthy(pc *etcdv1.PacemakerCluster, nodeName string) error { + if pc.Status.Nodes == nil { + return fmt.Errorf("PacemakerCluster has no nodes in status") + } + for _, node := range *pc.Status.Nodes { + if node.NodeName != nodeName { + continue + } + c := findCondition(node.Conditions, etcdv1.NodeFencingHealthyConditionType) + if c == nil { + return fmt.Errorf("node %s missing %s condition", nodeName, etcdv1.NodeFencingHealthyConditionType) + } + if c.Status != metav1.ConditionFalse { + return fmt.Errorf("node %s %s=%s, expected False (reason: %s, message: %s)", + nodeName, etcdv1.NodeFencingHealthyConditionType, c.Status, c.Reason, c.Message) + } + return nil + } + return fmt.Errorf("node %s not found in PacemakerCluster status", nodeName) +} + +func ExpectNodeMember(pc *etcdv1.PacemakerCluster, nodeName string) error { + if pc.Status.Nodes == nil { + return fmt.Errorf("PacemakerCluster has no nodes in status") + } + for _, node := range *pc.Status.Nodes { + if node.NodeName != nodeName { + continue + } + c := findCondition(node.Conditions, etcdv1.NodeMemberConditionType) + if c == nil { + return fmt.Errorf("node %s missing %s condition", nodeName, etcdv1.NodeMemberConditionType) + } + if c.Status != metav1.ConditionTrue { + return fmt.Errorf("node %s %s=%s (reason: %s, message: %s)", + nodeName, etcdv1.NodeMemberConditionType, c.Status, c.Reason, c.Message) + } + return nil + } + return fmt.Errorf("node %s not found in PacemakerCluster status", nodeName) +} + +func ExpectClusterNodeCountAsExpected(pc *etcdv1.PacemakerCluster) error { + c := findCondition(pc.Status.Conditions, etcdv1.ClusterNodeCountAsExpectedConditionType) if c == nil { - return fmt.Errorf("PacemakerCluster missing %s condition", etcdv1alpha1.ClusterHealthyConditionType) + return fmt.Errorf("PacemakerCluster missing %s condition", etcdv1.ClusterNodeCountAsExpectedConditionType) } if c.Status != metav1.ConditionTrue { return fmt.Errorf("PacemakerCluster %s=%s (reason: %s, message: %s)", - etcdv1alpha1.ClusterHealthyConditionType, c.Status, c.Reason, c.Message) + etcdv1.ClusterNodeCountAsExpectedConditionType, c.Status, c.Reason, c.Message) } return nil } -func ExpectNodeFencingHealthy(pc *etcdv1alpha1.PacemakerCluster, nodeName string) error { +func ExpectNodeFencingHealthy(pc *etcdv1.PacemakerCluster, nodeName string) error { if pc.Status.Nodes == nil { return fmt.Errorf("PacemakerCluster has no nodes in status") } @@ -65,13 +181,13 @@ func ExpectNodeFencingHealthy(pc *etcdv1alpha1.PacemakerCluster, nodeName string if node.NodeName != nodeName { continue } - c := findCondition(node.Conditions, etcdv1alpha1.NodeFencingHealthyConditionType) + c := findCondition(node.Conditions, etcdv1.NodeFencingHealthyConditionType) if c == nil { - return fmt.Errorf("node %s missing %s condition", nodeName, etcdv1alpha1.NodeFencingHealthyConditionType) + return fmt.Errorf("node %s missing %s condition", nodeName, etcdv1.NodeFencingHealthyConditionType) } if c.Status != metav1.ConditionTrue { return fmt.Errorf("node %s %s=%s (reason: %s, message: %s)", - nodeName, etcdv1alpha1.NodeFencingHealthyConditionType, c.Status, c.Reason, c.Message) + nodeName, etcdv1.NodeFencingHealthyConditionType, c.Status, c.Reason, c.Message) } return nil } diff --git a/test/extended/edge_topologies/utils/apis/pacemakerhealthcheck.go b/test/extended/edge_topologies/utils/apis/pacemakerhealthcheck.go new file mode 100644 index 000000000000..25c9cbcb1906 --- /dev/null +++ b/test/extended/edge_topologies/utils/apis/pacemakerhealthcheck.go @@ -0,0 +1,319 @@ +package apis + +import ( + "context" + "fmt" + "strings" + "time" + + operatorv1 "github.com/openshift/api/operator/v1" + exutil "github.com/openshift/origin/test/extended/util" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + PacemakerHealthCheckDegradedCondition = "PacemakerHealthCheckDegraded" + + healthCheckPollInterval = 10 * time.Second + + // pacemakerTargetNamespace is where the status-collector CronJob and the + // PacemakerCluster data pipeline run. + pacemakerTargetNamespace = "openshift-etcd" + + // statusCollectorCronJobName is the CronJob that snapshots pacemaker status + // into the PacemakerCluster CR (see cluster-etcd-operator). + statusCollectorCronJobName = "pacemaker-status-collector" +) + +func getEtcdOperator(oc *exutil.CLI) (*operatorv1.Etcd, error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + return oc.AdminOperatorClient().OperatorV1().Etcds().Get(ctx, "cluster", metav1.GetOptions{}) +} + +func findOperatorCondition(etcd *operatorv1.Etcd, condType string) *operatorv1.OperatorCondition { + for i := range etcd.Status.Conditions { + if etcd.Status.Conditions[i].Type == condType { + return &etcd.Status.Conditions[i] + } + } + return nil +} + +// dumpHealthCheckDiagnostics logs the state most useful for triaging why a +// PacemakerHealthCheckDegraded transition did not happen within the timeout: the +// etcd operator condition, the PacemakerCluster CR staleness and conditions, the +// status-collector CronJob's last run, and node readiness. Together these show +// whether the data pipeline was flowing (CR fresh, CronJob running, nodes Ready) +// or broken. Every step is best-effort — this runs on an already-failing path and +// must never itself fail or panic. +func dumpHealthCheckDiagnostics(oc *exutil.CLI, reason string) { + framework.Logf("========== PACEMAKER HEALTHCHECK DIAGNOSTICS (%s) ==========", reason) + + // 1. etcd operator PacemakerHealthCheckDegraded condition + if etcd, err := getEtcdOperator(oc); err != nil { + framework.Logf("diagnostics: get etcd operator: %v", err) + } else if cond := findOperatorCondition(etcd, PacemakerHealthCheckDegradedCondition); cond == nil { + framework.Logf("diagnostics: etcd operator %s condition absent", PacemakerHealthCheckDegradedCondition) + } else { + framework.Logf("diagnostics: etcd operator %s: Status=%s reason=%s message=%q lastTransition=%s", + PacemakerHealthCheckDegradedCondition, cond.Status, cond.Reason, cond.Message, + cond.LastTransitionTime.Format(time.RFC3339)) + } + + // 2. PacemakerCluster CR staleness and conditions + if pc, err := GetPacemakerCluster(oc); err != nil { + framework.Logf("diagnostics: get PacemakerCluster CR: %v", err) + } else { + lastUpdated := pc.Status.LastUpdated.Time + if lastUpdated.IsZero() { + framework.Logf("diagnostics: PacemakerCluster CR lastUpdated is zero (status never populated)") + } else { + framework.Logf("diagnostics: PacemakerCluster CR lastUpdated=%s (age %v)", + lastUpdated.Format(time.RFC3339), time.Since(lastUpdated).Round(time.Second)) + } + for i := range pc.Status.Conditions { + c := &pc.Status.Conditions[i] + framework.Logf("diagnostics: PacemakerCluster condition %s=%s reason=%s message=%q", + c.Type, c.Status, c.Reason, c.Message) + } + } + + // 3. status-collector CronJob last run + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + cronJob, err := oc.AdminKubeClient().BatchV1().CronJobs(pacemakerTargetNamespace).Get(ctx, statusCollectorCronJobName, metav1.GetOptions{}) + cancel() + if err != nil { + framework.Logf("diagnostics: get CronJob %s/%s: %v", pacemakerTargetNamespace, statusCollectorCronJobName, err) + } else { + lastSchedule := "never" + if cronJob.Status.LastScheduleTime != nil { + lastSchedule = cronJob.Status.LastScheduleTime.Format(time.RFC3339) + } + lastSuccess := "never" + if cronJob.Status.LastSuccessfulTime != nil { + lastSuccess = cronJob.Status.LastSuccessfulTime.Format(time.RFC3339) + } + framework.Logf("diagnostics: CronJob %s lastSchedule=%s lastSuccessful=%s activeJobs=%d", + statusCollectorCronJobName, lastSchedule, lastSuccess, len(cronJob.Status.Active)) + } + + // 4. Node readiness + ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second) + nodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(ctx2, metav1.ListOptions{}) + cancel2() + if err != nil { + framework.Logf("diagnostics: list nodes: %v", err) + } else { + for i := range nodes.Items { + n := &nodes.Items[i] + ready := "Unknown" + for _, c := range n.Status.Conditions { + if c.Type == corev1.NodeReady { + ready = string(c.Status) + break + } + } + framework.Logf("diagnostics: node %s Ready=%s", n.Name, ready) + } + } + + framework.Logf("========== END PACEMAKER HEALTHCHECK DIAGNOSTICS ==========") +} + +// describeNotDegradedCondition formats a not-yet-True PacemakerHealthCheckDegraded +// condition for logging. The operator's clearPacemakerDegradedCondition intentionally +// leaves Reason/Message empty when setting Status=False (see healthcheck.go), so an +// empty reason/message here reflects the healthy baseline, not a missing field bug. +func describeNotDegradedCondition(cond *operatorv1.OperatorCondition) string { + if cond.Reason == "" && cond.Message == "" { + return fmt.Sprintf("Status=%s (healthy)", cond.Status) + } + return fmt.Sprintf("Status=%s reason=%s message=%q", cond.Status, cond.Reason, cond.Message) +} + +// WaitForPacemakerHealthCheckDegraded polls the etcd operator resource until +// PacemakerHealthCheckDegraded=True with a message containing expectedSubstring. +// Pass an empty expectedSubstring to accept any message. +func WaitForPacemakerHealthCheckDegraded(oc *exutil.CLI, expectedSubstring string, timeout time.Duration) error { + deadline := time.After(timeout) + ticker := time.NewTicker(healthCheckPollInterval) + defer ticker.Stop() + + var lastErr string + for { + select { + case <-deadline: + dumpHealthCheckDiagnostics(oc, "WaitForPacemakerHealthCheckDegraded timeout") + return fmt.Errorf("timed out after %v waiting for PacemakerHealthCheckDegraded=True (last: %s)", timeout, lastErr) + case <-ticker.C: + etcd, err := getEtcdOperator(oc) + if err != nil { + lastErr = fmt.Sprintf("get etcd operator: %v", err) + framework.Logf("WaitForPacemakerHealthCheckDegraded: %s", lastErr) + continue + } + + cond := findOperatorCondition(etcd, PacemakerHealthCheckDegradedCondition) + if cond == nil { + lastErr = "condition not found" + framework.Logf("WaitForPacemakerHealthCheckDegraded: condition not yet present on etcd operator") + continue + } + + if cond.Status != operatorv1.ConditionTrue { + lastErr = describeNotDegradedCondition(cond) + framework.Logf("WaitForPacemakerHealthCheckDegraded: %s", lastErr) + continue + } + + if expectedSubstring != "" && !strings.Contains(cond.Message, expectedSubstring) { + lastErr = fmt.Sprintf("True but message %q does not contain %q", cond.Message, expectedSubstring) + framework.Logf("WaitForPacemakerHealthCheckDegraded: %s", lastErr) + continue + } + + framework.Logf("PacemakerHealthCheckDegraded=True confirmed (reason=%s, message=%q)", cond.Reason, cond.Message) + return nil + } + } +} + +// WaitForPacemakerHealthCheckCleared polls the etcd operator resource until +// PacemakerHealthCheckDegraded=False or the condition is absent. +func WaitForPacemakerHealthCheckCleared(oc *exutil.CLI, timeout time.Duration) error { + deadline := time.After(timeout) + ticker := time.NewTicker(healthCheckPollInterval) + defer ticker.Stop() + + var lastErr string + for { + select { + case <-deadline: + dumpHealthCheckDiagnostics(oc, "WaitForPacemakerHealthCheckCleared timeout") + return fmt.Errorf("timed out after %v waiting for PacemakerHealthCheckDegraded to clear (last: %s)", timeout, lastErr) + case <-ticker.C: + etcd, err := getEtcdOperator(oc) + if err != nil { + lastErr = fmt.Sprintf("get etcd operator: %v", err) + framework.Logf("WaitForPacemakerHealthCheckCleared: %s", lastErr) + continue + } + + cond := findOperatorCondition(etcd, PacemakerHealthCheckDegradedCondition) + if cond == nil { + framework.Logf("PacemakerHealthCheckDegraded condition absent — treating as cleared") + return nil + } + + if cond.Status == operatorv1.ConditionFalse { + framework.Logf("PacemakerHealthCheckDegraded=False confirmed") + return nil + } + + lastErr = fmt.Sprintf("Status=%s reason=%s message=%q", cond.Status, cond.Reason, cond.Message) + framework.Logf("WaitForPacemakerHealthCheckCleared: still degraded — %s", lastErr) + } + } +} + +// pacemakerHealthCheckEventNamespace is where the healthcheck controller's +// library-go event recorder writes events — the operator's own pod namespace +// (openshift-etcd-operator), not the target namespace (openshift-etcd) that +// the status collector uses for its own PacemakerFailedResourceAction / +// PacemakerStatusCollectionError events. +const pacemakerHealthCheckEventNamespace = "openshift-etcd-operator" + +// eventTime returns the most recent activity timestamp for an event, preferring +// EventTime, then LastTimestamp, then the object's creation timestamp. This is +// used to distinguish freshly-emitted events from stale ones left over from an +// earlier reconcile or a previous test run. +func eventTime(ev *corev1.Event) time.Time { + if !ev.EventTime.IsZero() { + return ev.EventTime.Time + } + if !ev.LastTimestamp.IsZero() { + return ev.LastTimestamp.Time + } + return ev.CreationTimestamp.Time +} + +// WaitForPacemakerEvent polls events in the openshift-etcd-operator namespace +// until one with the given Reason emitted at or after the provided lower bound +// (since) appears. This applies to healthcheck-controller reasons (e.g. +// PacemakerHealthy, PacemakerClusterInMaintenance, PacemakerNodeOffline). The +// since bound prevents a stale event from a prior reconcile or test from +// satisfying the wait. +func WaitForPacemakerEvent(oc *exutil.CLI, reason string, since time.Time, timeout time.Duration) error { + deadline := time.After(timeout) + ticker := time.NewTicker(healthCheckPollInterval) + defer ticker.Stop() + + for { + select { + case <-deadline: + return fmt.Errorf("timed out after %v waiting for event with reason %q emitted at or after %s in %s", + timeout, reason, since.Format(time.RFC3339), pacemakerHealthCheckEventNamespace) + case <-ticker.C: + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + events, err := oc.AdminKubeClient().CoreV1().Events(pacemakerHealthCheckEventNamespace).List(ctx, metav1.ListOptions{ + FieldSelector: fmt.Sprintf("reason=%s", reason), + }) + cancel() + if err != nil { + framework.Logf("WaitForPacemakerEvent: list events: %v", err) + continue + } + for i := range events.Items { + ev := &events.Items[i] + if eventTime(ev).Before(since) { + continue + } + framework.Logf("Found event reason=%s message=%q at %s (baseline %s)", + ev.Reason, ev.Message, eventTime(ev).Format(time.RFC3339), since.Format(time.RFC3339)) + return nil + } + } + } +} + +// IsPacemakerHealthCheckDegraded performs a single check of the etcd operator +// resource and reports whether PacemakerHealthCheckDegraded is currently True, +// along with the condition message. A missing condition is reported as not +// degraded. Unlike WaitForPacemakerHealthCheckDegraded, this does not poll, so +// callers can interleave it with other actions (e.g. re-inducing a failure that +// Pacemaker would otherwise auto-recover before the next status snapshot). +func IsPacemakerHealthCheckDegraded(oc *exutil.CLI) (bool, string, error) { + etcd, err := getEtcdOperator(oc) + if err != nil { + return false, "", fmt.Errorf("get etcd operator: %w", err) + } + + cond := findOperatorCondition(etcd, PacemakerHealthCheckDegradedCondition) + if cond == nil { + return false, "", nil + } + + return cond.Status == operatorv1.ConditionTrue, cond.Message, nil +} + +// ExpectPacemakerHealthCheckNotDegraded checks the etcd operator resource +// and returns an error if PacemakerHealthCheckDegraded is True. +func ExpectPacemakerHealthCheckNotDegraded(oc *exutil.CLI) error { + etcd, err := getEtcdOperator(oc) + if err != nil { + return fmt.Errorf("get etcd operator: %w", err) + } + + cond := findOperatorCondition(etcd, PacemakerHealthCheckDegradedCondition) + if cond == nil { + return nil + } + + if cond.Status == operatorv1.ConditionTrue { + return fmt.Errorf("PacemakerHealthCheckDegraded=True (reason=%s, message=%q)", cond.Reason, cond.Message) + } + return nil +}