From 40a51528c2287604661645580401c20781fcf801 Mon Sep 17 00:00:00 2001 From: Derek Frank Date: Wed, 12 Aug 2026 22:17:05 +0000 Subject: [PATCH] feat(status): label transition_seconds with the destination status The transition_seconds histogram only labeled the status a condition was LEFT (`status`), never the status it moved TO. A dwell in e.g. Ready=Unknown or Ready=False was therefore indistinguishable between "recovered to True" and any other outcome, so recovery-duration for a condition could not be isolated. Add a `to_status` label to `operator[_]_status_condition_transition_seconds`. `status` still means the state being left (backward compatible); `to_status` is the state entered. Recovery duration is then transition_seconds filtered to status=False/Unknown, to_status=True. The recovery *rate* can be derived by comparing that against transitions_total entries into the unhealthy state. Adds a test asserting the destination label is recorded on a transition. --- status/controller.go | 5 +++-- status/controller_test.go | 32 ++++++++++++++++++++++++++++++++ status/metrics.go | 9 ++++++++- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/status/controller.go b/status/controller.go index 853a0e6..bf5c61d 100644 --- a/status/controller.go +++ b/status/controller.go @@ -344,8 +344,9 @@ func (c *Controller[T]) reconcile(ctx context.Context, req reconcile.Request, o } duration := condition.LastTransitionTime.Time.Sub(observedCondition.LastTransitionTime.Time).Seconds() c.observeHistogram(c.ConditionDuration, ConditionDuration, duration, map[string]string{ - pmetrics.LabelType: observedCondition.Type, - MetricLabelConditionStatus: string(observedCondition.Status), + pmetrics.LabelType: observedCondition.Type, + MetricLabelConditionStatus: string(observedCondition.Status), + MetricLabelToConditionStatus: string(condition.Status), }, c.toAdditionalMetricLabels(o)) c.eventRecorder.Event(o, v1.EventTypeNormal, condition.Type, fmt.Sprintf("Status condition transitioned, Type: %s, Status: %s -> %s, Reason: %s%s", condition.Type, diff --git a/status/controller_test.go b/status/controller_test.go index b7063dd..ad8ec68 100644 --- a/status/controller_test.go +++ b/status/controller_test.go @@ -95,6 +95,38 @@ var _ = Describe("Controller", func() { Expect(metric).ToNot(BeNil()) Expect(metric.GetHistogram().GetSampleCount()).To(BeNumerically(">", 0)) }) + It("should label transition_seconds with the destination status a condition transitioned to", func() { + testObject := test.Object(&test.CustomObject{}) + gvk := object.GVK(testObject) + ExpectApplied(ctx, kubeClient, testObject) + ExpectReconciled(ctx, controller, testObject) // observe Foo=Unknown + + // Recover Foo: Unknown -> True + time.Sleep(time.Second * 1) + testObject.StatusConditions().SetTrue(ConditionTypeFoo) + ExpectApplied(ctx, kubeClient, testObject) + ExpectReconciled(ctx, controller, testObject) + + // The dwell spent in Unknown is recorded labeled with the state left + // (status=Unknown) AND the destination (to_status=True). + recovered := map[string]string{ + pmetrics.LabelType: string(ConditionTypeFoo), + status.MetricLabelConditionStatus: string(metav1.ConditionUnknown), + status.MetricLabelToConditionStatus: string(metav1.ConditionTrue), + } + Expect(GetMetric("operator_customobject_status_condition_transition_seconds", recovered).GetHistogram().GetSampleCount()).To(BeNumerically(">", 0)) + // No sample is recorded for the same dwell under a different destination. + Expect(GetMetric("operator_customobject_status_condition_transition_seconds", map[string]string{ + pmetrics.LabelType: string(ConditionTypeFoo), + status.MetricLabelConditionStatus: string(metav1.ConditionUnknown), + status.MetricLabelToConditionStatus: string(metav1.ConditionFalse), + })).To(BeNil()) + // The deprecated (group/kind-labeled) variant carries to_status as well. + Expect(GetMetric("operator_status_condition_transition_seconds", lo.Assign(recovered, map[string]string{ + pmetrics.LabelGroup: gvk.Group, + pmetrics.LabelKind: gvk.Kind, + })).GetHistogram().GetSampleCount()).To(BeNumerically(">", 0)) + }) It("should emit metrics and events on a transition", func() { testObject := test.Object(&test.CustomObject{}) gvk := object.GVK(testObject) diff --git a/status/metrics.go b/status/metrics.go index a0d5881..96991e6 100644 --- a/status/metrics.go +++ b/status/metrics.go @@ -13,6 +13,12 @@ const ( MetricLabelNamespace = "namespace" MetricLabelName = "name" MetricLabelConditionStatus = "status" + // MetricLabelToConditionStatus labels the transition_seconds histogram with + // the status the condition transitioned TO. Combined with the existing + // "status" label (the status being left), this lets consumers distinguish + // how a dwell ended -- e.g. how long a condition was unhealthy + // (status="False") before it recovered (to_status="True"). + MetricLabelToConditionStatus = "to_status" ) const ( @@ -33,12 +39,13 @@ func conditionDurationMetric(objectName string, buckets []float64, additionalLab Namespace: pmetrics.Namespace, Subsystem: subsystem, Name: "transition_seconds", - Help: "The amount of time a condition was in a given state before transitioning. e.g. Alarm := P99(Updated=False) > 5 minutes", + Help: "The amount of time a condition was in a given state (status) before transitioning to another state (to_status). e.g. Alarm := P99(Updated=False) > 5 minutes", Buckets: buckets, }, append([]string{ pmetrics.LabelType, MetricLabelConditionStatus, + MetricLabelToConditionStatus, }, additionalLabels...), ) }