diff --git a/status/controller.go b/status/controller.go index c30198e..113dfdc 100644 --- a/status/controller.go +++ b/status/controller.go @@ -372,8 +372,9 @@ func (c *Controller[T]) reconcileFound(ctx context.Context, req reconcile.Reques } duration := condition.LastTransitionTime.Time.Sub(observedCondition.LastTransitionTime.Time).Seconds() c.observeHistogram(c.ConditionDuration, ConditionDuration, duration, map[string]string{ - pmetrics.LabelType: observedCondition.Type, - MetricLabelConditionStatus: string(observedCondition.Status), + pmetrics.LabelType: observedCondition.Type, + MetricLabelConditionStatus: string(observedCondition.Status), + MetricLabelToConditionStatus: string(condition.Status), }, c.toAdditionalMetricLabels(o)) c.eventRecorder.Event(o, v1.EventTypeNormal, condition.Type, fmt.Sprintf("Status condition transitioned, Type: %s, Status: %s -> %s, Reason: %s%s", condition.Type, diff --git a/status/controller_test.go b/status/controller_test.go index 1a15d87..c01c58b 100644 --- a/status/controller_test.go +++ b/status/controller_test.go @@ -119,6 +119,38 @@ var _ = Describe("Controller", func() { Expect(metric).ToNot(BeNil()) Expect(metric.GetHistogram().GetSampleCount()).To(BeNumerically(">", 0)) }) + It("should label transition_seconds with the destination status a condition transitioned to", func() { + testObject := test.Object(&test.CustomObject{}) + gvk := object.GVK(testObject) + ExpectApplied(ctx, kubeClient, testObject) + ExpectReconciled(ctx, controller, testObject) // observe Foo=Unknown + + // Recover Foo: Unknown -> True + time.Sleep(time.Second * 1) + testObject.StatusConditions().SetTrue(ConditionTypeFoo) + ExpectApplied(ctx, kubeClient, testObject) + ExpectReconciled(ctx, controller, testObject) + + // The dwell spent in Unknown is recorded labeled with the state left + // (status=Unknown) AND the destination (to_status=True). + recovered := map[string]string{ + pmetrics.LabelType: string(ConditionTypeFoo), + status.MetricLabelConditionStatus: string(metav1.ConditionUnknown), + status.MetricLabelToConditionStatus: string(metav1.ConditionTrue), + } + Expect(GetMetric("operator_customobject_status_condition_transition_seconds", recovered).GetHistogram().GetSampleCount()).To(BeNumerically(">", 0)) + // No sample is recorded for the same dwell under a different destination. + Expect(GetMetric("operator_customobject_status_condition_transition_seconds", map[string]string{ + pmetrics.LabelType: string(ConditionTypeFoo), + status.MetricLabelConditionStatus: string(metav1.ConditionUnknown), + status.MetricLabelToConditionStatus: string(metav1.ConditionFalse), + })).To(BeNil()) + // The deprecated (group/kind-labeled) variant carries to_status as well. + Expect(GetMetric("operator_status_condition_transition_seconds", lo.Assign(recovered, map[string]string{ + pmetrics.LabelGroup: gvk.Group, + pmetrics.LabelKind: gvk.Kind, + })).GetHistogram().GetSampleCount()).To(BeNumerically(">", 0)) + }) It("should emit metrics and events on a transition", func() { testObject := test.Object(&test.CustomObject{}) gvk := object.GVK(testObject) diff --git a/status/metrics.go b/status/metrics.go index a0d5881..96991e6 100644 --- a/status/metrics.go +++ b/status/metrics.go @@ -13,6 +13,12 @@ const ( MetricLabelNamespace = "namespace" MetricLabelName = "name" MetricLabelConditionStatus = "status" + // MetricLabelToConditionStatus labels the transition_seconds histogram with + // the status the condition transitioned TO. Combined with the existing + // "status" label (the status being left), this lets consumers distinguish + // how a dwell ended -- e.g. how long a condition was unhealthy + // (status="False") before it recovered (to_status="True"). + MetricLabelToConditionStatus = "to_status" ) const ( @@ -33,12 +39,13 @@ func conditionDurationMetric(objectName string, buckets []float64, additionalLab Namespace: pmetrics.Namespace, Subsystem: subsystem, Name: "transition_seconds", - Help: "The amount of time a condition was in a given state before transitioning. e.g. Alarm := P99(Updated=False) > 5 minutes", + Help: "The amount of time a condition was in a given state (status) before transitioning to another state (to_status). e.g. Alarm := P99(Updated=False) > 5 minutes", Buckets: buckets, }, append([]string{ pmetrics.LabelType, MetricLabelConditionStatus, + MetricLabelToConditionStatus, }, additionalLabels...), ) }