CNV-87535: k8s: add orphan AlertRelabelConfig GC

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/openshift/library-go v0.0.0-20240905123346-5bdbfe35a6f5
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.87.0
 	github.com/prometheus-operator/prometheus-operator/pkg/client v0.87.0
+	github.com/prometheus/client_golang v1.23.2
 	github.com/prometheus/common v0.67.4
 	github.com/prometheus/prometheus v0.308.0
 	github.com/sirupsen/logrus v1.9.3
@@ -52,12 +53,12 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect

pkg/k8s/alert_relabel_config_gc.go

@@ -0,0 +1,64 @@
+package k8s
+
+import (
+	"context"
+
+	"github.com/openshift/monitoring-plugin/pkg/managementlabels"
+)
+
+// gcOrphanedARCs deletes AlertRelabelConfigs whose associated alert rule no
+// longer exists. This handles the case where an operator (or manual action)
+// removes rules from a PrometheusRule or deletes the CR entirely — the
+// AlertRelabelConfigs that were created by the plugin for
+// classification/drop/stamp become orphans.
+//
+// Only AlertRelabelConfigs carrying the plugin's alertRuleId annotation
+// are considered. GitOps-managed configs are never deleted automatically;
+// scrapeable metrics surface them so cluster-monitoring-operator can alert.
+//
+// liveRuleIDs must include every alerting-rule ID still present on a
+// PrometheusRule, including platform rules dropped by relabel configs.
+// IDs are recorded in collectAlerts before the drop continue, so a Drop
+// AlertRelabelConfig for a disabled rule is not treated as an orphan.
+func (rrm *relabeledRulesManager) gcOrphanedARCs(ctx context.Context, liveRuleIDs map[string]struct{}) {
+	if rrm.alertRelabelConfigs == nil {
+		return
+	}
+
+	metrics := rrm.gcMetrics
+	arcs, err := rrm.alertRelabelConfigs.List(ctx, "")
+	if err != nil {
+		metrics.observeListError()
+		log.Errorf("orphan AlertRelabelConfig cleanup: failed to list AlertRelabelConfigs: %v", err)
+		return
+	}
+
+	gitOpsOrphans := 0
+	for i := range arcs {
+		arc := &arcs[i]
+
+		ruleID, ok := arc.Annotations[managementlabels.ARCAnnotationAlertRuleIDKey]
+		if !ok || ruleID == "" {
+			continue
+		}
+
+		if _, alive := liveRuleIDs[ruleID]; alive {
+			continue
+		}
+
+		if IsManagedByGitOps(arc.Annotations, arc.Labels) {
+			gitOpsOrphans++
+			log.Warnf("orphan AlertRelabelConfig cleanup: AlertRelabelConfig %s/%s (ruleId=%s) is orphaned but GitOps-managed — skipping deletion, manual cleanup required", arc.Namespace, arc.Name, ruleID)
+			continue
+		}
+
+		if err := rrm.alertRelabelConfigs.Delete(ctx, arc.Namespace, arc.Name); err != nil {
+			metrics.observeDeleteError()
+			log.Errorf("orphan AlertRelabelConfig cleanup: failed to delete AlertRelabelConfig %s/%s: %v", arc.Namespace, arc.Name, err)
+			continue
+		}
+
+		log.Infof("orphan AlertRelabelConfig cleanup: deleted orphaned AlertRelabelConfig %s/%s (ruleId=%s)", arc.Namespace, arc.Name, ruleID)
+	}
+	metrics.setGitOpsOrphans(float64(gitOpsOrphans))
+}

pkg/k8s/alert_relabel_config_gc_metrics.go

@@ -0,0 +1,89 @@
+package k8s
+
+import (
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+const (
+	MetricAlertRelabelConfigGCListErrorsTotal   = "monitoring_plugin_alert_relabel_config_gc_list_errors_total"
+	MetricAlertRelabelConfigGCDeleteErrorsTotal = "monitoring_plugin_alert_relabel_config_gc_delete_errors_total"
+	MetricAlertRelabelConfigGitOpsOrphans       = "monitoring_plugin_alert_relabel_config_gitops_orphans"
+)
+
+type alertRelabelConfigGCMetrics struct {
+	listErrors    prometheus.Counter
+	deleteErrors  prometheus.Counter
+	gitopsOrphans prometheus.Gauge
+}
+
+func newAlertRelabelConfigGCMetrics() *alertRelabelConfigGCMetrics {
+	return &alertRelabelConfigGCMetrics{
+		listErrors: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: MetricAlertRelabelConfigGCListErrorsTotal,
+			Help: "Count of failed List calls while cleaning up orphaned AlertRelabelConfigs.",
+		}),
+		deleteErrors: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: MetricAlertRelabelConfigGCDeleteErrorsTotal,
+			Help: "Count of failed Delete calls while cleaning up orphaned AlertRelabelConfigs.",
+		}),
+		gitopsOrphans: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: MetricAlertRelabelConfigGitOpsOrphans,
+			Help: "Number of GitOps-managed AlertRelabelConfigs that are orphaned and were not deleted.",
+		}),
+	}
+}
+
+func (m *alertRelabelConfigGCMetrics) mustRegister(reg prometheus.Registerer) {
+	reg.MustRegister(m.listErrors, m.deleteErrors, m.gitopsOrphans)
+}
+
+func (m *alertRelabelConfigGCMetrics) observeListError() {
+	if m == nil {
+		return
+	}
+	m.listErrors.Inc()
+}
+
+func (m *alertRelabelConfigGCMetrics) observeDeleteError() {
+	if m == nil {
+		return
+	}
+	m.deleteErrors.Inc()
+}
+
+func (m *alertRelabelConfigGCMetrics) setGitOpsOrphans(n float64) {
+	if m == nil {
+		return
+	}
+	m.gitopsOrphans.Set(n)
+}
+
+var (
+	alertRelabelConfigGCMetricsRegistry = prometheus.NewRegistry()
+	defaultAlertRelabelConfigGCMetrics  = newAlertRelabelConfigGCMetrics()
+)
+
+func init() {
+	defaultAlertRelabelConfigGCMetrics.mustRegister(alertRelabelConfigGCMetricsRegistry)
+}
+
+// AlertRelabelConfigGCMetricsRegistry is the registry served at /metrics
+// when alert-management-api is enabled. Additional collectors should
+// register here so a single scrape endpoint exposes all series.
+func AlertRelabelConfigGCMetricsRegistry() *prometheus.Registry {
+	return alertRelabelConfigGCMetricsRegistry
+}
+
+// AlertRelabelConfigGCMetricsHandler serves the GC metrics registry.
+func AlertRelabelConfigGCMetricsHandler() http.Handler {
+	return promhttp.HandlerFor(alertRelabelConfigGCMetricsRegistry, promhttp.HandlerOpts{})
+}
+
+// EmptyMetricsHandler serves an empty Prometheus registry so /metrics
+// still returns 200 when alert-management-api is off.
+func EmptyMetricsHandler() http.Handler {
+	return promhttp.HandlerFor(prometheus.NewRegistry(), promhttp.HandlerOpts{})
+}

pkg/k8s/alert_relabel_config_gc_metrics_test.go

@@ -0,0 +1,99 @@
+package k8s
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+
+	osmv1 "github.com/openshift/api/monitoring/v1"
+)
+
+func TestGCOrphanedARCs_ListErrorIncrementsMetric(t *testing.T) {
+	metrics := newAlertRelabelConfigGCMetrics()
+	mock := &mockARCInterface{listErr: errors.New("list failed")}
+	rrm := &relabeledRulesManager{alertRelabelConfigs: mock, gcMetrics: metrics}
+
+	rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{})
+
+	if got := testutil.ToFloat64(metrics.listErrors); got != 1 {
+		t.Fatalf("list errors = %v, want 1", got)
+	}
+	if got := testutil.ToFloat64(metrics.deleteErrors); got != 0 {
+		t.Fatalf("delete errors = %v, want 0", got)
+	}
+}
+
+func TestGCOrphanedARCs_DeleteErrorIncrementsMetric(t *testing.T) {
+	metrics := newAlertRelabelConfigGCMetrics()
+	mock := &mockARCInterface{
+		arcs: map[string]*osmv1.AlertRelabelConfig{
+			"openshift-monitoring/arc-orphan": newARC("openshift-monitoring", "arc-orphan", "rule-gone", nil, nil),
+		},
+		deleteErr: errors.New("delete failed"),
+	}
+	rrm := &relabeledRulesManager{alertRelabelConfigs: mock, gcMetrics: metrics}
+
+	rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{})
+
+	if got := testutil.ToFloat64(metrics.deleteErrors); got != 1 {
+		t.Fatalf("delete errors = %v, want 1", got)
+	}
+	if len(mock.deleted) != 0 {
+		t.Fatalf("expected no deletions, got %v", mock.deleted)
+	}
+}
+
+func TestGCOrphanedARCs_GitOpsOrphanSetsGauge(t *testing.T) {
+	metrics := newAlertRelabelConfigGCMetrics()
+	mock := &mockARCInterface{
+		arcs: map[string]*osmv1.AlertRelabelConfig{
+			"openshift-monitoring/arc-gitops": newARC("openshift-monitoring", "arc-gitops", "rule-gone",
+				map[string]string{"argocd.argoproj.io/tracking-id": "some-id"}, nil),
+			"openshift-monitoring/arc-live": newARC("openshift-monitoring", "arc-live", "rule-alive", nil, nil),
+		},
+	}
+	rrm := &relabeledRulesManager{alertRelabelConfigs: mock, gcMetrics: metrics}
+
+	rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{"rule-alive": {}})
+
+	if got := testutil.ToFloat64(metrics.gitopsOrphans); got != 1 {
+		t.Fatalf("gitops orphans = %v, want 1", got)
+	}
+}
+
+func TestAlertRelabelConfigGCMetricsHandlerExposesSeries(t *testing.T) {
+	req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	rec := httptest.NewRecorder()
+	AlertRelabelConfigGCMetricsHandler().ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status %d", rec.Code)
+	}
+	body := rec.Body.String()
+	for _, name := range []string{
+		MetricAlertRelabelConfigGCListErrorsTotal,
+		MetricAlertRelabelConfigGCDeleteErrorsTotal,
+		MetricAlertRelabelConfigGitOpsOrphans,
+	} {
+		if !strings.Contains(body, name) {
+			t.Errorf("handler body missing metric %s:\n%s", name, body)
+		}
+	}
+}
+
+func TestEmptyMetricsHandlerHasNoGCSeries(t *testing.T) {
+	req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	rec := httptest.NewRecorder()
+	EmptyMetricsHandler().ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status %d", rec.Code)
+	}
+	body := rec.Body.String()
+	if strings.Contains(body, MetricAlertRelabelConfigGCListErrorsTotal) {
+		t.Fatalf("empty handler unexpectedly exposed GC metrics: %s", body)
+	}
+}