Apply filter when scan SQL results

config/config.yaml

@@ -256,7 +256,7 @@ clickhouse:
     # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.
     # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics.
     tablesRegexp: "^(metrics|custom_metrics)$"
-    # List of regexps to match ClickHouse metrics to exclude from export.
+    # List of regexps to match ClickHouse metrics to exclude from collection/export.
     # Regexps match internal metric names before Prometheus normalization and prefixing.
     # Default is the per-CPU OS metrics filter shown below; set to [] to disable.
     excludeRegexp:

deploy/builder/templates-config/config.yaml

@@ -250,7 +250,7 @@ clickhouse:
     # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.
     # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics.
     tablesRegexp: "^(metrics|custom_metrics)$"
-    # List of regexps to match ClickHouse metrics to exclude from export.
+    # List of regexps to match ClickHouse metrics to exclude from collection/export.
     # Regexps match internal metric names before Prometheus normalization and prefixing.
     # Default is the per-CPU OS metrics filter shown below; set to [] to disable.
     excludeRegexp:

deploy/helm/clickhouse-operator/values.yaml

@@ -531,7 +531,7 @@ configs:
           # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.
           # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics.
           tablesRegexp: "^(metrics|custom_metrics)$"
-          # List of regexps to match ClickHouse metrics to exclude from export.
+          # List of regexps to match ClickHouse metrics to exclude from collection/export.
           # Regexps match internal metric names before Prometheus normalization and prefixing.
           # Default is the per-CPU OS metrics filter shown below; set to [] to disable.
           excludeRegexp:

docs/chi-examples/70-chop-config.yaml

@@ -98,7 +98,7 @@ spec:
       # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.
       # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics.
       tablesRegexp: "^(metrics|custom_metrics)$"
-      # List of regexps to match ClickHouse metrics to exclude from export.
+      # List of regexps to match ClickHouse metrics to exclude from collection/export.
       # Regexps match internal metric names before Prometheus normalization and prefixing.
       # Default is the per-CPU OS metrics filter shown below; set to [] to disable.
       excludeRegexp:

pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go

@@ -142,16 +142,20 @@ const (
 type MetricsFetcher struct {
 	connectionParams *clickhouse.EndpointConnectionParams
 	tablesRegexp     string
+	// Used to filter system-metric names while fetching metrics. Nil means keep all.
+	metricsFilter MetricsFilter
 }
 
 // NewMetricsFetcher creates new clickhouse fetcher object
 func NewMetricsFetcher(
 	endpointConnectionParams *clickhouse.EndpointConnectionParams,
 	tablesRegexp string,
+	metricsFilter MetricsFilter,
 ) *MetricsFetcher {
 	return &MetricsFetcher{
 		connectionParams: endpointConnectionParams,
 		tablesRegexp:     tablesRegexp,
+		metricsFilter:    metricsFilter,
 	}
 }
 
@@ -179,25 +183,30 @@ func (f *MetricsFetcher) buildMetricsSQL() string {
 }
 
 // getClickHouseQueryMetrics requests metrics data from ClickHouse.
-// Exclusion of "noisy" metrics is enforced solely by the writer-side filter
-// (see CHIPrometheusWriter.metricsFilter). A SQL-side filter was tried and
-// dropped: wrapping the UNION-ALL chain in `FROM (...) WHERE NOT (...)` left
-// the metrics query returning zero rows across restart-then-scrape windows;
-// the writer-side filter is sufficient and avoids that fragility.
+// Excluded names are dropped during row scan so they never enter the in-memory buffer.
+// SQL-side filtering was tried and abandoned: wrapping the UNION-ALL in
+// `FROM (...) WHERE NOT (...)` caused zero rows on restart-then-scrape windows.
 func (f *MetricsFetcher) getClickHouseQueryMetrics(ctx context.Context) (Table, error) {
 	return f.clickHouseQueryScanRows(
 		ctx,
 		f.buildMetricsSQL(),
 		func(rows *sql.Rows, data *Table) error {
 			var metric, value, description, _type string
 			if err := rows.Scan(&metric, &value, &description, &_type); err == nil {
-				*data = append(*data, []string{metric, value, description, _type})
+				f.appendMetricRow(data, metric, value, description, _type)
 			}
 			return nil
 		},
 	)
 }
 
+func (f *MetricsFetcher) appendMetricRow(data *Table, metric, value, description, _type string) {
+	if f.metricsFilter != nil && f.metricsFilter.IsExcluded(metric) {
+		return
+	}
+	*data = append(*data, []string{metric, value, description, _type})
+}
+
 // getClickHouseSystemParts requests data sizes from ClickHouse
 func (f *MetricsFetcher) getClickHouseSystemParts(ctx context.Context) (Table, error) {
 	return f.clickHouseQueryScanRows(

pkg/metrics/clickhouse/exporter.go

@@ -135,6 +135,7 @@ func (e *Exporter) newHostFetcher(host *metrics.WatchedHost) *MetricsFetcher {
 	return NewMetricsFetcher(
 		clusterConnectionParams.NewEndpointConnectionParams(host.Hostname),
 		chop.Config().ClickHouse.Metrics.TablesRegexp,
+		e.metricsFilter,
 	)
 }