diff --git a/config/config.yaml b/config/config.yaml index 4bfc9d015..38bcc39d2 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -256,7 +256,7 @@ clickhouse: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: diff --git a/deploy/builder/templates-config/config.yaml b/deploy/builder/templates-config/config.yaml index 46740bd15..ff2b77ecb 100644 --- a/deploy/builder/templates-config/config.yaml +++ b/deploy/builder/templates-config/config.yaml @@ -250,7 +250,7 @@ clickhouse: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml index b7579fce9..2ffe5a7dc 100644 --- a/deploy/helm/clickhouse-operator/values.yaml +++ b/deploy/helm/clickhouse-operator/values.yaml @@ -531,7 +531,7 @@ configs: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: diff --git a/docs/chi-examples/70-chop-config.yaml b/docs/chi-examples/70-chop-config.yaml index b9f573ec1..8c68d300c 100644 --- a/docs/chi-examples/70-chop-config.yaml +++ b/docs/chi-examples/70-chop-config.yaml @@ -98,7 +98,7 @@ spec: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: diff --git a/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go b/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go index 9f0bc0d80..613bd5023 100644 --- a/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go +++ b/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go @@ -142,16 +142,20 @@ const ( type MetricsFetcher struct { connectionParams *clickhouse.EndpointConnectionParams tablesRegexp string + // Used to filter system-metric names while fetching metrics. Nil means keep all. + metricsFilter MetricsFilter } // NewMetricsFetcher creates new clickhouse fetcher object func NewMetricsFetcher( endpointConnectionParams *clickhouse.EndpointConnectionParams, tablesRegexp string, + metricsFilter MetricsFilter, ) *MetricsFetcher { return &MetricsFetcher{ connectionParams: endpointConnectionParams, tablesRegexp: tablesRegexp, + metricsFilter: metricsFilter, } } @@ -179,11 +183,9 @@ func (f *MetricsFetcher) buildMetricsSQL() string { } // getClickHouseQueryMetrics requests metrics data from ClickHouse. -// Exclusion of "noisy" metrics is enforced solely by the writer-side filter -// (see CHIPrometheusWriter.metricsFilter). A SQL-side filter was tried and -// dropped: wrapping the UNION-ALL chain in `FROM (...) WHERE NOT (...)` left -// the metrics query returning zero rows across restart-then-scrape windows; -// the writer-side filter is sufficient and avoids that fragility. +// Excluded names are dropped during row scan so they never enter the in-memory buffer. +// SQL-side filtering was tried and abandoned: wrapping the UNION-ALL in +// `FROM (...) WHERE NOT (...)` caused zero rows on restart-then-scrape windows. func (f *MetricsFetcher) getClickHouseQueryMetrics(ctx context.Context) (Table, error) { return f.clickHouseQueryScanRows( ctx, @@ -191,13 +193,20 @@ func (f *MetricsFetcher) getClickHouseQueryMetrics(ctx context.Context) (Table, func(rows *sql.Rows, data *Table) error { var metric, value, description, _type string if err := rows.Scan(&metric, &value, &description, &_type); err == nil { - *data = append(*data, []string{metric, value, description, _type}) + f.appendMetricRow(data, metric, value, description, _type) } return nil }, ) } +func (f *MetricsFetcher) appendMetricRow(data *Table, metric, value, description, _type string) { + if f.metricsFilter != nil && f.metricsFilter.IsExcluded(metric) { + return + } + *data = append(*data, []string{metric, value, description, _type}) +} + // getClickHouseSystemParts requests data sizes from ClickHouse func (f *MetricsFetcher) getClickHouseSystemParts(ctx context.Context) (Table, error) { return f.clickHouseQueryScanRows( diff --git a/pkg/metrics/clickhouse/exporter.go b/pkg/metrics/clickhouse/exporter.go index ac494ab02..f2acf64ed 100644 --- a/pkg/metrics/clickhouse/exporter.go +++ b/pkg/metrics/clickhouse/exporter.go @@ -135,6 +135,7 @@ func (e *Exporter) newHostFetcher(host *metrics.WatchedHost) *MetricsFetcher { return NewMetricsFetcher( clusterConnectionParams.NewEndpointConnectionParams(host.Hostname), chop.Config().ClickHouse.Metrics.TablesRegexp, + e.metricsFilter, ) }