feat(beholder): reduce PerWorkflow histogram buckets 16→8

pkg/beholder/client.go

@@ -549,11 +549,11 @@ func newMeterProvider(cfg Config, resource *sdkresource.Resource, auth Auth, cre
 	for _, p := range cfg.MetricProducers {
 		readerOpts = append(readerOpts, sdkmetric.WithProducer(p))
 	}
-	return sdkmetric.NewMeterProvider(
+	mpOpts := appendMeterProviderOptions(cfg,
 		sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter, readerOpts...)),
 		sdkmetric.WithResource(resource),
-		sdkmetric.WithView(cfg.MetricViews...),
-	), nil
+	)
+	return sdkmetric.NewMeterProvider(mpOpts...), nil
 }
 
 // newLoggerOpts creates options for a logger exporter

pkg/beholder/config.go

@@ -26,7 +26,7 @@ type Config struct {
 	EmitterMaxQueueSize       int
 	// EmitterBatchProcessor controls custom-message export mode:
 	// true = batched async export; false = immediate per-record export.
-	EmitterBatchProcessor     bool
+	EmitterBatchProcessor bool
 
 	// OTel Trace
 	TraceSampleRatio  float64
@@ -40,6 +40,11 @@ type Config struct {
 	MetricReaderInterval time.Duration
 	MetricRetryConfig    *RetryConfig
 	MetricViews          []metric.View
+	// MetricViewsDisabled skips DefaultViews attribute filtering (for tests).
+	MetricViewsDisabled bool
+	// MetricCardinalityLimit sets the SDK per-instrument attribute-set limit (0 = disabled).
+	// DefaultConfig uses 10000 as a production safety valve for high-cardinality workloads.
+	MetricCardinalityLimit int
 	// MetricCompressor sets the gRPC compressor for metrics. Valid values: "gzip" (default), "none".
 	MetricCompressor string
 	MetricProducers  []metric.Producer // For example, a prometheus bridge
@@ -127,7 +132,7 @@ func DefaultConfig() Config {
 		EmitterExportInterval:     1 * time.Second,
 		EmitterMaxQueueSize:       2048,
 		// Keep batched export enabled by default for throughput.
-		EmitterBatchProcessor:     true,
+		EmitterBatchProcessor: true,
 		// OTel message log exporter retry config
 		LogRetryConfig: defaultRetryConfig.Copy(),
 		// Trace
@@ -137,8 +142,9 @@ func DefaultConfig() Config {
 		// OTel trace exporter retry config
 		TraceRetryConfig: defaultRetryConfig.Copy(),
 		// Metric
-		MetricReaderInterval: 1 * time.Second,
-		MetricCompressor:     "gzip",
+		MetricReaderInterval:   1 * time.Second,
+		MetricCompressor:       "gzip",
+		MetricCardinalityLimit: 10000,
 		// OTel metric exporter retry config
 		MetricRetryConfig: defaultRetryConfig.Copy(),
 		// Log
@@ -148,8 +154,8 @@ func DefaultConfig() Config {
 		LogMaxQueueSize:       2048,
 		LogBatchProcessor:     true,
 		LogStreamingEnabled:   true, // Enable logs streaming by default
-		LogLevel:      zapcore.InfoLevel,
-		LogCompressor: "gzip",
+		LogLevel:              zapcore.InfoLevel,
+		LogCompressor:         "gzip",
 		// Chip Ingress Batch Emitter
 		ChipIngressBatchEmitterEnabled: false,
 		ChipIngressBufferSize:          1000,
@@ -173,6 +179,7 @@ func TestDefaultConfig() Config {
 	config.LogRetryConfig.MaxElapsedTime = 0    // Retry is disabled
 	config.TraceRetryConfig.MaxElapsedTime = 0  // Retry is disabled
 	config.MetricRetryConfig.MaxElapsedTime = 0 // Retry is disabled
+	config.MetricCardinalityLimit = 0           // Disable overflow aggregation in unit tests
 	// Auth disabled for testing (TTL=0 means static auth mode)
 	config.AuthHeadersTTL = 0
 	return config
@@ -186,6 +193,7 @@ func TestDefaultConfigHTTPClient() Config {
 	config.LogBatchProcessor = false
 	config.OtelExporterGRPCEndpoint = ""
 	config.OtelExporterHTTPEndpoint = "localhost:4318"
+	config.MetricCardinalityLimit = 0
 	// Auth disabled for testing (TTL=0 means static auth mode)
 	config.AuthHeadersTTL = 0
 	return config

pkg/beholder/config_test.go

@@ -53,8 +53,10 @@ func TestConfig(t *testing.T) {
 		// OTel trace exporter retry config
 		TraceRetryConfig: nil,
 		// Metric
-		MetricReaderInterval: 1 * time.Second,
-		MetricCompressor:     "gzip",
+		MetricReaderInterval:   1 * time.Second,
+		MetricCompressor:       "gzip",
+		MetricViewsDisabled:    false,
+		MetricCardinalityLimit: 0,
 		// OTel metric exporter retry config
 		MetricRetryConfig: nil,
 		// Log

pkg/beholder/httpclient.go

@@ -285,14 +285,13 @@ func newHTTPMeterProvider(config Config, resource *sdkresource.Resource, tlsConf
 		return nil, err
 	}
 
-	mp := sdkmetric.NewMeterProvider(
+	mpOpts := appendMeterProviderOptions(config,
 		sdkmetric.WithReader(
 			sdkmetric.NewPeriodicReader(
 				exporter,
 				sdkmetric.WithInterval(config.MetricReaderInterval), // Default is 10s
 			)),
 		sdkmetric.WithResource(resource),
-		sdkmetric.WithView(config.MetricViews...),
 	)
-	return mp, nil
+	return sdkmetric.NewMeterProvider(mpOpts...), nil
 }

pkg/beholder/meter_provider.go

@@ -0,0 +1,37 @@
+package beholder
+
+import (
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+
+	"github.com/smartcontractkit/chainlink-common/pkg/beholder/metricviews"
+)
+
+// mergeMetricViews builds the view list passed to sdkmetric.WithView.
+//
+// Final order:
+//  1. cfg.MetricViews — caller overrides (e.g. chainlink metricViews() histogram buckets)
+//  2. metricviews.DefaultViews() — attribute filters for high-cardinality labels
+//
+// Caller views must come first: when multiple views match an instrument and
+// resolve to the same output stream (same name/description/unit/kind), the SDK
+// keeps only the first in registration order and drops the rest. If the default
+// "*" denylist ran first, caller histogram-bucket views would be silently dropped.
+//
+// A consequence is that filters do not compose: once a caller view wins for a
+// stream, the default attribute filter no longer applies to it. That is acceptable
+// because caller views target metrics that do not emit the denied labels; capability
+// trigger metrics carry no caller view and fall through to the defaults below.
+func mergeMetricViews(cfg Config) []sdkmetric.View {
+	if cfg.MetricViewsDisabled {
+		return cfg.MetricViews
+	}
+	return append(append([]sdkmetric.View{}, cfg.MetricViews...), metricviews.DefaultViews()...)
+}
+
+func appendMeterProviderOptions(cfg Config, opts ...sdkmetric.Option) []sdkmetric.Option {
+	opts = append(opts, sdkmetric.WithView(mergeMetricViews(cfg)...))
+	if cfg.MetricCardinalityLimit > 0 {
+		opts = append(opts, sdkmetric.WithCardinalityLimit(cfg.MetricCardinalityLimit))
+	}
+	return opts
+}

pkg/beholder/meter_provider_test.go

@@ -0,0 +1,67 @@
+package beholder
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+
+	"github.com/smartcontractkit/chainlink-common/pkg/beholder/metricviews"
+)
+
+func TestAppendMeterProviderOptions_cardinalityLimit(t *testing.T) {
+	t.Parallel()
+
+	const (
+		uniqueAttributes = 10
+		limit            = 5
+	)
+
+	reader := sdkmetric.NewManualReader()
+	cfg := DefaultConfig()
+	cfg.MetricCardinalityLimit = limit
+	cfg.MetricViewsDisabled = true
+
+	mpOpts := appendMeterProviderOptions(cfg, sdkmetric.WithReader(reader))
+	mp := sdkmetric.NewMeterProvider(mpOpts...)
+	t.Cleanup(func() { _ = mp.Shutdown(context.Background()) })
+
+	meter := mp.Meter("test")
+	counter, err := meter.Int64Counter("overflow_test_total")
+	require.NoError(t, err)
+
+	for i := range uniqueAttributes {
+		counter.Add(context.Background(), 1, metric.WithAttributes(attribute.Int("key", i)))
+	}
+
+	var rm metricdata.ResourceMetrics
+	require.NoError(t, reader.Collect(context.Background(), &rm))
+
+	sum := rm.ScopeMetrics[0].Metrics[0].Data.(metricdata.Sum[int64])
+	assert.Len(t, sum.DataPoints, limit)
+
+	var total int64
+	for _, dp := range sum.DataPoints {
+		total += dp.Value
+	}
+	assert.Equal(t, int64(uniqueAttributes), total)
+}
+
+func TestMergeMetricViews_prependsDefaults(t *testing.T) {
+	t.Parallel()
+
+	views := mergeMetricViews(Config{
+		MetricViews: []sdkmetric.View{
+			sdkmetric.NewView(
+				sdkmetric.Instrument{Name: "custom_metric"},
+				sdkmetric.Stream{},
+			),
+		},
+	})
+	require.GreaterOrEqual(t, len(views), len(metricviews.DefaultViews())+1)
+}

pkg/beholder/metricviews/perworkflow_histograms.go

@@ -0,0 +1,98 @@
+package metricviews
+
+import sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+
+const perWorkflowInstrumentGlob = "*.PerWorkflow.*"
+
+// OTel SDK default histogram boundaries (15 values → 16 Prometheus buckets).
+// PerWorkflow limit metrics from pkg/settings/limits use the default because
+// they do not pass metric.WithExplicitBucketBoundaries at creation time.
+var (
+	perWorkflowBytesBoundaries = []float64{
+		0, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8,
+	}
+	perWorkflowSecondsBoundaries = []float64{
+		0, 1, 10, 60, 300, 900, 3600,
+	}
+	// perWorkflowGasBoundaries covers pkg/settings/cresettings ChainWrite gas
+	// limit defaults (Solana 300_000, Aptos 2_000_000, EVM 5_000_000, up to
+	// 50_000_000 for per-chain-selector overrides) without collapsing them
+	// into the +Inf overflow bucket.
+	perWorkflowGasBoundaries = []float64{
+		0, 1e5, 5e5, 1e6, 5e6, 1e7, 5e7,
+	}
+	// perWorkflowCountBoundaries is the fallback for PerWorkflow histograms
+	// whose unit is neither "By", "s", nor "{gas}" (e.g. dimensionless counts).
+	perWorkflowCountBoundaries = []float64{
+		0, 1, 10, 100, 1e3, 1e4, 1e5,
+	}
+)
+
+// perWorkflowHistogramViews returns bucket-boundary overrides for
+// *.PerWorkflow.* histograms, keyed by unit. Each view also carries
+// globalHighCardinalityDeny so the attribute deny-list travels with the
+// bucket override: see the ordering note on DefaultViews for why a view
+// registered here would otherwise silently bypass the deny-filter views
+// below it.
+//
+// The unit-less (count) view matches any unit and only acts as a fallback
+// because the more specific By/s/gas views are registered first: the OTel
+// SDK dedupes views that resolve to the same stream identity (name/
+// description/unit/kind) and keeps only the first match, so an instrument
+// with unit "By" is already claimed by perWorkflowBytesBoundaries by the
+// time the count view is evaluated for it.
+func perWorkflowHistogramViews() []sdkmetric.View {
+	return []sdkmetric.View{
+		sdkmetric.NewView(
+			sdkmetric.Instrument{
+				Name: perWorkflowInstrumentGlob,
+				Kind: sdkmetric.InstrumentKindHistogram,
+				Unit: "By",
+			},
+			sdkmetric.Stream{
+				Aggregation: sdkmetric.AggregationExplicitBucketHistogram{
+					Boundaries: perWorkflowBytesBoundaries,
+				},
+				AttributeFilter: globalHighCardinalityDeny,
+			},
+		),
+		sdkmetric.NewView(
+			sdkmetric.Instrument{
+				Name: perWorkflowInstrumentGlob,
+				Kind: sdkmetric.InstrumentKindHistogram,
+				Unit: "s",
+			},
+			sdkmetric.Stream{
+				Aggregation: sdkmetric.AggregationExplicitBucketHistogram{
+					Boundaries: perWorkflowSecondsBoundaries,
+				},
+				AttributeFilter: globalHighCardinalityDeny,
+			},
+		),
+		sdkmetric.NewView(
+			sdkmetric.Instrument{
+				Name: perWorkflowInstrumentGlob,
+				Kind: sdkmetric.InstrumentKindHistogram,
+				Unit: "{gas}",
+			},
+			sdkmetric.Stream{
+				Aggregation: sdkmetric.AggregationExplicitBucketHistogram{
+					Boundaries: perWorkflowGasBoundaries,
+				},
+				AttributeFilter: globalHighCardinalityDeny,
+			},
+		),
+		sdkmetric.NewView(
+			sdkmetric.Instrument{
+				Name: perWorkflowInstrumentGlob,
+				Kind: sdkmetric.InstrumentKindHistogram,
+			},
+			sdkmetric.Stream{
+				Aggregation: sdkmetric.AggregationExplicitBucketHistogram{
+					Boundaries: perWorkflowCountBoundaries,
+				},
+				AttributeFilter: globalHighCardinalityDeny,
+			},
+		),
+	}
+}

pkg/beholder/metricviews/views.go

@@ -0,0 +1,73 @@
+// Package metricviews defines Beholder's default OTel metric views for
+// cardinality control: PerWorkflow histogram bucket reduction and
+// attribute-filter deny/allow lists.
+//
+// Callers (e.g. chainlink core/cmd/shell.go via beholder.Config.MetricViews)
+// may supply additional views—typically histogram bucket Aggregation overrides
+// for specific instrument names. Beholder merges caller views before these
+// defaults (see beholder.mergeMetricViews); callers do not need to invoke
+// DefaultViews themselves.
+//
+// An instrument may match multiple views. When several matching views resolve
+// to the same output stream (same name/description/unit/kind), the SDK keeps
+// only the first in registration order and logs a duplicate-stream warning for
+// the rest; attribute filters and aggregations do not compose across them.
+// Because Beholder registers caller views ahead of these defaults, a matching
+// caller view wins and the default attribute filter for that stream is dropped.
+package metricviews
+
+import (
+	"go.opentelemetry.io/otel/attribute"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+)
+
+const (
+	baseTriggerInstrumentGlob  = "capabilities_base_trigger_*"
+	stoppedResendingInstrument = "capabilities_base_trigger_stopped_resending_timestamp"
+)
+
+var (
+	globalHighCardinalityDeny = attribute.NewDenyKeysFilter(
+		attribute.Key("event_id"),
+		attribute.Key("trigger_id"),
+		attribute.Key("workflow_execution_id"),
+	)
+
+	baseTriggerAllow = attribute.NewAllowKeysFilter(
+		attribute.Key("capability_id"),
+		attribute.Key("reason"),
+		attribute.Key("outcome"),
+	)
+
+	stoppedResendingAllow = attribute.NewAllowKeysFilter(
+		attribute.Key("capability_id"),
+		attribute.Key("trigger_id"),
+	)
+)
+
+// DefaultViews returns views appended after caller-supplied MetricViews by
+// beholder.mergeMetricViews. PerWorkflow histogram bucket views are registered
+// first so they apply to CRE limit metrics (chainlink does not supply caller
+// views for those names); each carries globalHighCardinalityDeny directly on
+// its Stream mask so claiming the stream identity for a bucket override does
+// not also bypass the deny-filter views below (see the dedup note above).
+// Attribute-filter views follow; within that group, more specific instrument
+// matchers precede the global "*" catch-all.
+func DefaultViews() []sdkmetric.View {
+	views := perWorkflowHistogramViews()
+	views = append(views,
+		sdkmetric.NewView(
+			sdkmetric.Instrument{Name: stoppedResendingInstrument},
+			sdkmetric.Stream{AttributeFilter: stoppedResendingAllow},
+		),
+		sdkmetric.NewView(
+			sdkmetric.Instrument{Name: baseTriggerInstrumentGlob},
+			sdkmetric.Stream{AttributeFilter: baseTriggerAllow},
+		),
+		sdkmetric.NewView(
+			sdkmetric.Instrument{Name: "*"},
+			sdkmetric.Stream{AttributeFilter: globalHighCardinalityDeny},
+		),
+	)
+	return views
+}