Skip to content

Add reduced Beholder metric resources #2249

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
pkcll wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

Add reduced Beholder metric resources #2249

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
56e7e08
Add reduced Beholder metric resources
pkcll Jul 13, 2026
dbde6fa
Fix reduced resource filtering
pkcll Jul 14, 2026
8d37a24
Test reduced resource edge cases
pkcll Jul 14, 2026
417fe69
Use telemetry resource node identity
pkcll Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 38 additions & 48 deletions pkg/beholder/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ type Client struct {
// lazySigner allows updating the keystore after client initialization.
lazySigner *lazySigner

// fullResource retains all resource attributes for the info gauge export.
fullResource *sdkresource.Resource

// OnClose
OnClose func() error
}
Expand Down Expand Up @@ -111,10 +114,12 @@ type otlploggrpcFactory func(options ...otlploggrpc.Option) (sdklog.Exporter, er
// NewGRPCClient creates a GRPC based beholder Client. Use NewClient to create a client from a Config which will pick
// the best client type from the Config.
func NewGRPCClient(cfg Config, otlploggrpcNew otlploggrpcFactory) (*Client, error) {
baseResource, err := newOtelResource(cfg)
resources, err := buildOtelResources(cfg)
if err != nil {
return nil, err
}
fullResource := resources.Full
metricResource := resources.Metric
creds := insecure.NewCredentials()
if !cfg.InsecureConnection && cfg.CACertFile != "" {
creds, err = credentials.NewClientTLSFromFile(cfg.CACertFile, "")
Expand All @@ -133,14 +138,14 @@ func NewGRPCClient(cfg Config, otlploggrpcNew otlploggrpcFactory) (*Client, erro
}

// Tracer
tracerProvider, err := newTracerProvider(cfg, baseResource, auth, creds)
tracerProvider, err := newTracerProvider(cfg, fullResource, auth, creds)
if err != nil {
return nil, err
}
tracer := tracerProvider.Tracer(defaultPackageName)

// Meter
meterProvider, err := newMeterProvider(cfg, baseResource, auth, creds)
meterProvider, err := newMeterProvider(cfg, metricResource, auth, creds)
if err != nil {
return nil, err
}
Expand All @@ -161,7 +166,7 @@ func NewGRPCClient(cfg Config, otlploggrpcNew otlploggrpcFactory) (*Client, erro
if !cfg.LogStreamingEnabled {
loggerProvider = BeholderNoopLoggerProvider()
} else {
loggerOpts, err := newLoggerProviderOpts(cfg, baseResource, sharedLogExporter)
loggerOpts, err := newLoggerProviderOpts(cfg, fullResource, sharedLogExporter)
if err != nil {
return nil, err
}
Expand All @@ -170,7 +175,7 @@ func NewGRPCClient(cfg Config, otlploggrpcNew otlploggrpcFactory) (*Client, erro
logger := loggerProvider.Logger(defaultPackageName)

// Message emitter
messageLoggerOpts, err := newMessageLoggerProviderOpts(cfg, baseResource, sharedLogExporter)
messageLoggerOpts, err := newMessageLoggerProviderOpts(cfg, fullResource, sharedLogExporter)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -265,9 +270,15 @@ func NewGRPCClient(cfg Config, otlploggrpcNew otlploggrpcFactory) (*Client, erro
MeterProvider: meterProvider,
MessageLoggerProvider: messageLoggerProvider,
lazySigner: signer,
fullResource: fullResource,
OnClose: onClose,
}
c.initService(lggr, batchEmitterService)
if cfg.metricResourceFilteringEnabled() {
if err := c.RecordFullResourceAttributesMetric(context.Background()); err != nil {
return nil, err
}
}
return c, nil
}

Expand Down Expand Up @@ -331,49 +342,6 @@ func (c *Client) IsSignerSet() bool {
return c.lazySigner != nil && c.lazySigner.IsSet()
}

func newOtelResource(cfg Config) (resource *sdkresource.Resource, err error) {
extraResources, err := sdkresource.New(
context.Background(),
sdkresource.WithOS(),
sdkresource.WithContainer(),
sdkresource.WithHost(),
)
if err != nil {
return nil, err
}
resource, err = sdkresource.Merge(
sdkresource.Default(),
extraResources,
)
if err != nil {
return nil, err
}

// Add csa public key resource attribute
csaPublicKeyHex := "not-configured"
if len(cfg.AuthPublicKeyHex) > 0 {
csaPublicKeyHex = cfg.AuthPublicKeyHex
}
csaPublicKeyAttr := attribute.String("csa_public_key", csaPublicKeyHex)
resource, err = sdkresource.Merge(
sdkresource.NewSchemaless(csaPublicKeyAttr),
resource,
)
if err != nil {
return nil, err
}

// Add custom resource attributes
resource, err = sdkresource.Merge(
sdkresource.NewSchemaless(cfg.ResourceAttributes...),
resource,
)
if err != nil {
return nil, err
}
return
}

// RecordConfig records the beholder config as a metric.
func (c *Client) RecordConfigMetric(ctx context.Context) error {
configGauge, configAttrs, err := createConfigMetric(c.Meter, c.Config)
Expand All @@ -384,6 +352,22 @@ func (c *Client) RecordConfigMetric(ctx context.Context) error {
return nil
}

// RecordFullResourceAttributesMetric records all full-resource attributes on a
// single info gauge. The metric resource itself remains reduced when enabled.
func (c *Client) RecordFullResourceAttributesMetric(ctx context.Context) error {
resourceGauge, err := c.Meter.Int64Gauge(
"beholder.otel.resource_attributes.full",
otelmetric.WithDescription("Full OTel resource attributes before metric-resource filtering"),
otelmetric.WithUnit("{info}"),
)
if err != nil {
return fmt.Errorf("failed to create full resource attributes metric: %w", err)
}

resourceGauge.Record(ctx, 1, otelmetric.WithAttributes(resourceAttributesAsKV(c.fullResource)...))
return nil
}

// createConfigMetric creates a configuration info metric with Beholder settings as attributes.
func createConfigMetric(meter otelmetric.Meter, cfg Config) (otelmetric.Int64Gauge, []attribute.KeyValue, error) {
configGauge, err := meter.Int64Gauge(
Expand Down Expand Up @@ -437,6 +421,12 @@ func createConfigMetric(meter otelmetric.Meter, cfg Config) (otelmetric.Int64Gau
"metric_reader_interval", cfg.MetricReaderInterval.String()),
attribute.String(
"metric_compressor", cfg.MetricCompressor),
attribute.Bool(
"reduced_metric_resource_attributes_enabled",
cfg.ReducedMetricResourceAttributesEnabled),
attribute.Bool(
"exclude_volatile_resource_attributes_from_metrics_enabled",
cfg.ExcludeVolatileResourceAttributesFromMetricsEnabled),
}

return configGauge, configAttrs, nil
Expand Down
14 changes: 10 additions & 4 deletions pkg/beholder/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,20 @@ type Config struct {

// OTel Resource
ResourceAttributes []attribute.KeyValue
// ReducedMetricResourceAttributesEnabled uses a slim OTel resource on the metric
// export path while traces and logs keep the full resource.
ReducedMetricResourceAttributesEnabled bool
// ExcludeVolatileResourceAttributesFromMetricsEnabled strips volatile resource
// attributes (process.pid, k8s.pod.*, etc.) from the metric resource only.
ExcludeVolatileResourceAttributesFromMetricsEnabled bool
// Message Emitter
EmitterExportTimeout time.Duration
EmitterExportInterval time.Duration
EmitterExportMaxBatchSize int
EmitterMaxQueueSize int
// EmitterBatchProcessor controls custom-message export mode:
// true = batched async export; false = immediate per-record export.
EmitterBatchProcessor bool
EmitterBatchProcessor bool

// OTel Trace
TraceSampleRatio float64
Expand Down Expand Up @@ -127,7 +133,7 @@ func DefaultConfig() Config {
EmitterExportInterval: 1 * time.Second,
EmitterMaxQueueSize: 2048,
// Keep batched export enabled by default for throughput.
EmitterBatchProcessor: true,
EmitterBatchProcessor: true,
// OTel message log exporter retry config
LogRetryConfig: defaultRetryConfig.Copy(),
// Trace
Expand All @@ -148,8 +154,8 @@ func DefaultConfig() Config {
LogMaxQueueSize: 2048,
LogBatchProcessor: true,
LogStreamingEnabled: true, // Enable logs streaming by default
LogLevel: zapcore.InfoLevel,
LogCompressor: "gzip",
LogLevel: zapcore.InfoLevel,
LogCompressor: "gzip",
// Chip Ingress Batch Emitter
ChipIngressBatchEmitterEnabled: false,
ChipIngressBufferSize: 1000,
Expand Down
18 changes: 13 additions & 5 deletions pkg/beholder/httpclient.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,12 @@ func newCertFromFile(certFile string) (*x509.CertPool, error) {
// NewHTTPClient creates a HTTP based beholder Client. Use NewClient to create a client from a Config which will pick
// the best client type from the Config.
func NewHTTPClient(cfg Config, otlploghttpNew otlploghttpFactory) (*Client, error) {
baseResource, err := newOtelResource(cfg)
resources, err := buildOtelResources(cfg)
if err != nil {
return nil, err
}
fullResource := resources.Full
metricResource := resources.Metric
var tlsConfig *tls.Config
if !cfg.InsecureConnection {
tlsConfig = &tls.Config{
Expand Down Expand Up @@ -106,7 +108,7 @@ func NewHTTPClient(cfg Config, otlploghttpNew otlploghttpFactory) (*Client, erro
}
loggerResource, err := sdkresource.Merge(
sdkresource.NewSchemaless(loggerAttributes...),
baseResource,
fullResource,
)
if err != nil {
return nil, err
Expand All @@ -124,14 +126,14 @@ func NewHTTPClient(cfg Config, otlploghttpNew otlploghttpFactory) (*Client, erro
logger := loggerProvider.Logger(defaultPackageName)

// Tracer
tracerProvider, err := newHTTPTracerProvider(cfg, baseResource, tlsConfig)
tracerProvider, err := newHTTPTracerProvider(cfg, fullResource, tlsConfig)
if err != nil {
return nil, err
}
tracer := tracerProvider.Tracer(defaultPackageName)

// Meter
meterProvider, err := newHTTPMeterProvider(cfg, baseResource, tlsConfig)
meterProvider, err := newHTTPMeterProvider(cfg, metricResource, tlsConfig)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -168,7 +170,7 @@ func NewHTTPClient(cfg Config, otlploghttpNew otlploghttpFactory) (*Client, erro
}
messageLoggerResource, err := sdkresource.Merge(
sdkresource.NewSchemaless(messageAttributes...),
baseResource,
fullResource,
)
if err != nil {
return nil, err
Expand Down Expand Up @@ -203,13 +205,19 @@ func NewHTTPClient(cfg Config, otlploghttpNew otlploghttpFactory) (*Client, erro
MeterProvider: meterProvider,
MessageLoggerProvider: messageLoggerProvider,
lazySigner: nil,
fullResource: fullResource,
OnClose: onClose,
}
lggr := cfg.ChipIngressLogger
if lggr == nil {
lggr = pkglogger.Nop()
}
c.initService(lggr, nil)
if cfg.metricResourceFilteringEnabled() {
if err := c.RecordFullResourceAttributesMetric(context.Background()); err != nil {
return nil, err
}
}
return c, nil
}

Expand Down
Loading
Loading