From f9dfc556d95a5f5999e64f8b48b4c3799a10bcb9 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Sun, 14 Jun 2026 20:22:02 -0400 Subject: [PATCH 01/11] WIP: resource manager --- go.mod | 4 + pkg/resourcemanager/idempotency.go | 102 ++++ pkg/resourcemanager/labels.go | 15 + pkg/resourcemanager/labels_test.go | 30 + pkg/resourcemanager/meterable.go | 119 ++++ pkg/resourcemanager/resourcemanager.go | 430 ++++++++++++++ pkg/resourcemanager/resourcemanager_test.go | 525 ++++++++++++++++++ pkg/types/core/gateway_connector.go | 2 +- .../standard_capabilities_dependencies.go | 24 + 9 files changed, 1250 insertions(+), 1 deletion(-) create mode 100644 pkg/resourcemanager/idempotency.go create mode 100644 pkg/resourcemanager/labels.go create mode 100644 pkg/resourcemanager/labels_test.go create mode 100644 pkg/resourcemanager/meterable.go create mode 100644 pkg/resourcemanager/resourcemanager.go create mode 100644 pkg/resourcemanager/resourcemanager_test.go diff --git a/go.mod b/go.mod index 1299440695..671464ac4b 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 @@ -158,3 +159,6 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) + +// Unpublished module; local replace until chainlink-protos/metering/go is tagged. +replace github.com/smartcontractkit/chainlink-protos/metering/go => ../chainlink-protos/metering/go diff --git a/pkg/resourcemanager/idempotency.go b/pkg/resourcemanager/idempotency.go new file mode 100644 index 0000000000..ceff606499 --- /dev/null +++ b/pkg/resourcemanager/idempotency.go @@ -0,0 +1,102 @@ +package resourcemanager + +import ( + "crypto/sha256" + "encoding/hex" + "strconv" + "strings" + + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" +) + +// IdempotencyKey returns the canonical deduplication key for a MeterRecord: +// the lowercase hex SHA-256 over the full structured identity (including +// node_id), the action, and an event identity, joined with "|": +// +// product|environment|zone|don_id|node_id|service|resource|resource_type|action|resource_id|eventIdentity +// +// where action is the MeterAction enum name (e.g. "METER_ACTION_RESERVE"). +// +// node_id is included, so keys are unique per node: billing dedups a node's +// retries and counts distinct nodes for quorum, while cross-node grouping and +// convergence is the consumer's job on resource_id + dimensions, independent +// of the key. +// +// Keys are level-triggered by design. Producers intentionally reuse the same +// key whenever they re-emit the same resource lifecycle edge — for example a +// node restart re-registering an existing trigger. A repeated key asserts +// "this lifecycle edge happened" rather than "a new event happened". +// +// Consumers therefore use the key for exact-duplicate suppression only. +// Lifecycle state must not be ordered by key arrival: consumers order +// lifecycle state by record timestamp per resource_id, last write wins. +// +// RESERVE and RELEASE are emitted only for genuine allocation/deallocation, +// never for process-lifecycle cleanup of leaked/orphaned resources; a RELEASE +// carries the same value as its paired RESERVE. A reservation lost without a +// RELEASE (e.g. a node crash) is reconciled by its absence from subsequent +// Snapshots, not by a synthetic release. +// +// This helper is the only MeterRecord key generator; producers must not derive +// keys themselves. Inputs are identifiers and must not contain "|". +func IdempotencyKey(id ResourceIdentity, action meteringpb.MeterAction, eventIdentity string) string { + preimage := strings.Join([]string{ + id.Product, + id.Environment, + id.Zone, + id.DONID, + id.NodeID, + id.Service, + id.Resource, + id.ResourceType, + action.String(), + id.ResourceID, + eventIdentity, + }, "|") + sum := sha256.Sum256([]byte(preimage)) + return hex.EncodeToString(sum[:]) +} + +// SnapshotIdempotencyKey returns the canonical deduplication key for one +// MeterSnapshot: the lowercase hex SHA-256 over the literal "snapshot", the +// node-scoped dimensions, the resource and resource_id, and the interval +// bucket, joined with "|": +// +// snapshot|product|environment|zone|don_id|node_id|service|resource|resource_id|interval-bucket +// +// intervalBucket is the snapshot timestamp truncated to the snapshot interval +// (e.g. unix seconds of the interval boundary). It makes each interval's +// snapshot of a resource distinct — so per-interval increments are not +// collapsed — while deduping retries of the same interval. Snapshots are +// action-less, so no action is mixed into the key. +// +// This helper is the only MeterSnapshot key generator; producers must not +// derive keys themselves. Inputs are identifiers and must not contain "|". +func SnapshotIdempotencyKey(id ResourceIdentity, intervalBucket int64) string { + preimage := strings.Join([]string{ + "snapshot", + id.Product, + id.Environment, + id.Zone, + id.DONID, + id.NodeID, + id.Service, + id.Resource, + id.ResourceID, + strconv.FormatInt(intervalBucket, 10), + }, "|") + sum := sha256.Sum256([]byte(preimage)) + return hex.EncodeToString(sum[:]) +} + +// NewUtilization builds a Utilization for EmitMeterRecord with its idempotency +// key derived via IdempotencyKey, so producers never construct keys by hand. +// The resource being metered is identified entirely by id (with its ResourceID +// set, typically via ResourceIdentity.WithResourceID); Utilization carries no +// labels. +func NewUtilization(id ResourceIdentity, action meteringpb.MeterAction, value int64, eventIdentity string) *meteringpb.Utilization { + return &meteringpb.Utilization{ + Value: value, + IdempotencyKey: IdempotencyKey(id, action, eventIdentity), + } +} diff --git a/pkg/resourcemanager/labels.go b/pkg/resourcemanager/labels.go new file mode 100644 index 0000000000..bb8b921cd3 --- /dev/null +++ b/pkg/resourcemanager/labels.go @@ -0,0 +1,15 @@ +package resourcemanager + +import "strings" + +// OwnerLabel returns the canonical form of the "owner" label, used by all +// producers: a leading "0x"/"0X" prefix is stripped and the remainder is +// lowercased. Downstream joins on the owner label depend on every producer +// emitting exactly this form, so producers must not write the owner label +// any other way. +func OwnerLabel(owner string) string { + if strings.HasPrefix(owner, "0x") || strings.HasPrefix(owner, "0X") { + owner = owner[2:] + } + return strings.ToLower(owner) +} diff --git a/pkg/resourcemanager/labels_test.go b/pkg/resourcemanager/labels_test.go new file mode 100644 index 0000000000..5581250493 --- /dev/null +++ b/pkg/resourcemanager/labels_test.go @@ -0,0 +1,30 @@ +package resourcemanager + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOwnerLabel(t *testing.T) { + tests := []struct { + name string + owner string + want string + }{ + {name: "lowercase 0x prefix stripped", owner: "0xabc123def", want: "abc123def"}, + {name: "uppercase 0X prefix stripped", owner: "0Xabc123def", want: "abc123def"}, + {name: "no prefix unchanged", owner: "abc123def", want: "abc123def"}, + {name: "mixed case lowercased", owner: "0xAbC123dEF", want: "abc123def"}, + {name: "mixed case without prefix lowercased", owner: "AbC123dEF", want: "abc123def"}, + {name: "empty string", owner: "", want: ""}, + {name: "prefix only", owner: "0x", want: ""}, + {name: "only one prefix stripped", owner: "0x0Xabc", want: "0xabc"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, OwnerLabel(tt.owner)) + }) + } +} diff --git a/pkg/resourcemanager/meterable.go b/pkg/resourcemanager/meterable.go new file mode 100644 index 0000000000..d09f1f466f --- /dev/null +++ b/pkg/resourcemanager/meterable.go @@ -0,0 +1,119 @@ +package resourcemanager + +import ( + "context" + + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" +) + +// Meterable is implemented by producers that manage durable billable +// resources (trigger registrations, workflow specs, log filters). A producer +// registers itself with a ResourceManager (see ResourceManager.Register) so it +// is polled once per snapshot tick for the absolute state of its currently +// active resources, in addition to emitting lifecycle edges inline via +// EmitMeterRecord. +type Meterable interface { + // ResourceIdentity returns the producer's base identity: the six coarse + // dimensions (product, environment, zone, don_id, node_id, service) plus + // the service-level resource / resource_type. The per-resource resource_id + // is left empty here and populated per active resource by GetUtilization. + ResourceIdentity() ResourceIdentity + + // GetUtilization returns the current level of the producer's currently + // active resources, one SnapshotEntry per resource. The manager emits one + // MeterSnapshot per entry. + // + // It is called on the snapshot tick and MUST be a cheap, non-blocking + // read-snapshot of in-memory state: no network, no disk, no lock held + // across I/O. It must tolerate ctx cancellation (returning promptly, and + // nil/empty is acceptable) and tolerate concurrent registration of new + // resources. An empty or nil return is valid and means nothing is currently + // active: no snapshots are emitted, and billing zeroes the resource out by + // its absence from subsequent snapshots. + GetUtilization(ctx context.Context) []SnapshotEntry +} + +// SnapshotEntry is the current level of one active resource at a snapshot tick. +// Identity is the full per-resource identity (the producer's base dimensions +// with resource / resource_type / resource_id populated for this specific +// resource), and Value is its current level. The resource is identified +// entirely by Identity; there is no separate label metadata. +type SnapshotEntry struct { + Identity ResourceIdentity + Value int64 +} + +// ResourceIdentity is the structured, first-class identity of a durable +// resource. Its nine fields map 1:1 to metering.v1.ResourceIdentity so every +// emitted record carries each dimension as a discrete column rather than a +// parsed dotted string or out-of-band telemetry attribute. +type ResourceIdentity struct { + // Product is the deployment product, e.g. "cre-mainline". A coarse + // billing-rollup dimension. + Product string + + // Environment is the deployment environment, e.g. "production", + // "staging". A coarse billing-rollup dimension. + Environment string + + // Zone is the deployment zone, e.g. "wf-zone-a". A coarse billing-rollup + // dimension. + Zone string + + // DONID is the DON identifier the emitting service belongs to. A coarse + // billing-rollup dimension; used with NodeID to count distinct nodes for + // quorum. + DONID string + + // NodeID is the node identifier (the node's CSA public key). A coarse + // billing-rollup dimension; lets billing dedup a node's retries and count + // distinct nodes. + NodeID string + + // Service is the stable service constant identifying the emitting service, + // e.g. "cron-trigger", "http-trigger", "evm-log-trigger", + // "workflow-syncer-v2". A coarse billing-rollup dimension. It must not + // encode deployment environment or zone. + Service string + + // Resource is the resource pool the record applies to, e.g. + // "trigger_registrations", "log_filters". + Resource string + + // ResourceType is the billing unit used to convert the utilization value + // to universal credits, e.g. "operations", "log_filter_addresses". + ResourceType string + + // ResourceID is the physical/logical resource identity, workflow- + // independent where a shared physical resource exists. For EVM log filters + // it is the content hash of (chain_selector + canonicalized addresses + + // event sigs + positional topics), so identical filters from different + // workflows share one ResourceID. For cron/http/syncer (no shared physical + // resource) it is the workflow-scoped trigger_id / workflow_id, from which + // the workflow (and, downstream, the owner) is recoverable. ResourceIdentity + // is the sole identity of a metered resource; there is no label metadata. + ResourceID string +} + +// WithResourceID returns a copy of r with ResourceID set to id, leaving the +// receiver unchanged. Producers build a base identity once and derive a +// per-resource identity with this helper. +func (r ResourceIdentity) WithResourceID(id string) ResourceIdentity { + r.ResourceID = id + return r +} + +// toProto converts r to its wire form. Field order mirrors the proto. +func (r ResourceIdentity) toProto() *meteringpb.ResourceIdentity { + return &meteringpb.ResourceIdentity{ + Product: r.Product, + Environment: r.Environment, + Zone: r.Zone, + DonId: r.DONID, + NodeId: r.NodeID, + Service: r.Service, + Resource: r.Resource, + ResourceType: r.ResourceType, + ResourceId: r.ResourceID, + } +} diff --git a/pkg/resourcemanager/resourcemanager.go b/pkg/resourcemanager/resourcemanager.go new file mode 100644 index 0000000000..70e6ac0b20 --- /dev/null +++ b/pkg/resourcemanager/resourcemanager.go @@ -0,0 +1,430 @@ +// Package resourcemanager emits metering.v1 events for durable billable +// resources such as trigger registrations, workflow specs, and log filters. +// +// It emits two kinds of records, each covering exactly one resource identified +// by its ResourceIdentity: +// +// - MeterRecord lifecycle edges, inline, via EmitMeterRecord at the point a +// resource is reserved, released, or its utilization changes; and +// - MeterSnapshot records, on a timer, one per resource a registered +// Meterable reports as currently active. Snapshots are the +// liveness/utilization-over-time signal that pure RESERVE/RELEASE cannot +// provide (a node panic would otherwise leak a reservation forever). +// +// The ResourceManager is the single owner of the snapshot tick: each producer +// starts the manager as a sub-service and only Registers itself; producers +// never run their own snapshot loop. +// +// Emission is fail-open by design: EmitMeterRecord and the snapshot loop +// return no error, and a metering failure must never gate, delay, or retry the +// resource operation being metered. Failures surface via error-level logs and +// the resource_manager_*_failure_total counters; billing correctness is +// recovered downstream through idempotency keys and reconciliation. +package resourcemanager + +import ( + "context" + "sync" + "time" + + "github.com/jonboulle/clockwork" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" +) + +// Beholder routing attributes. Each entity value is the contract shared with +// the CHiP schema registration and the consumer topic name; all must match +// exactly. +const ( + beholderDomain = "platform" + beholderEntity = "metering.v1.MeterRecord" + beholderDataSchema = "metering.v1.meter_record" + beholderSnapshotEntity = "metering.v1.MeterSnapshot" + beholderSnapshotDataSchema = "metering.v1.meter_snapshot" +) + +// Counter names are a dashboard contract; do not rename. +// +// Success means the record was accepted for asynchronous export (enqueued +// with the OTel batch processor), not that it was delivered to Kafka; +// delivery failures past this point are invisible to these counters. Emit is +// non-blocking only while the batch processors are enabled, which is the +// default. +const ( + emitSuccessCounterName = "resource_manager_emit_success_total" + emitFailureCounterName = "resource_manager_emit_failure_total" + snapshotEmitSuccessCounterName = "resource_manager_snapshot_emit_success_total" + snapshotEmitFailureCounterName = "resource_manager_snapshot_emit_failure_total" +) + +// utilizationGaugeName is the per-resource utilization gauge. Unlike the +// low-cardinality emit counters, it is labeled with EVERY ResourceIdentity +// dimension (including node_id and resource_id) so dashboards can break +// utilization down by any dimension. That is high cardinality by construction; +// it is the intended, requested behavior. +const utilizationGaugeName = "resource_manager_utilization" + +// DefaultSnapshotInterval is the recommended snapshot period. It is NOT +// applied implicitly: a zero ResourceManagerConfig.SnapshotInterval disables +// snapshots. Callers that want snapshots pass this (or their own value) +// explicitly. +const DefaultSnapshotInterval = 60 * time.Second + +// Emitter delivers an encoded metering message with its routing attributes. +// beholder.Emitter satisfies it, so production wiring is +// beholder.GetEmitter(); tests substitute a fake. +type Emitter interface { + Emit(ctx context.Context, body []byte, attrKVs ...any) error +} + +// ResourceManagerConfig configures a ResourceManager. +type ResourceManagerConfig struct { + // Enabled is the metering rollout gate. When false (the default), the + // ResourceManager is a no-op: EmitMeterRecord neither marshals nor emits + // nor records utilization, and the snapshot loop does not start. + Enabled bool + + // Emitter delivers encoded records, typically beholder.GetEmitter(). A nil + // Emitter makes EmitMeterRecord a no-op even when Enabled is true and keeps + // the snapshot loop from starting. + Emitter Emitter + + // SnapshotInterval is the period between snapshots. Zero (the default) + // DISABLES the snapshot loop; the manager still starts as a service and + // EmitMeterRecord still works. Callers that want snapshots set a positive + // value, e.g. DefaultSnapshotInterval. The default is not substituted for + // zero — zero means off. + SnapshotInterval time.Duration + + // Clock drives snapshot tick timing and record timestamps. Nil selects the + // real clock. + Clock clockwork.Clock +} + +// registration is one registered Meterable. It is a pointer-identified handle +// so Register can return an idempotent unregister closure. +type registration struct { + m Meterable +} + +// ResourceManager emits MeterRecords and periodic MeterSnapshots for durable +// resources. It is a services.Service: callers start it (typically as a +// sub-service of the producer) and Register Meterables to be snapshotted. It +// is safe for concurrent use. +type ResourceManager struct { + services.Service + srvcEng *services.Engine + + lggr logger.SugaredLogger + enabled bool + emitter Emitter + snapshotInterval time.Duration + clock clockwork.Clock + + mu sync.RWMutex + registrations map[*registration]struct{} + + emitSuccess metric.Int64Counter + emitFailure metric.Int64Counter + snapshotEmitSuccess metric.Int64Counter + snapshotEmitFailure metric.Int64Counter + utilization metric.Int64Gauge +} + +// NewResourceManager returns a ResourceManager. A failure to create a metric +// instrument is logged and that instrument is skipped; it never prevents +// construction. The manager must be Started before its snapshot loop runs; +// EmitMeterRecord works regardless of Start. +func NewResourceManager(lggr logger.Logger, cfg ResourceManagerConfig) *ResourceManager { + meter := beholder.GetMeter() + sugared := logger.Sugared(lggr) + newCounter := func(name string) metric.Int64Counter { + c, err := meter.Int64Counter(name) + if err != nil { + sugared.Errorw("failed to create metering counter", "counter", name, "err", err) + return nil + } + return c + } + gauge, err := meter.Int64Gauge(utilizationGaugeName) + if err != nil { + sugared.Errorw("failed to create metering gauge", "gauge", utilizationGaugeName, "err", err) + gauge = nil + } + + clock := cfg.Clock + if clock == nil { + clock = clockwork.NewRealClock() + } + + rm := &ResourceManager{ + enabled: cfg.Enabled, + emitter: cfg.Emitter, + snapshotInterval: cfg.SnapshotInterval, + clock: clock, + registrations: make(map[*registration]struct{}), + emitSuccess: newCounter(emitSuccessCounterName), + emitFailure: newCounter(emitFailureCounterName), + snapshotEmitSuccess: newCounter(snapshotEmitSuccessCounterName), + snapshotEmitFailure: newCounter(snapshotEmitFailureCounterName), + utilization: gauge, + } + rm.Service, rm.srvcEng = services.Config{ + Name: "ResourceManager", + Start: rm.start, + Close: rm.close, + }.NewServiceEngine(lggr) + rm.lggr = logger.Sugared(rm.srvcEng) + return rm +} + +// start launches the snapshot loop. The loop runs only when metering is +// enabled, an emitter is configured, and a positive SnapshotInterval is set; +// otherwise the service starts cleanly with snapshots disabled and +// EmitMeterRecord remains available. +func (rm *ResourceManager) start(_ context.Context) error { + if !rm.enabled || rm.emitter == nil || rm.snapshotInterval <= 0 { + rm.lggr.Infow("snapshot loop disabled", + "enabled", rm.enabled, + "hasEmitter", rm.emitter != nil, + "snapshotInterval", rm.snapshotInterval, + ) + return nil + } + rm.srvcEng.Go(func(ctx context.Context) { + ticker := rm.clock.NewTicker(rm.snapshotInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.Chan(): + rm.emitSnapshots(ctx) + } + } + }) + return nil +} + +func (rm *ResourceManager) close() error { return nil } + +// Register adds m to the snapshot registry and returns an idempotent function +// that removes it. Calling the returned function more than once is a no-op. +// The returned closure is safe for concurrent use. +func (rm *ResourceManager) Register(m Meterable) (unregister func()) { + reg := ®istration{m: m} + + rm.mu.Lock() + rm.registrations[reg] = struct{}{} + rm.mu.Unlock() + + var once sync.Once + return func() { + once.Do(func() { + rm.mu.Lock() + delete(rm.registrations, reg) + rm.mu.Unlock() + }) + } +} + +// emitSnapshots is the snapshot tick: it snapshots the registry under a read +// lock, then emits snapshots for every registered Meterable. +func (rm *ResourceManager) emitSnapshots(ctx context.Context) { + rm.mu.RLock() + ms := make([]Meterable, 0, len(rm.registrations)) + for reg := range rm.registrations { + ms = append(ms, reg.m) + } + rm.mu.RUnlock() + + for _, m := range ms { + rm.emitSnapshot(ctx, m) + } +} + +// emitSnapshot emits one MeterSnapshot per active resource reported by m. Each +// snapshot covers exactly one resource, fully identified by its +// ResourceIdentity. The interval bucket (timestamp truncated to the snapshot +// interval) keys the snapshot per interval. An empty utilization list emits +// nothing: billing zeroes a resource out by its absence from later snapshots. +// Fail-open: per-resource errors are logged and counted, never returned. +func (rm *ResourceManager) emitSnapshot(ctx context.Context, m Meterable) { + if !rm.enabled || rm.emitter == nil { + return + } + + now := rm.clock.Now() + bucket := now.Truncate(rm.snapshotInterval).Unix() + ts := timestamppb.New(now) + interval := durationpb.New(rm.snapshotInterval) + + for _, e := range m.GetUtilization(ctx) { + rm.recordUtilization(ctx, e.Identity, e.Value) + + snapshot := &meteringpb.MeterSnapshot{ + Timestamp: ts, + Identity: e.Identity.toProto(), + Utilization: &meteringpb.Utilization{ + Value: e.Value, + IdempotencyKey: SnapshotIdempotencyKey(e.Identity, bucket), + }, + Interval: interval, + } + + body, err := proto.Marshal(snapshot) + if err != nil { + rm.lggr.Errorw("failed to marshal snapshot", + "service", e.Identity.Service, + "resource", e.Identity.Resource, + "resourceID", e.Identity.ResourceID, + "err", err, + ) + rm.countSnapshot(ctx, rm.snapshotEmitFailure, e.Identity, attribute.String("reason", "marshal")) + continue + } + + if err := rm.emitter.Emit(ctx, body, + beholder.AttrKeyDataSchema, beholderSnapshotDataSchema, + beholder.AttrKeyDomain, beholderDomain, + beholder.AttrKeyEntity, beholderSnapshotEntity, + ); err != nil { + rm.lggr.Errorw("failed to emit snapshot", + "service", e.Identity.Service, + "resource", e.Identity.Resource, + "resourceID", e.Identity.ResourceID, + "err", err, + ) + rm.countSnapshot(ctx, rm.snapshotEmitFailure, e.Identity, attribute.String("reason", "emit")) + continue + } + + rm.countSnapshot(ctx, rm.snapshotEmitSuccess, e.Identity) + } +} + +// EmitMeterRecord emits a metering.v1.MeterRecord, timestamped now, for action +// on the one resource described by identity. +// +// EmitMeterRecord is fail-open and returns no error: when the manager is +// disabled or has no emitter it does nothing, and marshal or emit failures are +// recorded only via error-level logs and the failure counter. Callers must +// never gate resource allocation or deallocation on emission. +func (rm *ResourceManager) EmitMeterRecord(ctx context.Context, identity ResourceIdentity, action meteringpb.MeterAction, utilization *meteringpb.Utilization) { + if !rm.enabled { + rm.lggr.Debugw("metering disabled; meter record not emitted", + "service", identity.Service, + "resource", identity.Resource, + "action", action.String(), + ) + return + } + + // Reflect the lifecycle edge in the utilization gauge: a RELEASE drops the + // resource to zero, every other action records its current level. Recorded + // before the emitter check so the gauge tracks state even if export is off. + gaugeValue := utilization.GetValue() + if action == meteringpb.MeterAction_METER_ACTION_RELEASE { + gaugeValue = 0 + } + rm.recordUtilization(ctx, identity, gaugeValue) + + if rm.emitter == nil { + return + } + + record := &meteringpb.MeterRecord{ + Timestamp: timestamppb.New(rm.clock.Now()), + Identity: identity.toProto(), + Action: action, + Utilization: utilization, + } + + body, err := proto.Marshal(record) + if err != nil { + rm.lggr.Errorw("failed to marshal meter record", + "service", identity.Service, + "resource", identity.Resource, + "action", action.String(), + "err", err, + ) + rm.countRecord(ctx, rm.emitFailure, identity, action, attribute.String("reason", "marshal")) + return + } + + if err := rm.emitter.Emit(ctx, body, + beholder.AttrKeyDataSchema, beholderDataSchema, + beholder.AttrKeyDomain, beholderDomain, + beholder.AttrKeyEntity, beholderEntity, + ); err != nil { + rm.lggr.Errorw("failed to emit meter record", + "service", identity.Service, + "resource", identity.Resource, + "action", action.String(), + "err", err, + ) + rm.countRecord(ctx, rm.emitFailure, identity, action, attribute.String("reason", "emit")) + return + } + + rm.countRecord(ctx, rm.emitSuccess, identity, action) +} + +// recordUtilization records value to the per-resource utilization gauge, +// labeled with every ResourceIdentity dimension. This is intentionally high +// cardinality (it includes node_id and resource_id) so utilization can be +// sliced by any dimension downstream. +func (rm *ResourceManager) recordUtilization(ctx context.Context, id ResourceIdentity, value int64) { + if rm.utilization == nil { + return + } + rm.utilization.Record(ctx, value, metric.WithAttributes( + attribute.String("product", id.Product), + attribute.String("environment", id.Environment), + attribute.String("zone", id.Zone), + attribute.String("don_id", id.DONID), + attribute.String("node_id", id.NodeID), + attribute.String("service", id.Service), + attribute.String("resource", id.Resource), + attribute.String("resource_type", id.ResourceType), + attribute.String("resource_id", id.ResourceID), + )) +} + +// countRecord records a MeterRecord emit outcome. Labels are intentionally +// low-cardinality: service, resource, action (+ reason on failure). node_id +// and resource_id are deliberately excluded here to keep the time series +// bounded; the utilization gauge carries the full identity instead. +func (rm *ResourceManager) countRecord(ctx context.Context, c metric.Int64Counter, identity ResourceIdentity, action meteringpb.MeterAction, extra ...attribute.KeyValue) { + if c == nil { + return + } + attrs := append([]attribute.KeyValue{ + attribute.String("service", identity.Service), + attribute.String("resource", identity.Resource), + attribute.String("action", action.String()), + }, extra...) + c.Add(ctx, 1, metric.WithAttributes(attrs...)) +} + +// countSnapshot records a MeterSnapshot emit outcome. Labels are intentionally +// low-cardinality: service, resource (+ reason on failure). +func (rm *ResourceManager) countSnapshot(ctx context.Context, c metric.Int64Counter, identity ResourceIdentity, extra ...attribute.KeyValue) { + if c == nil { + return + } + attrs := append([]attribute.KeyValue{ + attribute.String("service", identity.Service), + attribute.String("resource", identity.Resource), + }, extra...) + c.Add(ctx, 1, metric.WithAttributes(attrs...)) +} diff --git a/pkg/resourcemanager/resourcemanager_test.go b/pkg/resourcemanager/resourcemanager_test.go new file mode 100644 index 0000000000..4f7d0f2d7b --- /dev/null +++ b/pkg/resourcemanager/resourcemanager_test.go @@ -0,0 +1,525 @@ +package resourcemanager + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/beholder/beholdertest" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" +) + +var testIdentity = ResourceIdentity{ + Product: "cre-mainline", + Environment: "production", + Zone: "wf-zone-a", + DONID: "don-1", + NodeID: "node-1", + Service: "cron-trigger", + Resource: "trigger_registrations", + ResourceType: "operations", +} + +type emitCall struct { + body []byte + attrKVs []any +} + +type fakeEmitter struct { + mu sync.Mutex + err error + calls []emitCall +} + +func (f *fakeEmitter) Emit(_ context.Context, body []byte, attrKVs ...any) error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, emitCall{body: body, attrKVs: attrKVs}) + return f.err +} + +func (f *fakeEmitter) snapshot() []emitCall { + f.mu.Lock() + defer f.mu.Unlock() + return append([]emitCall(nil), f.calls...) +} + +// fakeMeterable is a test Meterable whose active entries can be swapped under +// lock to exercise registration/unregistration and changing utilization. +type fakeMeterable struct { + identity ResourceIdentity + + mu sync.Mutex + entries []SnapshotEntry +} + +func (f *fakeMeterable) ResourceIdentity() ResourceIdentity { return f.identity } + +func (f *fakeMeterable) GetUtilization(_ context.Context) []SnapshotEntry { + f.mu.Lock() + defer f.mu.Unlock() + return append([]SnapshotEntry(nil), f.entries...) +} + +func (f *fakeMeterable) set(entries []SnapshotEntry) { + f.mu.Lock() + defer f.mu.Unlock() + f.entries = entries +} + +// attrMap converts beholder-style attribute key/value pairs into a map. +func attrMap(t *testing.T, attrKVs []any) map[string]any { + t.Helper() + require.Zero(t, len(attrKVs)%2, "attrKVs must be key/value pairs") + m := make(map[string]any, len(attrKVs)/2) + for i := 0; i < len(attrKVs); i += 2 { + key, ok := attrKVs[i].(string) + require.True(t, ok, "attr key must be a string") + m[key] = attrKVs[i+1] + } + return m +} + +// waitForSnapshotCount polls until the emitter has recorded want snapshot emissions. +func waitForSnapshotCount(t *testing.T, emitter *fakeEmitter, want int) { + t.Helper() + require.Eventually(t, func() bool { + return len(decodeSnapshots(t, emitter.snapshot())) == want + }, time.Second, time.Millisecond) +} + +func TestEmitMeterRecord_Gating(t *testing.T) { + tests := []struct { + name string + enabled bool + emitter *fakeEmitter + wantEmits int + }{ + {name: "disabled does not emit", enabled: false, emitter: &fakeEmitter{}, wantEmits: 0}, + {name: "disabled with nil emitter does not panic", enabled: false, emitter: nil, wantEmits: 0}, + {name: "enabled with nil emitter does not panic", enabled: true, emitter: nil, wantEmits: 0}, + {name: "enabled emits", enabled: true, emitter: &fakeEmitter{}, wantEmits: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := ResourceManagerConfig{Enabled: tt.enabled} + if tt.emitter != nil { + cfg.Emitter = tt.emitter + } + rm := NewResourceManager(logger.Test(t), cfg) + + rm.EmitMeterRecord(t.Context(), testIdentity, meteringpb.MeterAction_METER_ACTION_RESERVE, + NewUtilization(testIdentity.WithResourceID("trigger-1"), meteringpb.MeterAction_METER_ACTION_RESERVE, 1, "event-1")) + + if tt.emitter != nil { + assert.Len(t, tt.emitter.calls, tt.wantEmits) + } + }) + } +} + +func TestEmitMeterRecord_Success(t *testing.T) { + emitter := &fakeEmitter{} + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{Enabled: true, Emitter: emitter}) + + id := testIdentity.WithResourceID("trigger-1") + utilization := NewUtilization(id, meteringpb.MeterAction_METER_ACTION_RESERVE, 1, "event-1") + before := time.Now() + rm.EmitMeterRecord(t.Context(), id, meteringpb.MeterAction_METER_ACTION_RESERVE, utilization) + after := time.Now() + + require.Len(t, emitter.calls, 1) + call := emitter.calls[0] + + attrs := attrMap(t, call.attrKVs) + assert.Equal(t, "metering.v1.meter_record", attrs[beholder.AttrKeyDataSchema]) + assert.Equal(t, "platform", attrs[beholder.AttrKeyDomain]) + assert.Equal(t, "metering.v1.MeterRecord", attrs[beholder.AttrKeyEntity]) + + var record meteringpb.MeterRecord + require.NoError(t, proto.Unmarshal(call.body, &record)) + require.NotNil(t, record.GetIdentity()) + assert.Equal(t, id.Product, record.GetIdentity().GetProduct()) + assert.Equal(t, id.NodeID, record.GetIdentity().GetNodeId()) + assert.Equal(t, id.Service, record.GetIdentity().GetService()) + assert.Equal(t, id.Resource, record.GetIdentity().GetResource()) + assert.Equal(t, id.ResourceType, record.GetIdentity().GetResourceType()) + assert.Equal(t, "trigger-1", record.GetIdentity().GetResourceId()) + assert.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, record.GetAction()) + + require.NotNil(t, record.GetUtilization()) + assert.Equal(t, int64(1), record.GetUtilization().GetValue()) + assert.Equal(t, + IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1"), + record.GetUtilization().GetIdempotencyKey()) + + require.NotNil(t, record.GetTimestamp()) + ts := record.GetTimestamp().AsTime() + assert.False(t, ts.Before(before.Add(-time.Second)), "timestamp too early: %s", ts) + assert.False(t, ts.After(after.Add(time.Second)), "timestamp too late: %s", ts) +} + +func TestEmitMeterRecord_EmitFailureIsSwallowed(t *testing.T) { + emitter := &fakeEmitter{err: errors.New("collector unavailable")} + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{Enabled: true, Emitter: emitter}) + + id := testIdentity.WithResourceID("trigger-1") + require.NotPanics(t, func() { + rm.EmitMeterRecord(t.Context(), id, meteringpb.MeterAction_METER_ACTION_RELEASE, + NewUtilization(id, meteringpb.MeterAction_METER_ACTION_RELEASE, 1, "event-2")) + }) + require.Len(t, emitter.calls, 1) + + // A subsequent emission still goes through; the failure left no state behind. + rm.EmitMeterRecord(t.Context(), id, meteringpb.MeterAction_METER_ACTION_RELEASE, + NewUtilization(id, meteringpb.MeterAction_METER_ACTION_RELEASE, 1, "event-3")) + require.Len(t, emitter.calls, 2) +} + +// TestEmitMeterRecord_BeholderObserver wires the manager to the global +// beholder emitter the way production does. beholdertest.NewObserver is not +// parallel-safe, so this test must not run in parallel. +func TestEmitMeterRecord_BeholderObserver(t *testing.T) { + observer := beholdertest.NewObserver(t) + + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ + Enabled: true, + Emitter: beholder.GetEmitter(), + }) + + id := testIdentity.WithResourceID("trigger-1") + rm.EmitMeterRecord(t.Context(), id, meteringpb.MeterAction_METER_ACTION_RESERVE, + NewUtilization(id, meteringpb.MeterAction_METER_ACTION_RESERVE, 1, "event-1")) + + msgs := observer.Messages(t, beholder.AttrKeyEntity, "metering.v1.MeterRecord") + require.Len(t, msgs, 1) + assert.Equal(t, "metering.v1.meter_record", msgs[0].Attrs[beholder.AttrKeyDataSchema]) + assert.Equal(t, "platform", msgs[0].Attrs[beholder.AttrKeyDomain]) + + var record meteringpb.MeterRecord + require.NoError(t, proto.Unmarshal(msgs[0].Body, &record)) + assert.Equal(t, id.Service, record.GetIdentity().GetService()) +} + +// TestResourceManager_StartClose asserts the manager starts and closes cleanly +// both with snapshots enabled and with the loop disabled (zero interval). +func TestResourceManager_StartClose(t *testing.T) { + t.Run("snapshot loop enabled", func(t *testing.T) { + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ + Enabled: true, + Emitter: &fakeEmitter{}, + SnapshotInterval: time.Hour, + }) + require.NoError(t, rm.Start(t.Context())) + require.NoError(t, rm.Close()) + }) + + t.Run("snapshot loop disabled by zero interval", func(t *testing.T) { + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ + Enabled: true, + Emitter: &fakeEmitter{}, + // SnapshotInterval unset -> disabled. + }) + require.NoError(t, rm.Start(t.Context())) + require.NoError(t, rm.Close()) + }) +} + +// decodeSnapshots extracts every MeterSnapshot from the emitter's recorded +// calls. +func decodeSnapshots(t *testing.T, calls []emitCall) []*meteringpb.MeterSnapshot { + t.Helper() + var snaps []*meteringpb.MeterSnapshot + for _, c := range calls { + attrs := attrMap(t, c.attrKVs) + if attrs[beholder.AttrKeyEntity] != "metering.v1.MeterSnapshot" { + continue + } + assert.Equal(t, "metering.v1.meter_snapshot", attrs[beholder.AttrKeyDataSchema]) + assert.Equal(t, "platform", attrs[beholder.AttrKeyDomain]) + var s meteringpb.MeterSnapshot + require.NoError(t, proto.Unmarshal(c.body, &s)) + snaps = append(snaps, &s) + } + return snaps +} + +// snapshotKey recomputes the expected idempotency key for a decoded snapshot +// from its own timestamp and interval, so the assertion does not depend on the +// wall clock at emit time. +func snapshotKey(s *meteringpb.MeterSnapshot, id ResourceIdentity) string { + bucket := s.GetTimestamp().AsTime().Truncate(s.GetInterval().AsDuration()).Unix() + return SnapshotIdempotencyKey(id, bucket) +} + +func TestEmitSnapshot_OnePerResource(t *testing.T) { + emitter := &fakeEmitter{} + clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + SnapshotInterval: 30 * time.Second, + Clock: clock, + }) + + m := &fakeMeterable{identity: testIdentity} + id1 := testIdentity.WithResourceID("trigger-1") + id2 := testIdentity.WithResourceID("trigger-2") + m.set([]SnapshotEntry{ + {Identity: id1, Value: 3}, + {Identity: id2, Value: 5}, + }) + unregister := rm.Register(m) + defer unregister() + + servicetest.Run(t, rm) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(30 * time.Second) + waitForSnapshotCount(t, emitter, 2) + + // Exactly one MeterSnapshot per active resource — never a batch. + snaps := decodeSnapshots(t, emitter.snapshot()) + require.Len(t, snaps, 2) + + byID := map[string]*meteringpb.MeterSnapshot{} + for _, s := range snaps { + // Each snapshot carries the full per-resource identity. + require.NotNil(t, s.GetIdentity()) + assert.Equal(t, testIdentity.Service, s.GetIdentity().GetService()) + require.NotNil(t, s.GetInterval()) + assert.Equal(t, 30*time.Second, s.GetInterval().AsDuration()) + require.NotNil(t, s.GetTimestamp()) + byID[s.GetIdentity().GetResourceId()] = s + } + + s1 := byID["trigger-1"] + require.NotNil(t, s1) + assert.Equal(t, int64(3), s1.GetUtilization().GetValue()) + assert.Equal(t, snapshotKey(s1, id1), s1.GetUtilization().GetIdempotencyKey()) + + s2 := byID["trigger-2"] + require.NotNil(t, s2) + assert.Equal(t, int64(5), s2.GetUtilization().GetValue()) + assert.Equal(t, snapshotKey(s2, id2), s2.GetUtilization().GetIdempotencyKey()) +} + +func TestEmitSnapshot_EmptyListEmitsNothing(t *testing.T) { + emitter := &fakeEmitter{} + clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, + }) + + m := &fakeMeterable{identity: testIdentity} // no active entries + unregister := rm.Register(m) + defer unregister() + + servicetest.Run(t, rm) + + // Nothing active -> no snapshots. Billing zeroes the resource out by its + // absence from subsequent snapshots, not by an explicit empty record. + assert.Empty(t, decodeSnapshots(t, emitter.snapshot())) +} + +func TestEmitSnapshots_KeyStableWithinInterval(t *testing.T) { + emitter := &fakeEmitter{} + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + SnapshotInterval: time.Hour, // wide bucket so all ticks share one interval + }) + + m := &fakeMeterable{identity: testIdentity} + m.set([]SnapshotEntry{{Identity: testIdentity.WithResourceID("trigger-1"), Value: 1}}) + unregister := rm.Register(m) + defer unregister() + + // Drive three ticks directly (no timer) to keep the test deterministic. + rm.emitSnapshots(t.Context()) + rm.emitSnapshots(t.Context()) + rm.emitSnapshots(t.Context()) + + snaps := decodeSnapshots(t, emitter.snapshot()) + require.Len(t, snaps, 3, "one MeterSnapshot per resource per tick") + + // Same resource, same interval bucket -> identical keys (retries dedup); + // successive intervals would differ. + k0 := snaps[0].GetUtilization().GetIdempotencyKey() + assert.Equal(t, k0, snaps[1].GetUtilization().GetIdempotencyKey()) + assert.Equal(t, k0, snaps[2].GetUtilization().GetIdempotencyKey()) +} + +func TestRegister_UnregisterStopsSnapshots(t *testing.T) { + emitter := &fakeEmitter{} + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + }) + + m := &fakeMeterable{identity: testIdentity} + m.set([]SnapshotEntry{{Identity: testIdentity.WithResourceID("trigger-1"), Value: 1}}) + unregister := rm.Register(m) + + rm.emitSnapshots(t.Context()) + require.Len(t, decodeSnapshots(t, emitter.snapshot()), 1) + + unregister() + // Idempotent: a second call is a no-op. + require.NotPanics(t, unregister) + + rm.emitSnapshots(t.Context()) + require.Len(t, decodeSnapshots(t, emitter.snapshot()), 1, "unregistered Meterable must not be snapshotted") +} + +func TestEmitSnapshot_FailureIsSwallowed(t *testing.T) { + emitter := &fakeEmitter{err: errors.New("collector unavailable")} + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ + Enabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + }) + + m := &fakeMeterable{identity: testIdentity} + m.set([]SnapshotEntry{{Identity: testIdentity.WithResourceID("trigger-1"), Value: 1}}) + unregister := rm.Register(m) + defer unregister() + + require.NotPanics(t, func() { rm.emitSnapshots(t.Context()) }) + require.Len(t, emitter.snapshot(), 1) +} + +// TestEmitSnapshot_BeholderObserver wires snapshots through the global beholder +// emitter. beholdertest.NewObserver is not parallel-safe; do not run this in +// parallel. +func TestEmitSnapshot_BeholderObserver(t *testing.T) { + observer := beholdertest.NewObserver(t) + clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ + Enabled: true, + Emitter: beholder.GetEmitter(), + SnapshotInterval: time.Minute, + Clock: clock, + }) + + m := &fakeMeterable{identity: testIdentity} + m.set([]SnapshotEntry{{Identity: testIdentity.WithResourceID("trigger-1"), Value: 7}}) + unregister := rm.Register(m) + defer unregister() + + servicetest.Run(t, rm) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + require.Eventually(t, func() bool { + return len(observer.Messages(t, beholder.AttrKeyEntity, "metering.v1.MeterSnapshot")) == 1 + }, time.Second, time.Millisecond) + + msgs := observer.Messages(t, beholder.AttrKeyEntity, "metering.v1.MeterSnapshot") + require.Len(t, msgs, 1) + assert.Equal(t, "metering.v1.meter_snapshot", msgs[0].Attrs[beholder.AttrKeyDataSchema]) + + var s meteringpb.MeterSnapshot + require.NoError(t, proto.Unmarshal(msgs[0].Body, &s)) + assert.Equal(t, "trigger-1", s.GetIdentity().GetResourceId()) + assert.Equal(t, int64(7), s.GetUtilization().GetValue()) +} + +func TestIdempotencyKey(t *testing.T) { + id := testIdentity.WithResourceID("trigger-1") + base := IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1") + + t.Run("deterministic", func(t *testing.T) { + assert.Equal(t, base, IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1")) + }) + + t.Run("format is sha256 hex", func(t *testing.T) { + assert.Regexp(t, "^[0-9a-f]{64}$", base) + }) + + otherNode := id + otherNode.NodeID = "node-2" + otherResource := id + otherResource.Resource = "log_filters" + + distinct := []struct { + name string + key string + }{ + {"action", IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RELEASE, "event-1")}, + {"node_id", IdempotencyKey(otherNode, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1")}, + {"resource", IdempotencyKey(otherResource, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1")}, + {"resource ID", IdempotencyKey(id.WithResourceID("trigger-2"), meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1")}, + {"event identity", IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-2")}, + } + for _, tt := range distinct { + t.Run("distinct across "+tt.name, func(t *testing.T) { + assert.NotEqual(t, base, tt.key) + }) + } +} + +func TestSnapshotIdempotencyKey(t *testing.T) { + id := testIdentity.WithResourceID("trigger-1") + base := SnapshotIdempotencyKey(id, 100) + + t.Run("format is sha256 hex", func(t *testing.T) { + assert.Regexp(t, "^[0-9a-f]{64}$", base) + }) + + t.Run("stable within an interval bucket", func(t *testing.T) { + assert.Equal(t, base, SnapshotIdempotencyKey(id, 100)) + }) + + t.Run("differs per interval bucket", func(t *testing.T) { + assert.NotEqual(t, base, SnapshotIdempotencyKey(id, 101)) + }) + + t.Run("differs per resource_id", func(t *testing.T) { + assert.NotEqual(t, base, SnapshotIdempotencyKey(id.WithResourceID("trigger-2"), 100)) + }) + + t.Run("differs per node_id", func(t *testing.T) { + other := id + other.NodeID = "node-2" + assert.NotEqual(t, base, SnapshotIdempotencyKey(other, 100)) + }) + + t.Run("differs from a MeterRecord key", func(t *testing.T) { + assert.NotEqual(t, base, IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RESERVE, "1")) + }) +} + +func TestNewUtilization(t *testing.T) { + id := testIdentity.WithResourceID("filter-1") + u := NewUtilization(id, meteringpb.MeterAction_METER_ACTION_UPDATE, 42, "event-9") + + assert.Equal(t, int64(42), u.GetValue()) + assert.Equal(t, + IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_UPDATE, "event-9"), + u.GetIdempotencyKey()) +} + +func TestResourceIdentity_WithResourceID(t *testing.T) { + got := testIdentity.WithResourceID("rid-1") + assert.Equal(t, "rid-1", got.ResourceID) + assert.Empty(t, testIdentity.ResourceID, "receiver must be unchanged") + // All other fields are copied through. + assert.Equal(t, testIdentity.Service, got.Service) + assert.Equal(t, testIdentity.NodeID, got.NodeID) +} diff --git a/pkg/types/core/gateway_connector.go b/pkg/types/core/gateway_connector.go index b402ad5083..6111435bc3 100644 --- a/pkg/types/core/gateway_connector.go +++ b/pkg/types/core/gateway_connector.go @@ -48,7 +48,7 @@ type GatewayConnectorHandler interface { } var ( - _ GatewayConnector = (*UnimplementedGatewayConnector)(nil) + _ GatewayConnector = (*UnimplementedGatewayConnector)(nil) _ MultiGatewayConnector = (*UnimplementedGatewayConnector)(nil) ) diff --git a/pkg/types/core/standard_capabilities_dependencies.go b/pkg/types/core/standard_capabilities_dependencies.go index 25add8e6f9..339205d6e8 100644 --- a/pkg/types/core/standard_capabilities_dependencies.go +++ b/pkg/types/core/standard_capabilities_dependencies.go @@ -34,4 +34,28 @@ type StandardCapabilitiesDependencies struct { // registry in that case, but the fallback path cannot disambiguate when // the local node belongs to multiple DONs running the same capability. CapabilityDonID uint32 + + // Product is the host-injected deployment product (e.g. "cre-mainline"), + // sourced once from node config by the host and provided before Initialise + // is called. Plugins use it as a coarse metering/billing identity + // dimension. An empty value means the host did not provide one (a legacy + // node or a boot path not yet updated to populate it). + Product string + + // Environment is the host-injected deployment environment (e.g. + // "production", "staging"), provided by the host before Initialise. Plugins + // use it as a coarse metering/billing identity dimension. Empty means the + // host did not provide one. + Environment string + + // Zone is the host-injected deployment zone (e.g. "wf-zone-a"), provided by + // the host before Initialise. Plugins use it as a coarse metering/billing + // identity dimension. Empty means the host did not provide one. + Zone string + + // NodeID is the host-injected node identity: the node's CSA public key + // (hex), provided by the host before Initialise. Plugins use it as a coarse + // metering/billing identity dimension to dedup a node's retries and count + // distinct nodes for quorum. Empty means the host did not provide one. + NodeID string } From c93f25106f68855e8725792672527522af5883d4 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Mon, 15 Jun 2026 14:44:30 -0400 Subject: [PATCH 02/11] bumping protos --- go.mod | 5 +---- go.sum | 2 ++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 671464ac4b..1e7d3c481e 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-00010101000000-000000000000 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260615161920-ed05faf7f0a2 github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 @@ -159,6 +159,3 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) - -// Unpublished module; local replace until chainlink-protos/metering/go is tagged. -replace github.com/smartcontractkit/chainlink-protos/metering/go => ../chainlink-protos/metering/go diff --git a/go.sum b/go.sum index 3441393f7f..cd68d5e8c4 100644 --- a/go.sum +++ b/go.sum @@ -266,6 +266,8 @@ github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260615161920-ed05faf7f0a2 h1:zKfst+Wyr95rbxLNSsc4nKybe9tLXOC2r2YbDRQwNxM= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260615161920-ed05faf7f0a2/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 h1:B7itmjy+CMJ26elVw/cAJqqhBQ3Xa/mBYWK0/rQ5MuI= From fff43abfa624b12cc13503af162d83d49cf562b6 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Fri, 26 Jun 2026 18:28:16 -0400 Subject: [PATCH 03/11] adapting resource managers after proto change --- go.mod | 2 +- go.sum | 4 +- pkg/loop/config.go | 15 + pkg/loop/config_test.go | 6 + pkg/resourcemanager/idempotency.go | 119 ++--- pkg/resourcemanager/meterable.go | 71 ++- pkg/resourcemanager/resourcemanager.go | 149 +++--- pkg/resourcemanager/resourcemanager_test.go | 473 ++++++-------------- 8 files changed, 308 insertions(+), 531 deletions(-) diff --git a/go.mod b/go.mod index 1e7d3c481e..629f9c302c 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260615161920-ed05faf7f0a2 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260626162307-b56f9af1d196 github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 diff --git a/go.sum b/go.sum index cd68d5e8c4..bd2c8328ef 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,8 @@ github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260615161920-ed05faf7f0a2 h1:zKfst+Wyr95rbxLNSsc4nKybe9tLXOC2r2YbDRQwNxM= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260615161920-ed05faf7f0a2/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260626162307-b56f9af1d196 h1:5SZtASpkbCyK7xtxqiUNwLxYEt7rXE6mkjZKecve2yY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260626162307-b56f9af1d196/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 h1:B7itmjy+CMJ26elVw/cAJqqhBQ3Xa/mBYWK0/rQ5MuI= diff --git a/pkg/loop/config.go b/pkg/loop/config.go index 396c7d979d..a0bf626dd2 100644 --- a/pkg/loop/config.go +++ b/pkg/loop/config.go @@ -85,6 +85,8 @@ const ( envTelemetryPrometheusBridgeEnabled = "CL_TELEMETRY_PROMETHEUS_BRIDGE_ENABLED" envTelemetryPrometheusBridgePrefixes = "CL_TELEMETRY_PROMETHEUS_BRIDGE_PREFIXES" envTelemetryLogCompressor = "CL_TELEMETRY_LOG_COMPRESSOR" + envMeterRecordsEnabled = "CL_METER_RECORDS_ENABLED" + envMeterSnapshotsEnabled = "CL_METER_SNAPSHOTS_ENABLED" envChipIngressEndpoint = "CL_CHIP_INGRESS_ENDPOINT" envChipIngressInsecureConnection = "CL_CHIP_INGRESS_INSECURE_CONNECTION" @@ -171,6 +173,8 @@ type EnvConfig struct { TelemetryPrometheusBridgeEnabled bool TelemetryPrometheusBridgePrefixes []string TelemetryLogCompressor string + MeterRecordsEnabled bool + MeterSnapshotsEnabled bool TracingEnabled bool TracingCollectorTarget string @@ -264,6 +268,8 @@ func (e *EnvConfig) AsCmdEnv() (env []string) { add(envTelemetryPrometheusBridgeEnabled, strconv.FormatBool(e.TelemetryPrometheusBridgeEnabled)) add(envTelemetryPrometheusBridgePrefixes, strings.Join(e.TelemetryPrometheusBridgePrefixes, ",")) add(envTelemetryLogCompressor, e.TelemetryLogCompressor) + add(envMeterRecordsEnabled, strconv.FormatBool(e.MeterRecordsEnabled)) + add(envMeterSnapshotsEnabled, strconv.FormatBool(e.MeterSnapshotsEnabled)) add(envChipIngressEndpoint, e.ChipIngressEndpoint) add(envChipIngressInsecureConnection, strconv.FormatBool(e.ChipIngressInsecureConnection)) @@ -518,6 +524,15 @@ func (e *EnvConfig) parse() error { e.CRESettings = os.Getenv(envCRESettings) e.CRESettingsDefault = os.Getenv(envCRESettingsDefault) + e.MeterRecordsEnabled, err = getBool(envMeterRecordsEnabled) + if err != nil { + return fmt.Errorf("failed to parse %s: %w", envMeterRecordsEnabled, err) + } + e.MeterSnapshotsEnabled, err = getBool(envMeterSnapshotsEnabled) + if err != nil { + return fmt.Errorf("failed to parse %s: %w", envMeterSnapshotsEnabled, err) + } + return nil } diff --git a/pkg/loop/config_test.go b/pkg/loop/config_test.go index d592543fd4..cf53167da5 100644 --- a/pkg/loop/config_test.go +++ b/pkg/loop/config_test.go @@ -85,6 +85,8 @@ func TestEnvConfig_parse(t *testing.T) { envTelemetryLogStreamingEnabled: "false", envTelemetryPrometheusBridgeEnabled: "true", envTelemetryPrometheusBridgePrefixes: "foo,bar", + envMeterRecordsEnabled: "true", + envMeterSnapshotsEnabled: "false", envChipIngressEndpoint: "chip-ingress.example.com:50051", envChipIngressInsecureConnection: "true", @@ -199,6 +201,8 @@ var envCfgFull = EnvConfig{ TelemetryLogStreamingEnabled: false, TelemetryPrometheusBridgeEnabled: true, TelemetryPrometheusBridgePrefixes: []string{"foo", "bar"}, + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: false, ChipIngressEndpoint: "chip-ingress.example.com:50051", ChipIngressInsecureConnection: true, @@ -264,6 +268,8 @@ func TestEnvConfig_AsCmdEnv(t *testing.T) { assert.Equal(t, "false", got[envTelemetryLogStreamingEnabled]) assert.Equal(t, "true", got[envTelemetryPrometheusBridgeEnabled]) assert.Equal(t, "foo,bar", got[envTelemetryPrometheusBridgePrefixes]) + assert.Equal(t, "true", got[envMeterRecordsEnabled]) + assert.Equal(t, "false", got[envMeterSnapshotsEnabled]) // Assert ChipIngress environment variables assert.Equal(t, "chip-ingress.example.com:50051", got[envChipIngressEndpoint]) diff --git a/pkg/resourcemanager/idempotency.go b/pkg/resourcemanager/idempotency.go index ceff606499..e01e2938d3 100644 --- a/pkg/resourcemanager/idempotency.go +++ b/pkg/resourcemanager/idempotency.go @@ -1,102 +1,47 @@ package resourcemanager import ( - "crypto/sha256" - "encoding/hex" + "math/big" "strconv" - "strings" meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" ) -// IdempotencyKey returns the canonical deduplication key for a MeterRecord: -// the lowercase hex SHA-256 over the full structured identity (including -// node_id), the action, and an event identity, joined with "|": -// -// product|environment|zone|don_id|node_id|service|resource|resource_type|action|resource_id|eventIdentity -// -// where action is the MeterAction enum name (e.g. "METER_ACTION_RESERVE"). -// -// node_id is included, so keys are unique per node: billing dedups a node's -// retries and counts distinct nodes for quorum, while cross-node grouping and -// convergence is the consumer's job on resource_id + dimensions, independent -// of the key. -// -// Keys are level-triggered by design. Producers intentionally reuse the same -// key whenever they re-emit the same resource lifecycle edge — for example a -// node restart re-registering an existing trigger. A repeated key asserts -// "this lifecycle edge happened" rather than "a new event happened". -// -// Consumers therefore use the key for exact-duplicate suppression only. -// Lifecycle state must not be ordered by key arrival: consumers order -// lifecycle state by record timestamp per resource_id, last write wins. -// -// RESERVE and RELEASE are emitted only for genuine allocation/deallocation, -// never for process-lifecycle cleanup of leaked/orphaned resources; a RELEASE -// carries the same value as its paired RESERVE. A reservation lost without a -// RELEASE (e.g. a node crash) is reconciled by its absence from subsequent -// Snapshots, not by a synthetic release. -// -// This helper is the only MeterRecord key generator; producers must not derive -// keys themselves. Inputs are identifiers and must not contain "|". -func IdempotencyKey(id ResourceIdentity, action meteringpb.MeterAction, eventIdentity string) string { - preimage := strings.Join([]string{ - id.Product, - id.Environment, - id.Zone, - id.DONID, - id.NodeID, - id.Service, - id.Resource, - id.ResourceType, - action.String(), - id.ResourceID, - eventIdentity, - }, "|") - sum := sha256.Sum256([]byte(preimage)) - return hex.EncodeToString(sum[:]) +// UtilizationFields identifies one billed utilization dimension. +type UtilizationFields struct { + ResourceType string + ResourceID string + EventID string + OrgID string } -// SnapshotIdempotencyKey returns the canonical deduplication key for one -// MeterSnapshot: the lowercase hex SHA-256 over the literal "snapshot", the -// node-scoped dimensions, the resource and resource_id, and the interval -// bucket, joined with "|": -// -// snapshot|product|environment|zone|don_id|node_id|service|resource|resource_id|interval-bucket -// -// intervalBucket is the snapshot timestamp truncated to the snapshot interval -// (e.g. unix seconds of the interval boundary). It makes each interval's -// snapshot of a resource distinct — so per-interval increments are not -// collapsed — while deduping retries of the same interval. Snapshots are -// action-less, so no action is mixed into the key. -// -// This helper is the only MeterSnapshot key generator; producers must not -// derive keys themselves. Inputs are identifiers and must not contain "|". -func SnapshotIdempotencyKey(id ResourceIdentity, intervalBucket int64) string { - preimage := strings.Join([]string{ - "snapshot", - id.Product, - id.Environment, - id.Zone, - id.DONID, - id.NodeID, - id.Service, - id.Resource, - id.ResourceID, - strconv.FormatInt(intervalBucket, 10), - }, "|") - sum := sha256.Sum256([]byte(preimage)) - return hex.EncodeToString(sum[:]) +// NewUtilization builds a Utilization with int64 quantity encoded as a decimal +// string value. +func NewUtilization(value int64, fields UtilizationFields) *meteringpb.Utilization { + return NewUtilizationString(strconv.FormatInt(value, 10), fields) } -// NewUtilization builds a Utilization for EmitMeterRecord with its idempotency -// key derived via IdempotencyKey, so producers never construct keys by hand. -// The resource being metered is identified entirely by id (with its ResourceID -// set, typically via ResourceIdentity.WithResourceID); Utilization carries no -// labels. -func NewUtilization(id ResourceIdentity, action meteringpb.MeterAction, value int64, eventIdentity string) *meteringpb.Utilization { +// NewUtilizationString builds a Utilization from a pre-formatted numeric string +// value. +func NewUtilizationString(value string, fields UtilizationFields) *meteringpb.Utilization { return &meteringpb.Utilization{ - Value: value, - IdempotencyKey: IdempotencyKey(id, action, eventIdentity), + Value: value, + ResourceType: fields.ResourceType, + ResourceId: fields.ResourceID, + EventId: fields.EventID, + OrgId: fields.OrgID, } } + +// NewUtilizationBig builds a Utilization from an arbitrary-precision integer. +func NewUtilizationBig(value *big.Int, fields UtilizationFields) *meteringpb.Utilization { + if value == nil { + return NewUtilizationString("0", fields) + } + return NewUtilizationString(value.String(), fields) +} + +// NewUtilizationFloat builds a Utilization from a floating-point value. +func NewUtilizationFloat(value float64, fields UtilizationFields) *meteringpb.Utilization { + return NewUtilizationString(strconv.FormatFloat(value, 'g', -1, 64), fields) +} diff --git a/pkg/resourcemanager/meterable.go b/pkg/resourcemanager/meterable.go index d09f1f466f..5d39f81f58 100644 --- a/pkg/resourcemanager/meterable.go +++ b/pkg/resourcemanager/meterable.go @@ -4,6 +4,7 @@ import ( "context" meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" + "google.golang.org/protobuf/proto" ) // Meterable is implemented by producers that manage durable billable @@ -15,8 +16,9 @@ import ( type Meterable interface { // ResourceIdentity returns the producer's base identity: the six coarse // dimensions (product, environment, zone, don_id, node_id, service) plus - // the service-level resource / resource_type. The per-resource resource_id - // is left empty here and populated per active resource by GetUtilization. + // the service-level resource_pool / resource_pool_id. Per-resource billing + // fields (resource_type/resource_id/org_id/event_id/value) are carried by + // Utilizations on MeterRecord and MeterSnapshot. ResourceIdentity() ResourceIdentity // GetUtilization returns the current level of the producer's currently @@ -34,17 +36,15 @@ type Meterable interface { } // SnapshotEntry is the current level of one active resource at a snapshot tick. -// Identity is the full per-resource identity (the producer's base dimensions -// with resource / resource_type / resource_id populated for this specific -// resource), and Value is its current level. The resource is identified -// entirely by Identity; there is no separate label metadata. +// Identity is the base resource identity, and Utilizations carries one or more +// billed dimensions for that resource (resource_type/resource_id/org_id/event_id/value). type SnapshotEntry struct { - Identity ResourceIdentity - Value int64 + Identity ResourceIdentity + Utilizations []*meteringpb.Utilization } // ResourceIdentity is the structured, first-class identity of a durable -// resource. Its nine fields map 1:1 to metering.v1.ResourceIdentity so every +// resource. Its fields map 1:1 to metering.v1.ResourceIdentity so every // emitted record carries each dimension as a discrete column rather than a // parsed dotted string or out-of-band telemetry attribute. type ResourceIdentity struct { @@ -76,44 +76,29 @@ type ResourceIdentity struct { // encode deployment environment or zone. Service string - // Resource is the resource pool the record applies to, e.g. - // "trigger_registrations", "log_filters". - Resource string + // ResourcePool is the service-level resource pool the record applies to, + // e.g. "trigger_registrations", "log_filters", "workflow_storage". + ResourcePool string - // ResourceType is the billing unit used to convert the utilization value - // to universal credits, e.g. "operations", "log_filter_addresses". - ResourceType string - - // ResourceID is the physical/logical resource identity, workflow- - // independent where a shared physical resource exists. For EVM log filters - // it is the content hash of (chain_selector + canonicalized addresses + - // event sigs + positional topics), so identical filters from different - // workflows share one ResourceID. For cron/http/syncer (no shared physical - // resource) it is the workflow-scoped trigger_id / workflow_id, from which - // the workflow (and, downstream, the owner) is recoverable. ResourceIdentity - // is the sole identity of a metered resource; there is no label metadata. - ResourceID string -} - -// WithResourceID returns a copy of r with ResourceID set to id, leaving the -// receiver unchanged. Producers build a base identity once and derive a -// per-resource identity with this helper. -func (r ResourceIdentity) WithResourceID(id string) ResourceIdentity { - r.ResourceID = id - return r + // ResourcePoolID optionally scopes identity further within the resource pool. + ResourcePoolID string } // toProto converts r to its wire form. Field order mirrors the proto. func (r ResourceIdentity) toProto() *meteringpb.ResourceIdentity { - return &meteringpb.ResourceIdentity{ - Product: r.Product, - Environment: r.Environment, - Zone: r.Zone, - DonId: r.DONID, - NodeId: r.NodeID, - Service: r.Service, - Resource: r.Resource, - ResourceType: r.ResourceType, - ResourceId: r.ResourceID, + pb := &meteringpb.ResourceIdentity{ + Product: r.Product, + Environment: r.Environment, + Zone: r.Zone, + Service: r.Service, + ResourcePool: r.ResourcePool, + ResourcePoolId: r.ResourcePoolID, + } + if r.DONID != "" { + pb.DonId = proto.String(r.DONID) + } + if r.NodeID != "" { + pb.NodeId = proto.String(r.NodeID) } + return pb } diff --git a/pkg/resourcemanager/resourcemanager.go b/pkg/resourcemanager/resourcemanager.go index 70e6ac0b20..977967211c 100644 --- a/pkg/resourcemanager/resourcemanager.go +++ b/pkg/resourcemanager/resourcemanager.go @@ -24,6 +24,7 @@ package resourcemanager import ( "context" + "strconv" "sync" "time" @@ -87,10 +88,13 @@ type Emitter interface { // ResourceManagerConfig configures a ResourceManager. type ResourceManagerConfig struct { - // Enabled is the metering rollout gate. When false (the default), the - // ResourceManager is a no-op: EmitMeterRecord neither marshals nor emits - // nor records utilization, and the snapshot loop does not start. - Enabled bool + // MeterRecordsEnabled is the meter-record rollout gate. When false (the + // default), EmitMeterRecord is a no-op. + MeterRecordsEnabled bool + + // MeterSnapshotsEnabled gates snapshot emission. Snapshots are only emitted + // when this is true AND MeterRecordsEnabled is true. + MeterSnapshotsEnabled bool // Emitter delivers encoded records, typically beholder.GetEmitter(). A nil // Emitter makes EmitMeterRecord a no-op even when Enabled is true and keeps @@ -123,11 +127,12 @@ type ResourceManager struct { services.Service srvcEng *services.Engine - lggr logger.SugaredLogger - enabled bool - emitter Emitter - snapshotInterval time.Duration - clock clockwork.Clock + lggr logger.SugaredLogger + meterRecordsEnabled bool + meterSnapshotsEnabled bool + emitter Emitter + snapshotInterval time.Duration + clock clockwork.Clock mu sync.RWMutex registrations map[*registration]struct{} @@ -165,17 +170,23 @@ func NewResourceManager(lggr logger.Logger, cfg ResourceManagerConfig) *Resource clock = clockwork.NewRealClock() } + meterSnapshotsEnabled := cfg.MeterRecordsEnabled && cfg.MeterSnapshotsEnabled && cfg.SnapshotInterval > 0 + if cfg.MeterSnapshotsEnabled && !cfg.MeterRecordsEnabled { + sugared.Warn("MeterSnapshotsEnabled ignored because MeterRecordsEnabled is false") + } + rm := &ResourceManager{ - enabled: cfg.Enabled, - emitter: cfg.Emitter, - snapshotInterval: cfg.SnapshotInterval, - clock: clock, - registrations: make(map[*registration]struct{}), - emitSuccess: newCounter(emitSuccessCounterName), - emitFailure: newCounter(emitFailureCounterName), - snapshotEmitSuccess: newCounter(snapshotEmitSuccessCounterName), - snapshotEmitFailure: newCounter(snapshotEmitFailureCounterName), - utilization: gauge, + meterRecordsEnabled: cfg.MeterRecordsEnabled, + meterSnapshotsEnabled: meterSnapshotsEnabled, + emitter: cfg.Emitter, + snapshotInterval: cfg.SnapshotInterval, + clock: clock, + registrations: make(map[*registration]struct{}), + emitSuccess: newCounter(emitSuccessCounterName), + emitFailure: newCounter(emitFailureCounterName), + snapshotEmitSuccess: newCounter(snapshotEmitSuccessCounterName), + snapshotEmitFailure: newCounter(snapshotEmitFailureCounterName), + utilization: gauge, } rm.Service, rm.srvcEng = services.Config{ Name: "ResourceManager", @@ -191,9 +202,10 @@ func NewResourceManager(lggr logger.Logger, cfg ResourceManagerConfig) *Resource // otherwise the service starts cleanly with snapshots disabled and // EmitMeterRecord remains available. func (rm *ResourceManager) start(_ context.Context) error { - if !rm.enabled || rm.emitter == nil || rm.snapshotInterval <= 0 { + if !rm.meterSnapshotsEnabled || rm.emitter == nil { rm.lggr.Infow("snapshot loop disabled", - "enabled", rm.enabled, + "meterRecordsEnabled", rm.meterRecordsEnabled, + "meterSnapshotsEnabled", rm.meterSnapshotsEnabled, "hasEmitter", rm.emitter != nil, "snapshotInterval", rm.snapshotInterval, ) @@ -259,34 +271,35 @@ func (rm *ResourceManager) emitSnapshots(ctx context.Context) { // nothing: billing zeroes a resource out by its absence from later snapshots. // Fail-open: per-resource errors are logged and counted, never returned. func (rm *ResourceManager) emitSnapshot(ctx context.Context, m Meterable) { - if !rm.enabled || rm.emitter == nil { + if !rm.meterSnapshotsEnabled || rm.emitter == nil { return } now := rm.clock.Now() - bucket := now.Truncate(rm.snapshotInterval).Unix() ts := timestamppb.New(now) interval := durationpb.New(rm.snapshotInterval) for _, e := range m.GetUtilization(ctx) { - rm.recordUtilization(ctx, e.Identity, e.Value) + if len(e.Utilizations) == 0 { + continue + } + for _, u := range e.Utilizations { + rm.recordUtilization(ctx, e.Identity, u, meteringpb.MeterAction_METER_ACTION_UPDATE) + } snapshot := &meteringpb.MeterSnapshot{ - Timestamp: ts, - Identity: e.Identity.toProto(), - Utilization: &meteringpb.Utilization{ - Value: e.Value, - IdempotencyKey: SnapshotIdempotencyKey(e.Identity, bucket), - }, - Interval: interval, + Timestamp: ts, + Identity: e.Identity.toProto(), + Utilization: e.Utilizations, + Interval: interval, } body, err := proto.Marshal(snapshot) if err != nil { rm.lggr.Errorw("failed to marshal snapshot", "service", e.Identity.Service, - "resource", e.Identity.Resource, - "resourceID", e.Identity.ResourceID, + "resourcePool", e.Identity.ResourcePool, + "resourcePoolID", e.Identity.ResourcePoolID, "err", err, ) rm.countSnapshot(ctx, rm.snapshotEmitFailure, e.Identity, attribute.String("reason", "marshal")) @@ -300,8 +313,8 @@ func (rm *ResourceManager) emitSnapshot(ctx context.Context, m Meterable) { ); err != nil { rm.lggr.Errorw("failed to emit snapshot", "service", e.Identity.Service, - "resource", e.Identity.Resource, - "resourceID", e.Identity.ResourceID, + "resourcePool", e.Identity.ResourcePool, + "resourcePoolID", e.Identity.ResourcePoolID, "err", err, ) rm.countSnapshot(ctx, rm.snapshotEmitFailure, e.Identity, attribute.String("reason", "emit")) @@ -319,41 +332,36 @@ func (rm *ResourceManager) emitSnapshot(ctx context.Context, m Meterable) { // disabled or has no emitter it does nothing, and marshal or emit failures are // recorded only via error-level logs and the failure counter. Callers must // never gate resource allocation or deallocation on emission. -func (rm *ResourceManager) EmitMeterRecord(ctx context.Context, identity ResourceIdentity, action meteringpb.MeterAction, utilization *meteringpb.Utilization) { - if !rm.enabled { +func (rm *ResourceManager) EmitMeterRecord(ctx context.Context, identity ResourceIdentity, action meteringpb.MeterAction, utilizations []*meteringpb.Utilization) { + if !rm.meterRecordsEnabled { rm.lggr.Debugw("metering disabled; meter record not emitted", "service", identity.Service, - "resource", identity.Resource, + "resourcePool", identity.ResourcePool, "action", action.String(), ) return } - // Reflect the lifecycle edge in the utilization gauge: a RELEASE drops the - // resource to zero, every other action records its current level. Recorded - // before the emitter check so the gauge tracks state even if export is off. - gaugeValue := utilization.GetValue() - if action == meteringpb.MeterAction_METER_ACTION_RELEASE { - gaugeValue = 0 + for _, u := range utilizations { + rm.recordUtilization(ctx, identity, u, action) } - rm.recordUtilization(ctx, identity, gaugeValue) if rm.emitter == nil { return } record := &meteringpb.MeterRecord{ - Timestamp: timestamppb.New(rm.clock.Now()), - Identity: identity.toProto(), - Action: action, - Utilization: utilization, + Timestamp: timestamppb.New(rm.clock.Now()), + Identity: identity.toProto(), + Action: action, + Utilizations: utilizations, } body, err := proto.Marshal(record) if err != nil { rm.lggr.Errorw("failed to marshal meter record", "service", identity.Service, - "resource", identity.Resource, + "resourcePool", identity.ResourcePool, "action", action.String(), "err", err, ) @@ -368,7 +376,7 @@ func (rm *ResourceManager) EmitMeterRecord(ctx context.Context, identity Resourc ); err != nil { rm.lggr.Errorw("failed to emit meter record", "service", identity.Service, - "resource", identity.Resource, + "resourcePool", identity.ResourcePool, "action", action.String(), "err", err, ) @@ -380,13 +388,32 @@ func (rm *ResourceManager) EmitMeterRecord(ctx context.Context, identity Resourc } // recordUtilization records value to the per-resource utilization gauge, -// labeled with every ResourceIdentity dimension. This is intentionally high -// cardinality (it includes node_id and resource_id) so utilization can be -// sliced by any dimension downstream. -func (rm *ResourceManager) recordUtilization(ctx context.Context, id ResourceIdentity, value int64) { +// labeled with every ResourceIdentity dimension plus utilization identity. +func (rm *ResourceManager) recordUtilization(ctx context.Context, id ResourceIdentity, u *meteringpb.Utilization, action meteringpb.MeterAction) { if rm.utilization == nil { return } + if u == nil { + return + } + + var value int64 + if action == meteringpb.MeterAction_METER_ACTION_RELEASE { + value = 0 + } else { + parsed, err := strconv.ParseInt(u.GetValue(), 10, 64) + if err != nil { + rm.lggr.Debugw("skipping utilization gauge record for non-int64 value", + "value", u.GetValue(), + "resourceType", u.GetResourceType(), + "resourceID", u.GetResourceId(), + "orgID", u.GetOrgId(), + "err", err, + ) + return + } + value = parsed + } rm.utilization.Record(ctx, value, metric.WithAttributes( attribute.String("product", id.Product), attribute.String("environment", id.Environment), @@ -394,9 +421,11 @@ func (rm *ResourceManager) recordUtilization(ctx context.Context, id ResourceIde attribute.String("don_id", id.DONID), attribute.String("node_id", id.NodeID), attribute.String("service", id.Service), - attribute.String("resource", id.Resource), - attribute.String("resource_type", id.ResourceType), - attribute.String("resource_id", id.ResourceID), + attribute.String("resource_pool", id.ResourcePool), + attribute.String("resource_pool_id", id.ResourcePoolID), + attribute.String("resource_type", u.GetResourceType()), + attribute.String("resource_id", u.GetResourceId()), + attribute.String("org_id", u.GetOrgId()), )) } @@ -410,7 +439,7 @@ func (rm *ResourceManager) countRecord(ctx context.Context, c metric.Int64Counte } attrs := append([]attribute.KeyValue{ attribute.String("service", identity.Service), - attribute.String("resource", identity.Resource), + attribute.String("resource_pool", identity.ResourcePool), attribute.String("action", action.String()), }, extra...) c.Add(ctx, 1, metric.WithAttributes(attrs...)) @@ -424,7 +453,7 @@ func (rm *ResourceManager) countSnapshot(ctx context.Context, c metric.Int64Coun } attrs := append([]attribute.KeyValue{ attribute.String("service", identity.Service), - attribute.String("resource", identity.Resource), + attribute.String("resource_pool", identity.ResourcePool), }, extra...) c.Add(ctx, 1, metric.WithAttributes(attrs...)) } diff --git a/pkg/resourcemanager/resourcemanager_test.go b/pkg/resourcemanager/resourcemanager_test.go index 4f7d0f2d7b..b0ab380be1 100644 --- a/pkg/resourcemanager/resourcemanager_test.go +++ b/pkg/resourcemanager/resourcemanager_test.go @@ -3,6 +3,7 @@ package resourcemanager import ( "context" "errors" + "math/big" "sync" "testing" "time" @@ -20,14 +21,14 @@ import ( ) var testIdentity = ResourceIdentity{ - Product: "cre-mainline", - Environment: "production", - Zone: "wf-zone-a", - DONID: "don-1", - NodeID: "node-1", - Service: "cron-trigger", - Resource: "trigger_registrations", - ResourceType: "operations", + Product: "cre-mainline", + Environment: "production", + Zone: "wf-zone-a", + DONID: "don-1", + NodeID: "node-1", + Service: "cron-trigger", + ResourcePool: "trigger_registrations", + ResourcePoolID: "", } type emitCall struct { @@ -54,8 +55,6 @@ func (f *fakeEmitter) snapshot() []emitCall { return append([]emitCall(nil), f.calls...) } -// fakeMeterable is a test Meterable whose active entries can be swapped under -// lock to exercise registration/unregistration and changing utilization. type fakeMeterable struct { identity ResourceIdentity @@ -77,20 +76,18 @@ func (f *fakeMeterable) set(entries []SnapshotEntry) { f.entries = entries } -// attrMap converts beholder-style attribute key/value pairs into a map. func attrMap(t *testing.T, attrKVs []any) map[string]any { t.Helper() - require.Zero(t, len(attrKVs)%2, "attrKVs must be key/value pairs") + require.Zero(t, len(attrKVs)%2) m := make(map[string]any, len(attrKVs)/2) for i := 0; i < len(attrKVs); i += 2 { key, ok := attrKVs[i].(string) - require.True(t, ok, "attr key must be a string") + require.True(t, ok) m[key] = attrKVs[i+1] } return m } -// waitForSnapshotCount polls until the emitter has recorded want snapshot emissions. func waitForSnapshotCount(t *testing.T, emitter *fakeEmitter, want int) { t.Helper() require.Eventually(t, func() bool { @@ -100,48 +97,56 @@ func waitForSnapshotCount(t *testing.T, emitter *fakeEmitter, want int) { func TestEmitMeterRecord_Gating(t *testing.T) { tests := []struct { - name string - enabled bool - emitter *fakeEmitter - wantEmits int + name string + recordsEnabled bool + snapshotsEnabled bool + emitter *fakeEmitter + wantEmits int }{ - {name: "disabled does not emit", enabled: false, emitter: &fakeEmitter{}, wantEmits: 0}, - {name: "disabled with nil emitter does not panic", enabled: false, emitter: nil, wantEmits: 0}, - {name: "enabled with nil emitter does not panic", enabled: true, emitter: nil, wantEmits: 0}, - {name: "enabled emits", enabled: true, emitter: &fakeEmitter{}, wantEmits: 1}, + {name: "disabled does not emit", recordsEnabled: false, snapshotsEnabled: true, emitter: &fakeEmitter{}, wantEmits: 0}, + {name: "enabled emits", recordsEnabled: true, snapshotsEnabled: true, emitter: &fakeEmitter{}, wantEmits: 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cfg := ResourceManagerConfig{Enabled: tt.enabled} + cfg := ResourceManagerConfig{ + MeterRecordsEnabled: tt.recordsEnabled, + MeterSnapshotsEnabled: tt.snapshotsEnabled, + SnapshotInterval: time.Minute, + } if tt.emitter != nil { cfg.Emitter = tt.emitter } rm := NewResourceManager(logger.Test(t), cfg) - - rm.EmitMeterRecord(t.Context(), testIdentity, meteringpb.MeterAction_METER_ACTION_RESERVE, - NewUtilization(testIdentity.WithResourceID("trigger-1"), meteringpb.MeterAction_METER_ACTION_RESERVE, 1, "event-1")) - - if tt.emitter != nil { - assert.Len(t, tt.emitter.calls, tt.wantEmits) - } + u := NewUtilization(1, UtilizationFields{ + ResourceType: "operations", + ResourceID: "trigger-1", + EventID: "event-1", + OrgID: "org-1", + }) + rm.EmitMeterRecord(t.Context(), testIdentity, meteringpb.MeterAction_METER_ACTION_RESERVE, []*meteringpb.Utilization{u}) + assert.Len(t, tt.emitter.calls, tt.wantEmits) }) } } func TestEmitMeterRecord_Success(t *testing.T) { emitter := &fakeEmitter{} - rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{Enabled: true, Emitter: emitter}) + rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ + MeterRecordsEnabled: true, + Emitter: emitter, + }) - id := testIdentity.WithResourceID("trigger-1") - utilization := NewUtilization(id, meteringpb.MeterAction_METER_ACTION_RESERVE, 1, "event-1") - before := time.Now() - rm.EmitMeterRecord(t.Context(), id, meteringpb.MeterAction_METER_ACTION_RESERVE, utilization) - after := time.Now() + u := NewUtilization(1, UtilizationFields{ + ResourceType: "operations", + ResourceID: "trigger-1", + EventID: "event-1", + OrgID: "org-1", + }) + rm.EmitMeterRecord(t.Context(), testIdentity, meteringpb.MeterAction_METER_ACTION_RESERVE, []*meteringpb.Utilization{u}) require.Len(t, emitter.calls, 1) call := emitter.calls[0] - attrs := attrMap(t, call.attrKVs) assert.Equal(t, "metering.v1.meter_record", attrs[beholder.AttrKeyDataSchema]) assert.Equal(t, "platform", attrs[beholder.AttrKeyDomain]) @@ -150,94 +155,35 @@ func TestEmitMeterRecord_Success(t *testing.T) { var record meteringpb.MeterRecord require.NoError(t, proto.Unmarshal(call.body, &record)) require.NotNil(t, record.GetIdentity()) - assert.Equal(t, id.Product, record.GetIdentity().GetProduct()) - assert.Equal(t, id.NodeID, record.GetIdentity().GetNodeId()) - assert.Equal(t, id.Service, record.GetIdentity().GetService()) - assert.Equal(t, id.Resource, record.GetIdentity().GetResource()) - assert.Equal(t, id.ResourceType, record.GetIdentity().GetResourceType()) - assert.Equal(t, "trigger-1", record.GetIdentity().GetResourceId()) + assert.Equal(t, testIdentity.ResourcePool, record.GetIdentity().GetResourcePool()) assert.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, record.GetAction()) - - require.NotNil(t, record.GetUtilization()) - assert.Equal(t, int64(1), record.GetUtilization().GetValue()) - assert.Equal(t, - IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1"), - record.GetUtilization().GetIdempotencyKey()) - - require.NotNil(t, record.GetTimestamp()) - ts := record.GetTimestamp().AsTime() - assert.False(t, ts.Before(before.Add(-time.Second)), "timestamp too early: %s", ts) - assert.False(t, ts.After(after.Add(time.Second)), "timestamp too late: %s", ts) + require.Len(t, record.GetUtilizations(), 1) + assert.Equal(t, "1", record.GetUtilizations()[0].GetValue()) + assert.Equal(t, "operations", record.GetUtilizations()[0].GetResourceType()) + assert.Equal(t, "trigger-1", record.GetUtilizations()[0].GetResourceId()) + assert.Equal(t, "event-1", record.GetUtilizations()[0].GetEventId()) + assert.Equal(t, "org-1", record.GetUtilizations()[0].GetOrgId()) } func TestEmitMeterRecord_EmitFailureIsSwallowed(t *testing.T) { emitter := &fakeEmitter{err: errors.New("collector unavailable")} - rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{Enabled: true, Emitter: emitter}) - - id := testIdentity.WithResourceID("trigger-1") - require.NotPanics(t, func() { - rm.EmitMeterRecord(t.Context(), id, meteringpb.MeterAction_METER_ACTION_RELEASE, - NewUtilization(id, meteringpb.MeterAction_METER_ACTION_RELEASE, 1, "event-2")) - }) - require.Len(t, emitter.calls, 1) - - // A subsequent emission still goes through; the failure left no state behind. - rm.EmitMeterRecord(t.Context(), id, meteringpb.MeterAction_METER_ACTION_RELEASE, - NewUtilization(id, meteringpb.MeterAction_METER_ACTION_RELEASE, 1, "event-3")) - require.Len(t, emitter.calls, 2) -} - -// TestEmitMeterRecord_BeholderObserver wires the manager to the global -// beholder emitter the way production does. beholdertest.NewObserver is not -// parallel-safe, so this test must not run in parallel. -func TestEmitMeterRecord_BeholderObserver(t *testing.T) { - observer := beholdertest.NewObserver(t) - rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ - Enabled: true, - Emitter: beholder.GetEmitter(), + MeterRecordsEnabled: true, + Emitter: emitter, }) - id := testIdentity.WithResourceID("trigger-1") - rm.EmitMeterRecord(t.Context(), id, meteringpb.MeterAction_METER_ACTION_RESERVE, - NewUtilization(id, meteringpb.MeterAction_METER_ACTION_RESERVE, 1, "event-1")) - - msgs := observer.Messages(t, beholder.AttrKeyEntity, "metering.v1.MeterRecord") - require.Len(t, msgs, 1) - assert.Equal(t, "metering.v1.meter_record", msgs[0].Attrs[beholder.AttrKeyDataSchema]) - assert.Equal(t, "platform", msgs[0].Attrs[beholder.AttrKeyDomain]) - - var record meteringpb.MeterRecord - require.NoError(t, proto.Unmarshal(msgs[0].Body, &record)) - assert.Equal(t, id.Service, record.GetIdentity().GetService()) -} - -// TestResourceManager_StartClose asserts the manager starts and closes cleanly -// both with snapshots enabled and with the loop disabled (zero interval). -func TestResourceManager_StartClose(t *testing.T) { - t.Run("snapshot loop enabled", func(t *testing.T) { - rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ - Enabled: true, - Emitter: &fakeEmitter{}, - SnapshotInterval: time.Hour, - }) - require.NoError(t, rm.Start(t.Context())) - require.NoError(t, rm.Close()) + u := NewUtilization(1, UtilizationFields{ + ResourceType: "operations", + ResourceID: "trigger-1", + EventID: "event-2", + OrgID: "org-1", }) - - t.Run("snapshot loop disabled by zero interval", func(t *testing.T) { - rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ - Enabled: true, - Emitter: &fakeEmitter{}, - // SnapshotInterval unset -> disabled. - }) - require.NoError(t, rm.Start(t.Context())) - require.NoError(t, rm.Close()) + require.NotPanics(t, func() { + rm.EmitMeterRecord(t.Context(), testIdentity, meteringpb.MeterAction_METER_ACTION_RELEASE, []*meteringpb.Utilization{u}) }) + require.Len(t, emitter.calls, 1) } -// decodeSnapshots extracts every MeterSnapshot from the emitter's recorded -// calls. func decodeSnapshots(t *testing.T, calls []emitCall) []*meteringpb.MeterSnapshot { t.Helper() var snaps []*meteringpb.MeterSnapshot @@ -246,8 +192,6 @@ func decodeSnapshots(t *testing.T, calls []emitCall) []*meteringpb.MeterSnapshot if attrs[beholder.AttrKeyEntity] != "metering.v1.MeterSnapshot" { continue } - assert.Equal(t, "metering.v1.meter_snapshot", attrs[beholder.AttrKeyDataSchema]) - assert.Equal(t, "platform", attrs[beholder.AttrKeyDomain]) var s meteringpb.MeterSnapshot require.NoError(t, proto.Unmarshal(c.body, &s)) snaps = append(snaps, &s) @@ -255,30 +199,39 @@ func decodeSnapshots(t *testing.T, calls []emitCall) []*meteringpb.MeterSnapshot return snaps } -// snapshotKey recomputes the expected idempotency key for a decoded snapshot -// from its own timestamp and interval, so the assertion does not depend on the -// wall clock at emit time. -func snapshotKey(s *meteringpb.MeterSnapshot, id ResourceIdentity) string { - bucket := s.GetTimestamp().AsTime().Truncate(s.GetInterval().AsDuration()).Unix() - return SnapshotIdempotencyKey(id, bucket) -} - func TestEmitSnapshot_OnePerResource(t *testing.T) { emitter := &fakeEmitter{} clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, - SnapshotInterval: 30 * time.Second, - Clock: clock, + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: 30 * time.Second, + Clock: clock, }) m := &fakeMeterable{identity: testIdentity} - id1 := testIdentity.WithResourceID("trigger-1") - id2 := testIdentity.WithResourceID("trigger-2") m.set([]SnapshotEntry{ - {Identity: id1, Value: 3}, - {Identity: id2, Value: 5}, + { + Identity: testIdentity, + Utilizations: []*meteringpb.Utilization{ + NewUtilization(3, UtilizationFields{ + ResourceType: "operations", + ResourceID: "trigger-1", + OrgID: "org-1", + }), + }, + }, + { + Identity: testIdentity, + Utilizations: []*meteringpb.Utilization{ + NewUtilization(5, UtilizationFields{ + ResourceType: "operations", + ResourceID: "trigger-2", + OrgID: "org-2", + }), + }, + }, }) unregister := rm.Register(m) defer unregister() @@ -288,238 +241,82 @@ func TestEmitSnapshot_OnePerResource(t *testing.T) { clock.Advance(30 * time.Second) waitForSnapshotCount(t, emitter, 2) - // Exactly one MeterSnapshot per active resource — never a batch. snaps := decodeSnapshots(t, emitter.snapshot()) require.Len(t, snaps, 2) byID := map[string]*meteringpb.MeterSnapshot{} for _, s := range snaps { - // Each snapshot carries the full per-resource identity. - require.NotNil(t, s.GetIdentity()) - assert.Equal(t, testIdentity.Service, s.GetIdentity().GetService()) - require.NotNil(t, s.GetInterval()) - assert.Equal(t, 30*time.Second, s.GetInterval().AsDuration()) - require.NotNil(t, s.GetTimestamp()) - byID[s.GetIdentity().GetResourceId()] = s + require.Len(t, s.GetUtilization(), 1) + byID[s.GetUtilization()[0].GetResourceId()] = s } - - s1 := byID["trigger-1"] - require.NotNil(t, s1) - assert.Equal(t, int64(3), s1.GetUtilization().GetValue()) - assert.Equal(t, snapshotKey(s1, id1), s1.GetUtilization().GetIdempotencyKey()) - - s2 := byID["trigger-2"] - require.NotNil(t, s2) - assert.Equal(t, int64(5), s2.GetUtilization().GetValue()) - assert.Equal(t, snapshotKey(s2, id2), s2.GetUtilization().GetIdempotencyKey()) + require.NotNil(t, byID["trigger-1"]) + assert.Equal(t, "3", byID["trigger-1"].GetUtilization()[0].GetValue()) + require.NotNil(t, byID["trigger-2"]) + assert.Equal(t, "5", byID["trigger-2"].GetUtilization()[0].GetValue()) } -func TestEmitSnapshot_EmptyListEmitsNothing(t *testing.T) { +func TestSnapshotsCannotRunWithoutRecords(t *testing.T) { emitter := &fakeEmitter{} clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, - SnapshotInterval: time.Minute, - Clock: clock, + MeterRecordsEnabled: false, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: 10 * time.Second, + Clock: clock, }) - m := &fakeMeterable{identity: testIdentity} // no active entries - unregister := rm.Register(m) - defer unregister() - + m := &fakeMeterable{identity: testIdentity} + m.set([]SnapshotEntry{{ + Identity: testIdentity, + Utilizations: []*meteringpb.Utilization{ + NewUtilization(1, UtilizationFields{ + ResourceType: "operations", + ResourceID: "trigger-1", + }), + }, + }}) + rm.Register(m) servicetest.Run(t, rm) - - // Nothing active -> no snapshots. Billing zeroes the resource out by its - // absence from subsequent snapshots, not by an explicit empty record. assert.Empty(t, decodeSnapshots(t, emitter.snapshot())) } -func TestEmitSnapshots_KeyStableWithinInterval(t *testing.T) { - emitter := &fakeEmitter{} - rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, - SnapshotInterval: time.Hour, // wide bucket so all ticks share one interval - }) - - m := &fakeMeterable{identity: testIdentity} - m.set([]SnapshotEntry{{Identity: testIdentity.WithResourceID("trigger-1"), Value: 1}}) - unregister := rm.Register(m) - defer unregister() - - // Drive three ticks directly (no timer) to keep the test deterministic. - rm.emitSnapshots(t.Context()) - rm.emitSnapshots(t.Context()) - rm.emitSnapshots(t.Context()) - - snaps := decodeSnapshots(t, emitter.snapshot()) - require.Len(t, snaps, 3, "one MeterSnapshot per resource per tick") - - // Same resource, same interval bucket -> identical keys (retries dedup); - // successive intervals would differ. - k0 := snaps[0].GetUtilization().GetIdempotencyKey() - assert.Equal(t, k0, snaps[1].GetUtilization().GetIdempotencyKey()) - assert.Equal(t, k0, snaps[2].GetUtilization().GetIdempotencyKey()) -} - -func TestRegister_UnregisterStopsSnapshots(t *testing.T) { - emitter := &fakeEmitter{} - rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, - SnapshotInterval: time.Minute, - }) - - m := &fakeMeterable{identity: testIdentity} - m.set([]SnapshotEntry{{Identity: testIdentity.WithResourceID("trigger-1"), Value: 1}}) - unregister := rm.Register(m) +func TestNewUtilizationVariants(t *testing.T) { + fields := UtilizationFields{ + ResourceType: "operations", + ResourceID: "rid-1", + EventID: "evt-1", + OrgID: "org-1", + } - rm.emitSnapshots(t.Context()) - require.Len(t, decodeSnapshots(t, emitter.snapshot()), 1) + uInt := NewUtilization(42, fields) + assert.Equal(t, "42", uInt.GetValue()) + assert.Equal(t, "operations", uInt.GetResourceType()) + assert.Equal(t, "rid-1", uInt.GetResourceId()) + assert.Equal(t, "evt-1", uInt.GetEventId()) + assert.Equal(t, "org-1", uInt.GetOrgId()) - unregister() - // Idempotent: a second call is a no-op. - require.NotPanics(t, unregister) + uBig := NewUtilizationBig(big.NewInt(0).Exp(big.NewInt(2), big.NewInt(80), nil), fields) + assert.Equal(t, "1208925819614629174706176", uBig.GetValue()) - rm.emitSnapshots(t.Context()) - require.Len(t, decodeSnapshots(t, emitter.snapshot()), 1, "unregistered Meterable must not be snapshotted") + uFloat := NewUtilizationFloat(3.5, fields) + assert.Equal(t, "3.5", uFloat.GetValue()) } -func TestEmitSnapshot_FailureIsSwallowed(t *testing.T) { - emitter := &fakeEmitter{err: errors.New("collector unavailable")} - rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ - Enabled: true, - Emitter: emitter, - SnapshotInterval: time.Minute, - }) - - m := &fakeMeterable{identity: testIdentity} - m.set([]SnapshotEntry{{Identity: testIdentity.WithResourceID("trigger-1"), Value: 1}}) - unregister := rm.Register(m) - defer unregister() - - require.NotPanics(t, func() { rm.emitSnapshots(t.Context()) }) - require.Len(t, emitter.snapshot(), 1) -} - -// TestEmitSnapshot_BeholderObserver wires snapshots through the global beholder -// emitter. beholdertest.NewObserver is not parallel-safe; do not run this in -// parallel. -func TestEmitSnapshot_BeholderObserver(t *testing.T) { +func TestEmitMeterRecord_BeholderObserver(t *testing.T) { observer := beholdertest.NewObserver(t) - clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) - rm := NewResourceManager(logger.Test(t), ResourceManagerConfig{ - Enabled: true, - Emitter: beholder.GetEmitter(), - SnapshotInterval: time.Minute, - Clock: clock, + MeterRecordsEnabled: true, + Emitter: beholder.GetEmitter(), }) - - m := &fakeMeterable{identity: testIdentity} - m.set([]SnapshotEntry{{Identity: testIdentity.WithResourceID("trigger-1"), Value: 7}}) - unregister := rm.Register(m) - defer unregister() - - servicetest.Run(t, rm) - require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) - clock.Advance(time.Minute) - - require.Eventually(t, func() bool { - return len(observer.Messages(t, beholder.AttrKeyEntity, "metering.v1.MeterSnapshot")) == 1 - }, time.Second, time.Millisecond) - - msgs := observer.Messages(t, beholder.AttrKeyEntity, "metering.v1.MeterSnapshot") - require.Len(t, msgs, 1) - assert.Equal(t, "metering.v1.meter_snapshot", msgs[0].Attrs[beholder.AttrKeyDataSchema]) - - var s meteringpb.MeterSnapshot - require.NoError(t, proto.Unmarshal(msgs[0].Body, &s)) - assert.Equal(t, "trigger-1", s.GetIdentity().GetResourceId()) - assert.Equal(t, int64(7), s.GetUtilization().GetValue()) -} - -func TestIdempotencyKey(t *testing.T) { - id := testIdentity.WithResourceID("trigger-1") - base := IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1") - - t.Run("deterministic", func(t *testing.T) { - assert.Equal(t, base, IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1")) - }) - - t.Run("format is sha256 hex", func(t *testing.T) { - assert.Regexp(t, "^[0-9a-f]{64}$", base) - }) - - otherNode := id - otherNode.NodeID = "node-2" - otherResource := id - otherResource.Resource = "log_filters" - - distinct := []struct { - name string - key string - }{ - {"action", IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RELEASE, "event-1")}, - {"node_id", IdempotencyKey(otherNode, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1")}, - {"resource", IdempotencyKey(otherResource, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1")}, - {"resource ID", IdempotencyKey(id.WithResourceID("trigger-2"), meteringpb.MeterAction_METER_ACTION_RESERVE, "event-1")}, - {"event identity", IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RESERVE, "event-2")}, - } - for _, tt := range distinct { - t.Run("distinct across "+tt.name, func(t *testing.T) { - assert.NotEqual(t, base, tt.key) - }) - } -} - -func TestSnapshotIdempotencyKey(t *testing.T) { - id := testIdentity.WithResourceID("trigger-1") - base := SnapshotIdempotencyKey(id, 100) - - t.Run("format is sha256 hex", func(t *testing.T) { - assert.Regexp(t, "^[0-9a-f]{64}$", base) - }) - - t.Run("stable within an interval bucket", func(t *testing.T) { - assert.Equal(t, base, SnapshotIdempotencyKey(id, 100)) + u := NewUtilization(1, UtilizationFields{ + ResourceType: "operations", + ResourceID: "trigger-1", + EventID: "event-1", + OrgID: "org-1", }) + rm.EmitMeterRecord(t.Context(), testIdentity, meteringpb.MeterAction_METER_ACTION_RESERVE, []*meteringpb.Utilization{u}) - t.Run("differs per interval bucket", func(t *testing.T) { - assert.NotEqual(t, base, SnapshotIdempotencyKey(id, 101)) - }) - - t.Run("differs per resource_id", func(t *testing.T) { - assert.NotEqual(t, base, SnapshotIdempotencyKey(id.WithResourceID("trigger-2"), 100)) - }) - - t.Run("differs per node_id", func(t *testing.T) { - other := id - other.NodeID = "node-2" - assert.NotEqual(t, base, SnapshotIdempotencyKey(other, 100)) - }) - - t.Run("differs from a MeterRecord key", func(t *testing.T) { - assert.NotEqual(t, base, IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_RESERVE, "1")) - }) -} - -func TestNewUtilization(t *testing.T) { - id := testIdentity.WithResourceID("filter-1") - u := NewUtilization(id, meteringpb.MeterAction_METER_ACTION_UPDATE, 42, "event-9") - - assert.Equal(t, int64(42), u.GetValue()) - assert.Equal(t, - IdempotencyKey(id, meteringpb.MeterAction_METER_ACTION_UPDATE, "event-9"), - u.GetIdempotencyKey()) -} - -func TestResourceIdentity_WithResourceID(t *testing.T) { - got := testIdentity.WithResourceID("rid-1") - assert.Equal(t, "rid-1", got.ResourceID) - assert.Empty(t, testIdentity.ResourceID, "receiver must be unchanged") - // All other fields are copied through. - assert.Equal(t, testIdentity.Service, got.Service) - assert.Equal(t, testIdentity.NodeID, got.NodeID) + msgs := observer.Messages(t, beholder.AttrKeyEntity, "metering.v1.MeterRecord") + require.Len(t, msgs, 1) } From e4caef5944936acf0fbde2bb532962cb5f11c937 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Fri, 26 Jun 2026 20:11:44 -0400 Subject: [PATCH 04/11] fixing attributes drilled over standard cap boundary to loop boundary --- pkg/beholder/client.go | 22 ++++++++++++++ pkg/loop/config.go | 29 +++++++++++++++++++ pkg/loop/config_test.go | 12 ++++++++ pkg/resourcemanager/meterable.go | 20 +++++++++++++ pkg/resourcemanager/resourcemanager.go | 2 +- .../standard_capabilities_dependencies.go | 24 --------------- 6 files changed, 84 insertions(+), 25 deletions(-) diff --git a/pkg/beholder/client.go b/pkg/beholder/client.go index cf66c39306..45f6810746 100644 --- a/pkg/beholder/client.go +++ b/pkg/beholder/client.go @@ -247,6 +247,11 @@ func NewGRPCClient(cfg Config, otlploggrpcNew otlploggrpcFactory) (*Client, erro } } + // guarantees the CSA key is present on durable meter records + if len(cfg.AuthPublicKeyHex) > 0 { + emitter = csaKeyEmitter{Emitter: emitter, csaKeyHex: cfg.AuthPublicKeyHex} + } + onClose := func() (err error) { for _, provider := range []shutdowner{messageLoggerProvider, loggerProvider, tracerProvider, meterProvider} { err = errors.Join(err, provider.Shutdown(context.Background())) @@ -288,6 +293,23 @@ type noCloseEmitter struct{ Emitter } func (n noCloseEmitter) Close() error { return nil } +// csaKeyEmitter wraps an Emitter to include the node's CSA public key as the +// node_csa_key message attribute on every emitted event. +// +// The csa_public_key is already attached as an OTel resource attribute, but +// resource attributes only ride the OTLP message path. +type csaKeyEmitter struct { + Emitter + csaKeyHex string +} + +func (e csaKeyEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { + merged := make([]any, 0, len(attrKVs)+2) + merged = append(merged, attrKVs...) + merged = append(merged, "node_csa_key", e.csaKeyHex) + return e.Emitter.Emit(ctx, body, merged...) +} + // Returns a new Client with the same configuration but with a different package name // Deprecated: Use ForName func (c Client) ForPackage(name string) Client { diff --git a/pkg/loop/config.go b/pkg/loop/config.go index a0bf626dd2..a38d6921a5 100644 --- a/pkg/loop/config.go +++ b/pkg/loop/config.go @@ -87,6 +87,10 @@ const ( envTelemetryLogCompressor = "CL_TELEMETRY_LOG_COMPRESSOR" envMeterRecordsEnabled = "CL_METER_RECORDS_ENABLED" envMeterSnapshotsEnabled = "CL_METER_SNAPSHOTS_ENABLED" + envMeteringProduct = "CL_METERING_PRODUCT" + envMeteringEnvironment = "CL_METERING_ENVIRONMENT" + envMeteringZone = "CL_METERING_ZONE" + envMeteringNodeID = "CL_METERING_NODE_ID" envChipIngressEndpoint = "CL_CHIP_INGRESS_ENDPOINT" envChipIngressInsecureConnection = "CL_CHIP_INGRESS_INSECURE_CONNECTION" @@ -176,6 +180,22 @@ type EnvConfig struct { MeterRecordsEnabled bool MeterSnapshotsEnabled bool + // MeteringProduct / MeteringEnvironment / MeteringZone / MeteringNodeID are + // the static deployment+node identity dimensions used as coarse + // metering/billing rollup dimensions. They are resolved once from node + // config by the host and delivered to every LOOP plugin over the env, the + // same channel as the metering toggles above (rather than the + // standard-capabilities boundary). Any may be empty if the host did not + // provide it. + // + // MeteringNodeID is the node's logical name (e.g. "clp-cre-wf-zone-a-1"), + // not the CSA public key; the CSA key rides emitted events separately as the + // node_csa_key attribute. + MeteringProduct string + MeteringEnvironment string + MeteringZone string + MeteringNodeID string + TracingEnabled bool TracingCollectorTarget string TracingSamplingRatio float64 @@ -270,6 +290,10 @@ func (e *EnvConfig) AsCmdEnv() (env []string) { add(envTelemetryLogCompressor, e.TelemetryLogCompressor) add(envMeterRecordsEnabled, strconv.FormatBool(e.MeterRecordsEnabled)) add(envMeterSnapshotsEnabled, strconv.FormatBool(e.MeterSnapshotsEnabled)) + add(envMeteringProduct, e.MeteringProduct) + add(envMeteringEnvironment, e.MeteringEnvironment) + add(envMeteringZone, e.MeteringZone) + add(envMeteringNodeID, e.MeteringNodeID) add(envChipIngressEndpoint, e.ChipIngressEndpoint) add(envChipIngressInsecureConnection, strconv.FormatBool(e.ChipIngressInsecureConnection)) @@ -533,6 +557,11 @@ func (e *EnvConfig) parse() error { return fmt.Errorf("failed to parse %s: %w", envMeterSnapshotsEnabled, err) } + e.MeteringProduct = os.Getenv(envMeteringProduct) + e.MeteringEnvironment = os.Getenv(envMeteringEnvironment) + e.MeteringZone = os.Getenv(envMeteringZone) + e.MeteringNodeID = os.Getenv(envMeteringNodeID) + return nil } diff --git a/pkg/loop/config_test.go b/pkg/loop/config_test.go index cf53167da5..a878ce8114 100644 --- a/pkg/loop/config_test.go +++ b/pkg/loop/config_test.go @@ -87,6 +87,10 @@ func TestEnvConfig_parse(t *testing.T) { envTelemetryPrometheusBridgePrefixes: "foo,bar", envMeterRecordsEnabled: "true", envMeterSnapshotsEnabled: "false", + envMeteringProduct: "cre-mainline", + envMeteringEnvironment: "production", + envMeteringZone: "wf-zone-a", + envMeteringNodeID: "csa-pubkey-1", envChipIngressEndpoint: "chip-ingress.example.com:50051", envChipIngressInsecureConnection: "true", @@ -203,6 +207,10 @@ var envCfgFull = EnvConfig{ TelemetryPrometheusBridgePrefixes: []string{"foo", "bar"}, MeterRecordsEnabled: true, MeterSnapshotsEnabled: false, + MeteringProduct: "cre-mainline", + MeteringEnvironment: "production", + MeteringZone: "wf-zone-a", + MeteringNodeID: "csa-pubkey-1", ChipIngressEndpoint: "chip-ingress.example.com:50051", ChipIngressInsecureConnection: true, @@ -270,6 +278,10 @@ func TestEnvConfig_AsCmdEnv(t *testing.T) { assert.Equal(t, "foo,bar", got[envTelemetryPrometheusBridgePrefixes]) assert.Equal(t, "true", got[envMeterRecordsEnabled]) assert.Equal(t, "false", got[envMeterSnapshotsEnabled]) + assert.Equal(t, "cre-mainline", got[envMeteringProduct]) + assert.Equal(t, "production", got[envMeteringEnvironment]) + assert.Equal(t, "wf-zone-a", got[envMeteringZone]) + assert.Equal(t, "csa-pubkey-1", got[envMeteringNodeID]) // Assert ChipIngress environment variables assert.Equal(t, "chip-ingress.example.com:50051", got[envChipIngressEndpoint]) diff --git a/pkg/resourcemanager/meterable.go b/pkg/resourcemanager/meterable.go index 5d39f81f58..e6937daca4 100644 --- a/pkg/resourcemanager/meterable.go +++ b/pkg/resourcemanager/meterable.go @@ -43,6 +43,26 @@ type SnapshotEntry struct { Utilizations []*meteringpb.Utilization } +// DeploymentIdentity carries the static deployment + node identity dimensions +// that are fixed for a LOOP plugin process. They are resolved once from node +// config by the host and delivered to every LOOP plugin over the environment +// (loop.EnvConfig), not the standard-capabilities boundary, so any LOOP plugin +// can populate the coarse metering rollup dimensions. Any field may be empty if +// the host did not provide it. +type DeploymentIdentity struct { + // Product is the deployment product, e.g. "cre-mainline". + Product string + // Environment is the deployment environment, e.g. "production", "staging". + Environment string + // Zone is the deployment zone, e.g. "wf-zone-a". + Zone string + // NodeID is the node's logical name, e.g. "clp-cre-wf-zone-a-1". It is NOT + // the CSA public key; it is a stable name the billing service can use to + // look up the node's CSA key in the workflow registry. The CSA key itself is + // carried separately as the node_csa_key event attribute. + NodeID string +} + // ResourceIdentity is the structured, first-class identity of a durable // resource. Its fields map 1:1 to metering.v1.ResourceIdentity so every // emitted record carries each dimension as a discrete column rather than a diff --git a/pkg/resourcemanager/resourcemanager.go b/pkg/resourcemanager/resourcemanager.go index 977967211c..670d035bd2 100644 --- a/pkg/resourcemanager/resourcemanager.go +++ b/pkg/resourcemanager/resourcemanager.go @@ -45,7 +45,7 @@ import ( // the CHiP schema registration and the consumer topic name; all must match // exactly. const ( - beholderDomain = "platform" + beholderDomain = "cll-meter" beholderEntity = "metering.v1.MeterRecord" beholderDataSchema = "metering.v1.meter_record" beholderSnapshotEntity = "metering.v1.MeterSnapshot" diff --git a/pkg/types/core/standard_capabilities_dependencies.go b/pkg/types/core/standard_capabilities_dependencies.go index 339205d6e8..25add8e6f9 100644 --- a/pkg/types/core/standard_capabilities_dependencies.go +++ b/pkg/types/core/standard_capabilities_dependencies.go @@ -34,28 +34,4 @@ type StandardCapabilitiesDependencies struct { // registry in that case, but the fallback path cannot disambiguate when // the local node belongs to multiple DONs running the same capability. CapabilityDonID uint32 - - // Product is the host-injected deployment product (e.g. "cre-mainline"), - // sourced once from node config by the host and provided before Initialise - // is called. Plugins use it as a coarse metering/billing identity - // dimension. An empty value means the host did not provide one (a legacy - // node or a boot path not yet updated to populate it). - Product string - - // Environment is the host-injected deployment environment (e.g. - // "production", "staging"), provided by the host before Initialise. Plugins - // use it as a coarse metering/billing identity dimension. Empty means the - // host did not provide one. - Environment string - - // Zone is the host-injected deployment zone (e.g. "wf-zone-a"), provided by - // the host before Initialise. Plugins use it as a coarse metering/billing - // identity dimension. Empty means the host did not provide one. - Zone string - - // NodeID is the host-injected node identity: the node's CSA public key - // (hex), provided by the host before Initialise. Plugins use it as a coarse - // metering/billing identity dimension to dedup a node's retries and count - // distinct nodes for quorum. Empty means the host did not provide one. - NodeID string } From 9f0321cb1b7ac581646aedf10df515e83fff90d9 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Fri, 26 Jun 2026 21:45:49 -0400 Subject: [PATCH 05/11] moving csa key wrapper out of beholder package into durable emitter pkg --- pkg/beholder/client.go | 22 ------- pkg/durableemitter/durable_emitter.go | 15 +++++ pkg/durableemitter/durable_emitter_test.go | 75 ++++++++++++++++++++++ pkg/durableemitter/setup.go | 4 ++ 4 files changed, 94 insertions(+), 22 deletions(-) diff --git a/pkg/beholder/client.go b/pkg/beholder/client.go index 45f6810746..cf66c39306 100644 --- a/pkg/beholder/client.go +++ b/pkg/beholder/client.go @@ -247,11 +247,6 @@ func NewGRPCClient(cfg Config, otlploggrpcNew otlploggrpcFactory) (*Client, erro } } - // guarantees the CSA key is present on durable meter records - if len(cfg.AuthPublicKeyHex) > 0 { - emitter = csaKeyEmitter{Emitter: emitter, csaKeyHex: cfg.AuthPublicKeyHex} - } - onClose := func() (err error) { for _, provider := range []shutdowner{messageLoggerProvider, loggerProvider, tracerProvider, meterProvider} { err = errors.Join(err, provider.Shutdown(context.Background())) @@ -293,23 +288,6 @@ type noCloseEmitter struct{ Emitter } func (n noCloseEmitter) Close() error { return nil } -// csaKeyEmitter wraps an Emitter to include the node's CSA public key as the -// node_csa_key message attribute on every emitted event. -// -// The csa_public_key is already attached as an OTel resource attribute, but -// resource attributes only ride the OTLP message path. -type csaKeyEmitter struct { - Emitter - csaKeyHex string -} - -func (e csaKeyEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) error { - merged := make([]any, 0, len(attrKVs)+2) - merged = append(merged, attrKVs...) - merged = append(merged, "node_csa_key", e.csaKeyHex) - return e.Emitter.Emit(ctx, body, merged...) -} - // Returns a new Client with the same configuration but with a different package name // Deprecated: Use ForName func (c Client) ForPackage(name string) Client { diff --git a/pkg/durableemitter/durable_emitter.go b/pkg/durableemitter/durable_emitter.go index c120b08c4d..74b9513f6a 100644 --- a/pkg/durableemitter/durable_emitter.go +++ b/pkg/durableemitter/durable_emitter.go @@ -22,6 +22,12 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" ) +// nodeCSAKeyExtension is the CloudEvent extension name carrying the node's CSA +// public key on durable events. CloudEvents extension names must be lowercase +// alphanumeric, so this is the durable-path equivalent of beholder's +// node_csa_key attribute. +const nodeCSAKeyExtension = "nodecsakey" + // BatchEmitter is the transport interface DurableEmitter delegates to for // batched delivery of CloudEvents to Chip Ingress. // @@ -48,6 +54,10 @@ type BatchEmitter interface { // Config configures the DurableEmitter behaviour. type Config struct { + // NodeCSAKey is the node's CSA public key (hex). When non-empty it is + // stamped as the node_csa_key attribute on every emitted event, providing a + // durable node identity for downstream billing/lookup + NodeCSAKey string // RetransmitInterval controls how often the retransmit loop ticks. RetransmitInterval time.Duration // RetransmitAfter is the minimum age of an event before the retransmit @@ -382,6 +392,11 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) } event.SetExtension("emitter", "DurableEmitter") + // Include the node's CSA public key so every durable event carries a + // durable node identity for downstream billing/lookup + if d.cfg.NodeCSAKey != "" { + event.SetExtension(nodeCSAKeyExtension, d.cfg.NodeCSAKey) + } eventPb, err := chipingress.EventToProto(event) if err != nil { diff --git a/pkg/durableemitter/durable_emitter_test.go b/pkg/durableemitter/durable_emitter_test.go index ac85f2436b..6f9db07504 100644 --- a/pkg/durableemitter/durable_emitter_test.go +++ b/pkg/durableemitter/durable_emitter_test.go @@ -1413,3 +1413,78 @@ func (m *MemDurableEventStore) ObserveDurableQueue(_ context.Context, eventTTL, st.OldestPendingAge = now.Sub(oldest) return st, nil } + +// capturingBatchEmitter records every CloudEvent handed to QueueMessage so tests +// can assert on the emitted event's attributes/extensions. +type capturingBatchEmitter struct { + mu sync.Mutex + events []*chipingress.CloudEventPb +} + +func (b *capturingBatchEmitter) QueueMessage(event *chipingress.CloudEventPb, cb func(error)) error { + b.mu.Lock() + b.events = append(b.events, event) + b.mu.Unlock() + if cb != nil { + cb(nil) + } + return nil +} + +func (b *capturingBatchEmitter) Start(_ context.Context) {} +func (b *capturingBatchEmitter) Stop() {} + +func (b *capturingBatchEmitter) last() *chipingress.CloudEventPb { + b.mu.Lock() + defer b.mu.Unlock() + if len(b.events) == 0 { + return nil + } + return b.events[len(b.events)-1] +} + +func TestDurableEmitter_StampsNodeCSAKey(t *testing.T) { + be := &capturingBatchEmitter{} + + cfg := DefaultConfig() + cfg.DisablePruning = true + cfg.InsertBatchSize = 0 // inline insert for a deterministic single event + cfg.MarkBatchSize = 0 + cfg.NodeCSAKey = "deadbeef" + + em := newTestDurableEmitter(t, NewMemDurableEventStore(), be, &cfg) + require.NoError(t, em.Start(t.Context())) + t.Cleanup(func() { _ = em.Close() }) + + require.NoError(t, em.Emit(t.Context(), []byte("body"), testEmitAttrs()...)) + + require.Eventually(t, func() bool { return be.last() != nil }, 2*time.Second, 10*time.Millisecond) + + ev := be.last() + require.NotNil(t, ev) + attr := ev.GetAttributes()[nodeCSAKeyExtension] + require.NotNil(t, attr, "nodecsakey extension should be present on durable events") + assert.Equal(t, "deadbeef", attr.GetCeString()) +} + +func TestDurableEmitter_OmitsNodeCSAKeyWhenUnset(t *testing.T) { + be := &capturingBatchEmitter{} + + cfg := DefaultConfig() + cfg.DisablePruning = true + cfg.InsertBatchSize = 0 + cfg.MarkBatchSize = 0 + // NodeCSAKey intentionally left empty. + + em := newTestDurableEmitter(t, NewMemDurableEventStore(), be, &cfg) + require.NoError(t, em.Start(t.Context())) + t.Cleanup(func() { _ = em.Close() }) + + require.NoError(t, em.Emit(t.Context(), []byte("body"), testEmitAttrs()...)) + require.Eventually(t, func() bool { return be.last() != nil }, 2*time.Second, 10*time.Millisecond) + + ev := be.last() + require.NotNil(t, ev) + _, ok := ev.GetAttributes()[nodeCSAKeyExtension] + assert.False(t, ok, "nodecsakey extension must be absent when NodeCSAKey is unset") +} diff --git a/pkg/durableemitter/setup.go b/pkg/durableemitter/setup.go index 6cd01fab7c..35a5070802 100644 --- a/pkg/durableemitter/setup.go +++ b/pkg/durableemitter/setup.go @@ -133,6 +133,10 @@ func Setup( emitterCfg = *cfg.EmitterConfig } + if emitterCfg.NodeCSAKey == "" { + emitterCfg.NodeCSAKey = cfg.Auth.AuthPublicKeyHex + } + emitter, err := NewDurableEmitter(store, batchClient, fallbackClient, cfg.RetransmitEnabled, emitterCfg, lggr, cfg.Meter) if err != nil { batchClient.Stop() From 57e99bbc7b44f39c8864619b0eb62772f85581cb Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Wed, 1 Jul 2026 15:18:27 -0400 Subject: [PATCH 06/11] removing labels, and adapting to proto don struct changes --- go.mod | 2 +- go.sum | 4 +- pkg/loop/config.go | 7 ++- pkg/loop/config_test.go | 3 + pkg/resourcemanager/labels.go | 15 ----- pkg/resourcemanager/labels_test.go | 30 --------- pkg/resourcemanager/meterable.go | 69 ++++++++++++++------- pkg/resourcemanager/resourcemanager.go | 5 +- pkg/resourcemanager/resourcemanager_test.go | 10 ++- 9 files changed, 67 insertions(+), 78 deletions(-) delete mode 100644 pkg/resourcemanager/labels.go delete mode 100644 pkg/resourcemanager/labels_test.go diff --git a/go.mod b/go.mod index 629f9c302c..d4f68f6389 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260626162307-b56f9af1d196 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260701185448-696c075849ea github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 diff --git a/go.sum b/go.sum index bd2c8328ef..2fddfe8305 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,8 @@ github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260626162307-b56f9af1d196 h1:5SZtASpkbCyK7xtxqiUNwLxYEt7rXE6mkjZKecve2yY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260626162307-b56f9af1d196/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260701185448-696c075849ea h1:oo9Ir4TU9/cG8R0bICcVeprEE+7F31Ew0i7CnI4p4EA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260701185448-696c075849ea/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 h1:B7itmjy+CMJ26elVw/cAJqqhBQ3Xa/mBYWK0/rQ5MuI= diff --git a/pkg/loop/config.go b/pkg/loop/config.go index a38d6921a5..b7b7a96e10 100644 --- a/pkg/loop/config.go +++ b/pkg/loop/config.go @@ -88,6 +88,7 @@ const ( envMeterRecordsEnabled = "CL_METER_RECORDS_ENABLED" envMeterSnapshotsEnabled = "CL_METER_SNAPSHOTS_ENABLED" envMeteringProduct = "CL_METERING_PRODUCT" + envMeteringTenant = "CL_METERING_TENANT" envMeteringEnvironment = "CL_METERING_ENVIRONMENT" envMeteringZone = "CL_METERING_ZONE" envMeteringNodeID = "CL_METERING_NODE_ID" @@ -180,7 +181,8 @@ type EnvConfig struct { MeterRecordsEnabled bool MeterSnapshotsEnabled bool - // MeteringProduct / MeteringEnvironment / MeteringZone / MeteringNodeID are + // MeteringProduct / MeteringTenant / MeteringEnvironment / MeteringZone / + // MeteringNodeID are // the static deployment+node identity dimensions used as coarse // metering/billing rollup dimensions. They are resolved once from node // config by the host and delivered to every LOOP plugin over the env, the @@ -192,6 +194,7 @@ type EnvConfig struct { // not the CSA public key; the CSA key rides emitted events separately as the // node_csa_key attribute. MeteringProduct string + MeteringTenant string MeteringEnvironment string MeteringZone string MeteringNodeID string @@ -291,6 +294,7 @@ func (e *EnvConfig) AsCmdEnv() (env []string) { add(envMeterRecordsEnabled, strconv.FormatBool(e.MeterRecordsEnabled)) add(envMeterSnapshotsEnabled, strconv.FormatBool(e.MeterSnapshotsEnabled)) add(envMeteringProduct, e.MeteringProduct) + add(envMeteringTenant, e.MeteringTenant) add(envMeteringEnvironment, e.MeteringEnvironment) add(envMeteringZone, e.MeteringZone) add(envMeteringNodeID, e.MeteringNodeID) @@ -558,6 +562,7 @@ func (e *EnvConfig) parse() error { } e.MeteringProduct = os.Getenv(envMeteringProduct) + e.MeteringTenant = os.Getenv(envMeteringTenant) e.MeteringEnvironment = os.Getenv(envMeteringEnvironment) e.MeteringZone = os.Getenv(envMeteringZone) e.MeteringNodeID = os.Getenv(envMeteringNodeID) diff --git a/pkg/loop/config_test.go b/pkg/loop/config_test.go index a878ce8114..4df3ca23e8 100644 --- a/pkg/loop/config_test.go +++ b/pkg/loop/config_test.go @@ -88,6 +88,7 @@ func TestEnvConfig_parse(t *testing.T) { envMeterRecordsEnabled: "true", envMeterSnapshotsEnabled: "false", envMeteringProduct: "cre-mainline", + envMeteringTenant: "mainline", envMeteringEnvironment: "production", envMeteringZone: "wf-zone-a", envMeteringNodeID: "csa-pubkey-1", @@ -208,6 +209,7 @@ var envCfgFull = EnvConfig{ MeterRecordsEnabled: true, MeterSnapshotsEnabled: false, MeteringProduct: "cre-mainline", + MeteringTenant: "mainline", MeteringEnvironment: "production", MeteringZone: "wf-zone-a", MeteringNodeID: "csa-pubkey-1", @@ -279,6 +281,7 @@ func TestEnvConfig_AsCmdEnv(t *testing.T) { assert.Equal(t, "true", got[envMeterRecordsEnabled]) assert.Equal(t, "false", got[envMeterSnapshotsEnabled]) assert.Equal(t, "cre-mainline", got[envMeteringProduct]) + assert.Equal(t, "mainline", got[envMeteringTenant]) assert.Equal(t, "production", got[envMeteringEnvironment]) assert.Equal(t, "wf-zone-a", got[envMeteringZone]) assert.Equal(t, "csa-pubkey-1", got[envMeteringNodeID]) diff --git a/pkg/resourcemanager/labels.go b/pkg/resourcemanager/labels.go deleted file mode 100644 index bb8b921cd3..0000000000 --- a/pkg/resourcemanager/labels.go +++ /dev/null @@ -1,15 +0,0 @@ -package resourcemanager - -import "strings" - -// OwnerLabel returns the canonical form of the "owner" label, used by all -// producers: a leading "0x"/"0X" prefix is stripped and the remainder is -// lowercased. Downstream joins on the owner label depend on every producer -// emitting exactly this form, so producers must not write the owner label -// any other way. -func OwnerLabel(owner string) string { - if strings.HasPrefix(owner, "0x") || strings.HasPrefix(owner, "0X") { - owner = owner[2:] - } - return strings.ToLower(owner) -} diff --git a/pkg/resourcemanager/labels_test.go b/pkg/resourcemanager/labels_test.go deleted file mode 100644 index 5581250493..0000000000 --- a/pkg/resourcemanager/labels_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package resourcemanager - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestOwnerLabel(t *testing.T) { - tests := []struct { - name string - owner string - want string - }{ - {name: "lowercase 0x prefix stripped", owner: "0xabc123def", want: "abc123def"}, - {name: "uppercase 0X prefix stripped", owner: "0Xabc123def", want: "abc123def"}, - {name: "no prefix unchanged", owner: "abc123def", want: "abc123def"}, - {name: "mixed case lowercased", owner: "0xAbC123dEF", want: "abc123def"}, - {name: "mixed case without prefix lowercased", owner: "AbC123dEF", want: "abc123def"}, - {name: "empty string", owner: "", want: ""}, - {name: "prefix only", owner: "0x", want: ""}, - {name: "only one prefix stripped", owner: "0x0Xabc", want: "0xabc"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, OwnerLabel(tt.owner)) - }) - } -} diff --git a/pkg/resourcemanager/meterable.go b/pkg/resourcemanager/meterable.go index e6937daca4..d94f3d83f5 100644 --- a/pkg/resourcemanager/meterable.go +++ b/pkg/resourcemanager/meterable.go @@ -4,7 +4,6 @@ import ( "context" meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" - "google.golang.org/protobuf/proto" ) // Meterable is implemented by producers that manage durable billable @@ -14,8 +13,8 @@ import ( // active resources, in addition to emitting lifecycle edges inline via // EmitMeterRecord. type Meterable interface { - // ResourceIdentity returns the producer's base identity: the six coarse - // dimensions (product, environment, zone, don_id, node_id, service) plus + // ResourceIdentity returns the producer's base identity: the coarse + // dimensions (product, tenant, environment, zone, don_identifier, service) plus // the service-level resource_pool / resource_pool_id. Per-resource billing // fields (resource_type/resource_id/org_id/event_id/value) are carried by // Utilizations on MeterRecord and MeterSnapshot. @@ -50,8 +49,10 @@ type SnapshotEntry struct { // can populate the coarse metering rollup dimensions. Any field may be empty if // the host did not provide it. type DeploymentIdentity struct { - // Product is the deployment product, e.g. "cre-mainline". + // Product is the deployment product, e.g. "cre". Product string + // Tenant is the deployment tenant, e.g. "mainline" or "enterprise". + Tenant string // Environment is the deployment environment, e.g. "production", "staging". Environment string // Zone is the deployment zone, e.g. "wf-zone-a". @@ -63,36 +64,39 @@ type DeploymentIdentity struct { NodeID string } +// DonIdentifier captures DON-specific identity dimensions as one unit. +type DonIdentifier struct { + // DonID is the DON identifier the emitting service belongs to. + DonID string + // NodeID is the node identifier (the node's CSA public key). + NodeID string +} + // ResourceIdentity is the structured, first-class identity of a durable // resource. Its fields map 1:1 to metering.v1.ResourceIdentity so every // emitted record carries each dimension as a discrete column rather than a // parsed dotted string or out-of-band telemetry attribute. type ResourceIdentity struct { - // Product is the deployment product, e.g. "cre-mainline". A coarse - // billing-rollup dimension. + // Product is the deployment product, e.g. "cre". Product string + // Tenant is the deployment tenant, e.g. "mainline" or "enterprise". + Tenant string + // Environment is the deployment environment, e.g. "production", - // "staging". A coarse billing-rollup dimension. + // "staging". Environment string - // Zone is the deployment zone, e.g. "wf-zone-a". A coarse billing-rollup - // dimension. + // Zone is the deployment zone, e.g. "wf-zone-a". Zone string - // DONID is the DON identifier the emitting service belongs to. A coarse - // billing-rollup dimension; used with NodeID to count distinct nodes for - // quorum. - DONID string - - // NodeID is the node identifier (the node's CSA public key). A coarse - // billing-rollup dimension; lets billing dedup a node's retries and count - // distinct nodes. - NodeID string + // DonIdentifier groups DON-specific identity dimensions so consumers can + // branch on one struct instead of handling don/node permutations. + DonIdentifier *DonIdentifier // Service is the stable service constant identifying the emitting service, // e.g. "cron-trigger", "http-trigger", "evm-log-trigger", - // "workflow-syncer-v2". A coarse billing-rollup dimension. It must not + // "workflow-syncer-v2". It must not // encode deployment environment or zone. Service string @@ -108,17 +112,34 @@ type ResourceIdentity struct { func (r ResourceIdentity) toProto() *meteringpb.ResourceIdentity { pb := &meteringpb.ResourceIdentity{ Product: r.Product, + Tenant: r.Tenant, Environment: r.Environment, Zone: r.Zone, Service: r.Service, ResourcePool: r.ResourcePool, ResourcePoolId: r.ResourcePoolID, } - if r.DONID != "" { - pb.DonId = proto.String(r.DONID) - } - if r.NodeID != "" { - pb.NodeId = proto.String(r.NodeID) + if r.DonIdentifier != nil { + pb.DonIdentifier = &meteringpb.DonIdentifier{ + DonId: r.DonIdentifier.DonID, + NodeId: r.DonIdentifier.NodeID, + } } return pb } + +// DonID returns the DON identifier when present. +func (r ResourceIdentity) DonID() string { + if r.DonIdentifier == nil { + return "" + } + return r.DonIdentifier.DonID +} + +// NodeID returns the node identifier when present. +func (r ResourceIdentity) NodeID() string { + if r.DonIdentifier == nil { + return "" + } + return r.DonIdentifier.NodeID +} diff --git a/pkg/resourcemanager/resourcemanager.go b/pkg/resourcemanager/resourcemanager.go index 670d035bd2..2f80bd5282 100644 --- a/pkg/resourcemanager/resourcemanager.go +++ b/pkg/resourcemanager/resourcemanager.go @@ -416,10 +416,11 @@ func (rm *ResourceManager) recordUtilization(ctx context.Context, id ResourceIde } rm.utilization.Record(ctx, value, metric.WithAttributes( attribute.String("product", id.Product), + attribute.String("tenant", id.Tenant), attribute.String("environment", id.Environment), attribute.String("zone", id.Zone), - attribute.String("don_id", id.DONID), - attribute.String("node_id", id.NodeID), + attribute.String("don_id", id.DonID()), + attribute.String("node_id", id.NodeID()), attribute.String("service", id.Service), attribute.String("resource_pool", id.ResourcePool), attribute.String("resource_pool_id", id.ResourcePoolID), diff --git a/pkg/resourcemanager/resourcemanager_test.go b/pkg/resourcemanager/resourcemanager_test.go index b0ab380be1..16b9ecc915 100644 --- a/pkg/resourcemanager/resourcemanager_test.go +++ b/pkg/resourcemanager/resourcemanager_test.go @@ -22,10 +22,10 @@ import ( var testIdentity = ResourceIdentity{ Product: "cre-mainline", + Tenant: "mainline", Environment: "production", Zone: "wf-zone-a", - DONID: "don-1", - NodeID: "node-1", + DonIdentifier: &DonIdentifier{DonID: "don-1", NodeID: "node-1"}, Service: "cron-trigger", ResourcePool: "trigger_registrations", ResourcePoolID: "", @@ -149,12 +149,16 @@ func TestEmitMeterRecord_Success(t *testing.T) { call := emitter.calls[0] attrs := attrMap(t, call.attrKVs) assert.Equal(t, "metering.v1.meter_record", attrs[beholder.AttrKeyDataSchema]) - assert.Equal(t, "platform", attrs[beholder.AttrKeyDomain]) + assert.Equal(t, "cll-meter", attrs[beholder.AttrKeyDomain]) assert.Equal(t, "metering.v1.MeterRecord", attrs[beholder.AttrKeyEntity]) var record meteringpb.MeterRecord require.NoError(t, proto.Unmarshal(call.body, &record)) require.NotNil(t, record.GetIdentity()) + assert.Equal(t, testIdentity.Product, record.GetIdentity().GetProduct()) + assert.Equal(t, testIdentity.Tenant, record.GetIdentity().GetTenant()) + assert.Equal(t, testIdentity.DonID(), record.GetIdentity().GetDonIdentifier().GetDonId()) + assert.Equal(t, testIdentity.NodeID(), record.GetIdentity().GetDonIdentifier().GetNodeId()) assert.Equal(t, testIdentity.ResourcePool, record.GetIdentity().GetResourcePool()) assert.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, record.GetAction()) require.Len(t, record.GetUtilizations(), 1) From 6e4f527400deb44d2f214a9634c059d8c2e6e8e2 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Wed, 1 Jul 2026 16:12:40 -0400 Subject: [PATCH 07/11] removing unecessary csa key extension --- pkg/durableemitter/durable_emitter.go | 15 ---- pkg/durableemitter/durable_emitter_test.go | 75 ------------------- pkg/durableemitter/setup.go | 4 - .../{idempotency.go => utilization.go} | 0 4 files changed, 94 deletions(-) rename pkg/resourcemanager/{idempotency.go => utilization.go} (100%) diff --git a/pkg/durableemitter/durable_emitter.go b/pkg/durableemitter/durable_emitter.go index 74b9513f6a..c120b08c4d 100644 --- a/pkg/durableemitter/durable_emitter.go +++ b/pkg/durableemitter/durable_emitter.go @@ -22,12 +22,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" ) -// nodeCSAKeyExtension is the CloudEvent extension name carrying the node's CSA -// public key on durable events. CloudEvents extension names must be lowercase -// alphanumeric, so this is the durable-path equivalent of beholder's -// node_csa_key attribute. -const nodeCSAKeyExtension = "nodecsakey" - // BatchEmitter is the transport interface DurableEmitter delegates to for // batched delivery of CloudEvents to Chip Ingress. // @@ -54,10 +48,6 @@ type BatchEmitter interface { // Config configures the DurableEmitter behaviour. type Config struct { - // NodeCSAKey is the node's CSA public key (hex). When non-empty it is - // stamped as the node_csa_key attribute on every emitted event, providing a - // durable node identity for downstream billing/lookup - NodeCSAKey string // RetransmitInterval controls how often the retransmit loop ticks. RetransmitInterval time.Duration // RetransmitAfter is the minimum age of an event before the retransmit @@ -392,11 +382,6 @@ func (d *DurableEmitter) Emit(ctx context.Context, body []byte, attrKVs ...any) } event.SetExtension("emitter", "DurableEmitter") - // Include the node's CSA public key so every durable event carries a - // durable node identity for downstream billing/lookup - if d.cfg.NodeCSAKey != "" { - event.SetExtension(nodeCSAKeyExtension, d.cfg.NodeCSAKey) - } eventPb, err := chipingress.EventToProto(event) if err != nil { diff --git a/pkg/durableemitter/durable_emitter_test.go b/pkg/durableemitter/durable_emitter_test.go index 6f9db07504..ac85f2436b 100644 --- a/pkg/durableemitter/durable_emitter_test.go +++ b/pkg/durableemitter/durable_emitter_test.go @@ -1413,78 +1413,3 @@ func (m *MemDurableEventStore) ObserveDurableQueue(_ context.Context, eventTTL, st.OldestPendingAge = now.Sub(oldest) return st, nil } - -// capturingBatchEmitter records every CloudEvent handed to QueueMessage so tests -// can assert on the emitted event's attributes/extensions. -type capturingBatchEmitter struct { - mu sync.Mutex - events []*chipingress.CloudEventPb -} - -func (b *capturingBatchEmitter) QueueMessage(event *chipingress.CloudEventPb, cb func(error)) error { - b.mu.Lock() - b.events = append(b.events, event) - b.mu.Unlock() - if cb != nil { - cb(nil) - } - return nil -} - -func (b *capturingBatchEmitter) Start(_ context.Context) {} -func (b *capturingBatchEmitter) Stop() {} - -func (b *capturingBatchEmitter) last() *chipingress.CloudEventPb { - b.mu.Lock() - defer b.mu.Unlock() - if len(b.events) == 0 { - return nil - } - return b.events[len(b.events)-1] -} - -func TestDurableEmitter_StampsNodeCSAKey(t *testing.T) { - be := &capturingBatchEmitter{} - - cfg := DefaultConfig() - cfg.DisablePruning = true - cfg.InsertBatchSize = 0 // inline insert for a deterministic single event - cfg.MarkBatchSize = 0 - cfg.NodeCSAKey = "deadbeef" - - em := newTestDurableEmitter(t, NewMemDurableEventStore(), be, &cfg) - require.NoError(t, em.Start(t.Context())) - t.Cleanup(func() { _ = em.Close() }) - - require.NoError(t, em.Emit(t.Context(), []byte("body"), testEmitAttrs()...)) - - require.Eventually(t, func() bool { return be.last() != nil }, 2*time.Second, 10*time.Millisecond) - - ev := be.last() - require.NotNil(t, ev) - attr := ev.GetAttributes()[nodeCSAKeyExtension] - require.NotNil(t, attr, "nodecsakey extension should be present on durable events") - assert.Equal(t, "deadbeef", attr.GetCeString()) -} - -func TestDurableEmitter_OmitsNodeCSAKeyWhenUnset(t *testing.T) { - be := &capturingBatchEmitter{} - - cfg := DefaultConfig() - cfg.DisablePruning = true - cfg.InsertBatchSize = 0 - cfg.MarkBatchSize = 0 - // NodeCSAKey intentionally left empty. - - em := newTestDurableEmitter(t, NewMemDurableEventStore(), be, &cfg) - require.NoError(t, em.Start(t.Context())) - t.Cleanup(func() { _ = em.Close() }) - - require.NoError(t, em.Emit(t.Context(), []byte("body"), testEmitAttrs()...)) - require.Eventually(t, func() bool { return be.last() != nil }, 2*time.Second, 10*time.Millisecond) - - ev := be.last() - require.NotNil(t, ev) - _, ok := ev.GetAttributes()[nodeCSAKeyExtension] - assert.False(t, ok, "nodecsakey extension must be absent when NodeCSAKey is unset") -} diff --git a/pkg/durableemitter/setup.go b/pkg/durableemitter/setup.go index 35a5070802..6cd01fab7c 100644 --- a/pkg/durableemitter/setup.go +++ b/pkg/durableemitter/setup.go @@ -133,10 +133,6 @@ func Setup( emitterCfg = *cfg.EmitterConfig } - if emitterCfg.NodeCSAKey == "" { - emitterCfg.NodeCSAKey = cfg.Auth.AuthPublicKeyHex - } - emitter, err := NewDurableEmitter(store, batchClient, fallbackClient, cfg.RetransmitEnabled, emitterCfg, lggr, cfg.Meter) if err != nil { batchClient.Stop() diff --git a/pkg/resourcemanager/idempotency.go b/pkg/resourcemanager/utilization.go similarity index 100% rename from pkg/resourcemanager/idempotency.go rename to pkg/resourcemanager/utilization.go From 914c53414367c686ffc62e03619e23c201569bae Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Wed, 1 Jul 2026 16:38:45 -0400 Subject: [PATCH 08/11] make generate --- go.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go.md b/go.md index 01604721a7..88a7c3f475 100644 --- a/go.md +++ b/go.md @@ -9,6 +9,7 @@ flowchart LR chainlink-common --> chainlink-protos/billing/go chainlink-common --> chainlink-protos/cre/go chainlink-common --> chainlink-protos/linking-service/go + chainlink-common --> chainlink-protos/metering/go chainlink-common --> chainlink-protos/node-platform chainlink-common --> chainlink-protos/storage-service chainlink-common --> freeport @@ -31,6 +32,8 @@ flowchart LR click chainlink-protos/cre/go href "https://github.com/smartcontractkit/chainlink-protos" chainlink-protos/linking-service/go click chainlink-protos/linking-service/go href "https://github.com/smartcontractkit/chainlink-protos" + chainlink-protos/metering/go + click chainlink-protos/metering/go href "https://github.com/smartcontractkit/chainlink-protos" chainlink-protos/node-platform click chainlink-protos/node-platform href "https://github.com/smartcontractkit/chainlink-protos" chainlink-protos/storage-service @@ -66,6 +69,7 @@ flowchart LR chainlink-protos/billing/go chainlink-protos/cre/go chainlink-protos/linking-service/go + chainlink-protos/metering/go chainlink-protos/node-platform chainlink-protos/storage-service chainlink-protos/workflows/go From 75cd8608bbf03907a12f69bd80eba2232904f43a Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Wed, 1 Jul 2026 21:15:47 -0400 Subject: [PATCH 09/11] simplifying naming; one formatting fix to avoid scientific notation --- pkg/loop/config.go | 47 ++++++++++----------- pkg/loop/config_test.go | 30 ++++++------- pkg/resourcemanager/meterable.go | 6 ++- pkg/resourcemanager/resourcemanager.go | 24 +++++------ pkg/resourcemanager/resourcemanager_test.go | 16 +++---- pkg/resourcemanager/utilization.go | 22 +++++----- 6 files changed, 74 insertions(+), 71 deletions(-) diff --git a/pkg/loop/config.go b/pkg/loop/config.go index b7b7a96e10..23c442c59f 100644 --- a/pkg/loop/config.go +++ b/pkg/loop/config.go @@ -87,11 +87,11 @@ const ( envTelemetryLogCompressor = "CL_TELEMETRY_LOG_COMPRESSOR" envMeterRecordsEnabled = "CL_METER_RECORDS_ENABLED" envMeterSnapshotsEnabled = "CL_METER_SNAPSHOTS_ENABLED" - envMeteringProduct = "CL_METERING_PRODUCT" - envMeteringTenant = "CL_METERING_TENANT" - envMeteringEnvironment = "CL_METERING_ENVIRONMENT" - envMeteringZone = "CL_METERING_ZONE" - envMeteringNodeID = "CL_METERING_NODE_ID" + envMeterProduct = "CL_METER_PRODUCT" + envMeterTenant = "CL_METER_TENANT" + envMeterEnvironment = "CL_METER_ENVIRONMENT" + envMeterZone = "CL_METER_ZONE" + envMeterNodeID = "CL_METER_NODE_ID" envChipIngressEndpoint = "CL_CHIP_INGRESS_ENDPOINT" envChipIngressInsecureConnection = "CL_CHIP_INGRESS_INSECURE_CONNECTION" @@ -181,23 +181,22 @@ type EnvConfig struct { MeterRecordsEnabled bool MeterSnapshotsEnabled bool - // MeteringProduct / MeteringTenant / MeteringEnvironment / MeteringZone / - // MeteringNodeID are + // MeterProduct / MeterTenant / MeterEnvironment / MeterZone / MeterNodeID are // the static deployment+node identity dimensions used as coarse // metering/billing rollup dimensions. They are resolved once from node // config by the host and delivered to every LOOP plugin over the env, the - // same channel as the metering toggles above (rather than the + // same channel as the meter-record toggles above (rather than the // standard-capabilities boundary). Any may be empty if the host did not // provide it. // - // MeteringNodeID is the node's logical name (e.g. "clp-cre-wf-zone-a-1"), + // MeterNodeID is the node's logical name (e.g. "clp-cre-wf-zone-a-1"), // not the CSA public key; the CSA key rides emitted events separately as the // node_csa_key attribute. - MeteringProduct string - MeteringTenant string - MeteringEnvironment string - MeteringZone string - MeteringNodeID string + MeterProduct string + MeterTenant string + MeterEnvironment string + MeterZone string + MeterNodeID string TracingEnabled bool TracingCollectorTarget string @@ -293,11 +292,11 @@ func (e *EnvConfig) AsCmdEnv() (env []string) { add(envTelemetryLogCompressor, e.TelemetryLogCompressor) add(envMeterRecordsEnabled, strconv.FormatBool(e.MeterRecordsEnabled)) add(envMeterSnapshotsEnabled, strconv.FormatBool(e.MeterSnapshotsEnabled)) - add(envMeteringProduct, e.MeteringProduct) - add(envMeteringTenant, e.MeteringTenant) - add(envMeteringEnvironment, e.MeteringEnvironment) - add(envMeteringZone, e.MeteringZone) - add(envMeteringNodeID, e.MeteringNodeID) + add(envMeterProduct, e.MeterProduct) + add(envMeterTenant, e.MeterTenant) + add(envMeterEnvironment, e.MeterEnvironment) + add(envMeterZone, e.MeterZone) + add(envMeterNodeID, e.MeterNodeID) add(envChipIngressEndpoint, e.ChipIngressEndpoint) add(envChipIngressInsecureConnection, strconv.FormatBool(e.ChipIngressInsecureConnection)) @@ -561,11 +560,11 @@ func (e *EnvConfig) parse() error { return fmt.Errorf("failed to parse %s: %w", envMeterSnapshotsEnabled, err) } - e.MeteringProduct = os.Getenv(envMeteringProduct) - e.MeteringTenant = os.Getenv(envMeteringTenant) - e.MeteringEnvironment = os.Getenv(envMeteringEnvironment) - e.MeteringZone = os.Getenv(envMeteringZone) - e.MeteringNodeID = os.Getenv(envMeteringNodeID) + e.MeterProduct = os.Getenv(envMeterProduct) + e.MeterTenant = os.Getenv(envMeterTenant) + e.MeterEnvironment = os.Getenv(envMeterEnvironment) + e.MeterZone = os.Getenv(envMeterZone) + e.MeterNodeID = os.Getenv(envMeterNodeID) return nil } diff --git a/pkg/loop/config_test.go b/pkg/loop/config_test.go index 4df3ca23e8..0fece443b6 100644 --- a/pkg/loop/config_test.go +++ b/pkg/loop/config_test.go @@ -87,11 +87,11 @@ func TestEnvConfig_parse(t *testing.T) { envTelemetryPrometheusBridgePrefixes: "foo,bar", envMeterRecordsEnabled: "true", envMeterSnapshotsEnabled: "false", - envMeteringProduct: "cre-mainline", - envMeteringTenant: "mainline", - envMeteringEnvironment: "production", - envMeteringZone: "wf-zone-a", - envMeteringNodeID: "csa-pubkey-1", + envMeterProduct: "cre-mainline", + envMeterTenant: "mainline", + envMeterEnvironment: "production", + envMeterZone: "wf-zone-a", + envMeterNodeID: "csa-pubkey-1", envChipIngressEndpoint: "chip-ingress.example.com:50051", envChipIngressInsecureConnection: "true", @@ -208,11 +208,11 @@ var envCfgFull = EnvConfig{ TelemetryPrometheusBridgePrefixes: []string{"foo", "bar"}, MeterRecordsEnabled: true, MeterSnapshotsEnabled: false, - MeteringProduct: "cre-mainline", - MeteringTenant: "mainline", - MeteringEnvironment: "production", - MeteringZone: "wf-zone-a", - MeteringNodeID: "csa-pubkey-1", + MeterProduct: "cre-mainline", + MeterTenant: "mainline", + MeterEnvironment: "production", + MeterZone: "wf-zone-a", + MeterNodeID: "csa-pubkey-1", ChipIngressEndpoint: "chip-ingress.example.com:50051", ChipIngressInsecureConnection: true, @@ -280,11 +280,11 @@ func TestEnvConfig_AsCmdEnv(t *testing.T) { assert.Equal(t, "foo,bar", got[envTelemetryPrometheusBridgePrefixes]) assert.Equal(t, "true", got[envMeterRecordsEnabled]) assert.Equal(t, "false", got[envMeterSnapshotsEnabled]) - assert.Equal(t, "cre-mainline", got[envMeteringProduct]) - assert.Equal(t, "mainline", got[envMeteringTenant]) - assert.Equal(t, "production", got[envMeteringEnvironment]) - assert.Equal(t, "wf-zone-a", got[envMeteringZone]) - assert.Equal(t, "csa-pubkey-1", got[envMeteringNodeID]) + assert.Equal(t, "cre-mainline", got[envMeterProduct]) + assert.Equal(t, "mainline", got[envMeterTenant]) + assert.Equal(t, "production", got[envMeterEnvironment]) + assert.Equal(t, "wf-zone-a", got[envMeterZone]) + assert.Equal(t, "csa-pubkey-1", got[envMeterNodeID]) // Assert ChipIngress environment variables assert.Equal(t, "chip-ingress.example.com:50051", got[envChipIngressEndpoint]) diff --git a/pkg/resourcemanager/meterable.go b/pkg/resourcemanager/meterable.go index d94f3d83f5..37cfee8831 100644 --- a/pkg/resourcemanager/meterable.go +++ b/pkg/resourcemanager/meterable.go @@ -68,7 +68,11 @@ type DeploymentIdentity struct { type DonIdentifier struct { // DonID is the DON identifier the emitting service belongs to. DonID string - // NodeID is the node identifier (the node's CSA public key). + // NodeID is the node's logical name within the scope of the DON, e.g. + // "clp-cre-wf-zone-a-1". It is a human-readable identifier, NOT the CSA + // public key. The prefix can be redundant with other fully-qualified + // dimensions, but helps readability. The CSA key is emitted separately via + // the node_csa_key attribute. NodeID string } diff --git a/pkg/resourcemanager/resourcemanager.go b/pkg/resourcemanager/resourcemanager.go index 2f80bd5282..170679b87f 100644 --- a/pkg/resourcemanager/resourcemanager.go +++ b/pkg/resourcemanager/resourcemanager.go @@ -41,15 +41,15 @@ import ( meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" ) -// Beholder routing attributes. Each entity value is the contract shared with +// Beholder/ChIP routing attributes. Each entity value is the contract shared with // the CHiP schema registration and the consumer topic name; all must match // exactly. const ( - beholderDomain = "cll-meter" - beholderEntity = "metering.v1.MeterRecord" - beholderDataSchema = "metering.v1.meter_record" - beholderSnapshotEntity = "metering.v1.MeterSnapshot" - beholderSnapshotDataSchema = "metering.v1.meter_snapshot" + domain = "cll-meter" + entity = "metering.v1.MeterRecord" + dataSchema = "metering.v1.meter_record" + snapshotEntity = "metering.v1.MeterSnapshot" + snapshotDataSchema = "metering.v1.meter_snapshot" ) // Counter names are a dashboard contract; do not rename. @@ -307,9 +307,9 @@ func (rm *ResourceManager) emitSnapshot(ctx context.Context, m Meterable) { } if err := rm.emitter.Emit(ctx, body, - beholder.AttrKeyDataSchema, beholderSnapshotDataSchema, - beholder.AttrKeyDomain, beholderDomain, - beholder.AttrKeyEntity, beholderSnapshotEntity, + beholder.AttrKeyDataSchema, snapshotDataSchema, + beholder.AttrKeyDomain, domain, + beholder.AttrKeyEntity, snapshotEntity, ); err != nil { rm.lggr.Errorw("failed to emit snapshot", "service", e.Identity.Service, @@ -370,9 +370,9 @@ func (rm *ResourceManager) EmitMeterRecord(ctx context.Context, identity Resourc } if err := rm.emitter.Emit(ctx, body, - beholder.AttrKeyDataSchema, beholderDataSchema, - beholder.AttrKeyDomain, beholderDomain, - beholder.AttrKeyEntity, beholderEntity, + beholder.AttrKeyDataSchema, dataSchema, + beholder.AttrKeyDomain, domain, + beholder.AttrKeyEntity, entity, ); err != nil { rm.lggr.Errorw("failed to emit meter record", "service", identity.Service, diff --git a/pkg/resourcemanager/resourcemanager_test.go b/pkg/resourcemanager/resourcemanager_test.go index 16b9ecc915..56377d4497 100644 --- a/pkg/resourcemanager/resourcemanager_test.go +++ b/pkg/resourcemanager/resourcemanager_test.go @@ -118,7 +118,7 @@ func TestEmitMeterRecord_Gating(t *testing.T) { cfg.Emitter = tt.emitter } rm := NewResourceManager(logger.Test(t), cfg) - u := NewUtilization(1, UtilizationFields{ + u := NewUtilizationInt(1, UtilizationFields{ ResourceType: "operations", ResourceID: "trigger-1", EventID: "event-1", @@ -137,7 +137,7 @@ func TestEmitMeterRecord_Success(t *testing.T) { Emitter: emitter, }) - u := NewUtilization(1, UtilizationFields{ + u := NewUtilizationInt(1, UtilizationFields{ ResourceType: "operations", ResourceID: "trigger-1", EventID: "event-1", @@ -176,7 +176,7 @@ func TestEmitMeterRecord_EmitFailureIsSwallowed(t *testing.T) { Emitter: emitter, }) - u := NewUtilization(1, UtilizationFields{ + u := NewUtilizationInt(1, UtilizationFields{ ResourceType: "operations", ResourceID: "trigger-1", EventID: "event-2", @@ -219,7 +219,7 @@ func TestEmitSnapshot_OnePerResource(t *testing.T) { { Identity: testIdentity, Utilizations: []*meteringpb.Utilization{ - NewUtilization(3, UtilizationFields{ + NewUtilizationInt(3, UtilizationFields{ ResourceType: "operations", ResourceID: "trigger-1", OrgID: "org-1", @@ -229,7 +229,7 @@ func TestEmitSnapshot_OnePerResource(t *testing.T) { { Identity: testIdentity, Utilizations: []*meteringpb.Utilization{ - NewUtilization(5, UtilizationFields{ + NewUtilizationInt(5, UtilizationFields{ ResourceType: "operations", ResourceID: "trigger-2", OrgID: "org-2", @@ -274,7 +274,7 @@ func TestSnapshotsCannotRunWithoutRecords(t *testing.T) { m.set([]SnapshotEntry{{ Identity: testIdentity, Utilizations: []*meteringpb.Utilization{ - NewUtilization(1, UtilizationFields{ + NewUtilizationInt(1, UtilizationFields{ ResourceType: "operations", ResourceID: "trigger-1", }), @@ -293,7 +293,7 @@ func TestNewUtilizationVariants(t *testing.T) { OrgID: "org-1", } - uInt := NewUtilization(42, fields) + uInt := NewUtilizationInt(42, fields) assert.Equal(t, "42", uInt.GetValue()) assert.Equal(t, "operations", uInt.GetResourceType()) assert.Equal(t, "rid-1", uInt.GetResourceId()) @@ -313,7 +313,7 @@ func TestEmitMeterRecord_BeholderObserver(t *testing.T) { MeterRecordsEnabled: true, Emitter: beholder.GetEmitter(), }) - u := NewUtilization(1, UtilizationFields{ + u := NewUtilizationInt(1, UtilizationFields{ ResourceType: "operations", ResourceID: "trigger-1", EventID: "event-1", diff --git a/pkg/resourcemanager/utilization.go b/pkg/resourcemanager/utilization.go index e01e2938d3..9712809ece 100644 --- a/pkg/resourcemanager/utilization.go +++ b/pkg/resourcemanager/utilization.go @@ -15,15 +15,9 @@ type UtilizationFields struct { OrgID string } -// NewUtilization builds a Utilization with int64 quantity encoded as a decimal -// string value. -func NewUtilization(value int64, fields UtilizationFields) *meteringpb.Utilization { - return NewUtilizationString(strconv.FormatInt(value, 10), fields) -} - -// NewUtilizationString builds a Utilization from a pre-formatted numeric string +// NewUtilization builds a Utilization from a pre-formatted numeric string // value. -func NewUtilizationString(value string, fields UtilizationFields) *meteringpb.Utilization { +func NewUtilization(value string, fields UtilizationFields) *meteringpb.Utilization { return &meteringpb.Utilization{ Value: value, ResourceType: fields.ResourceType, @@ -33,15 +27,21 @@ func NewUtilizationString(value string, fields UtilizationFields) *meteringpb.Ut } } +// NewUtilizationInt builds a Utilization with int64 quantity encoded as a +// decimal string value. +func NewUtilizationInt(value int64, fields UtilizationFields) *meteringpb.Utilization { + return NewUtilization(strconv.FormatInt(value, 10), fields) +} + // NewUtilizationBig builds a Utilization from an arbitrary-precision integer. func NewUtilizationBig(value *big.Int, fields UtilizationFields) *meteringpb.Utilization { if value == nil { - return NewUtilizationString("0", fields) + return NewUtilization("0", fields) } - return NewUtilizationString(value.String(), fields) + return NewUtilization(value.String(), fields) } // NewUtilizationFloat builds a Utilization from a floating-point value. func NewUtilizationFloat(value float64, fields UtilizationFields) *meteringpb.Utilization { - return NewUtilizationString(strconv.FormatFloat(value, 'g', -1, 64), fields) + return NewUtilization(strconv.FormatFloat(value, 'f', -1, 64), fields) } From aeec25e63e382b4e11284cbacd6c3107d0b72a19 Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Mon, 6 Jul 2026 15:21:47 -0400 Subject: [PATCH 10/11] changes from proto: simplified naming + numericTenantID --- go.mod | 2 +- go.sum | 2 - pkg/loop/config.go | 17 +++--- pkg/loop/config_test.go | 3 ++ pkg/resourcemanager/meterable.go | 60 ++++++++++++--------- pkg/resourcemanager/resourcemanager.go | 1 + pkg/resourcemanager/resourcemanager_test.go | 22 ++++---- 7 files changed, 62 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index d4f68f6389..a7edf4a20a 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260701185448-696c075849ea + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 diff --git a/go.sum b/go.sum index 2fddfe8305..3441393f7f 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,6 @@ github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260701185448-696c075849ea h1:oo9Ir4TU9/cG8R0bICcVeprEE+7F31Ew0i7CnI4p4EA= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260701185448-696c075849ea/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 h1:B7itmjy+CMJ26elVw/cAJqqhBQ3Xa/mBYWK0/rQ5MuI= diff --git a/pkg/loop/config.go b/pkg/loop/config.go index 23c442c59f..58d956bbfc 100644 --- a/pkg/loop/config.go +++ b/pkg/loop/config.go @@ -89,6 +89,7 @@ const ( envMeterSnapshotsEnabled = "CL_METER_SNAPSHOTS_ENABLED" envMeterProduct = "CL_METER_PRODUCT" envMeterTenant = "CL_METER_TENANT" + envMeterNumericTenantID = "CL_METER_NUMERIC_TENANT_ID" envMeterEnvironment = "CL_METER_ENVIRONMENT" envMeterZone = "CL_METER_ZONE" envMeterNodeID = "CL_METER_NODE_ID" @@ -181,7 +182,8 @@ type EnvConfig struct { MeterRecordsEnabled bool MeterSnapshotsEnabled bool - // MeterProduct / MeterTenant / MeterEnvironment / MeterZone / MeterNodeID are + // MeterProduct / MeterTenant / MeterNumericTenantID / MeterEnvironment / + // MeterZone / MeterNodeID are // the static deployment+node identity dimensions used as coarse // metering/billing rollup dimensions. They are resolved once from node // config by the host and delivered to every LOOP plugin over the env, the @@ -192,11 +194,12 @@ type EnvConfig struct { // MeterNodeID is the node's logical name (e.g. "clp-cre-wf-zone-a-1"), // not the CSA public key; the CSA key rides emitted events separately as the // node_csa_key attribute. - MeterProduct string - MeterTenant string - MeterEnvironment string - MeterZone string - MeterNodeID string + MeterProduct string + MeterTenant string + MeterNumericTenantID string + MeterEnvironment string + MeterZone string + MeterNodeID string TracingEnabled bool TracingCollectorTarget string @@ -294,6 +297,7 @@ func (e *EnvConfig) AsCmdEnv() (env []string) { add(envMeterSnapshotsEnabled, strconv.FormatBool(e.MeterSnapshotsEnabled)) add(envMeterProduct, e.MeterProduct) add(envMeterTenant, e.MeterTenant) + add(envMeterNumericTenantID, e.MeterNumericTenantID) add(envMeterEnvironment, e.MeterEnvironment) add(envMeterZone, e.MeterZone) add(envMeterNodeID, e.MeterNodeID) @@ -562,6 +566,7 @@ func (e *EnvConfig) parse() error { e.MeterProduct = os.Getenv(envMeterProduct) e.MeterTenant = os.Getenv(envMeterTenant) + e.MeterNumericTenantID = os.Getenv(envMeterNumericTenantID) e.MeterEnvironment = os.Getenv(envMeterEnvironment) e.MeterZone = os.Getenv(envMeterZone) e.MeterNodeID = os.Getenv(envMeterNodeID) diff --git a/pkg/loop/config_test.go b/pkg/loop/config_test.go index 0fece443b6..4487868aa1 100644 --- a/pkg/loop/config_test.go +++ b/pkg/loop/config_test.go @@ -89,6 +89,7 @@ func TestEnvConfig_parse(t *testing.T) { envMeterSnapshotsEnabled: "false", envMeterProduct: "cre-mainline", envMeterTenant: "mainline", + envMeterNumericTenantID: "42", envMeterEnvironment: "production", envMeterZone: "wf-zone-a", envMeterNodeID: "csa-pubkey-1", @@ -210,6 +211,7 @@ var envCfgFull = EnvConfig{ MeterSnapshotsEnabled: false, MeterProduct: "cre-mainline", MeterTenant: "mainline", + MeterNumericTenantID: "42", MeterEnvironment: "production", MeterZone: "wf-zone-a", MeterNodeID: "csa-pubkey-1", @@ -282,6 +284,7 @@ func TestEnvConfig_AsCmdEnv(t *testing.T) { assert.Equal(t, "false", got[envMeterSnapshotsEnabled]) assert.Equal(t, "cre-mainline", got[envMeterProduct]) assert.Equal(t, "mainline", got[envMeterTenant]) + assert.Equal(t, "42", got[envMeterNumericTenantID]) assert.Equal(t, "production", got[envMeterEnvironment]) assert.Equal(t, "wf-zone-a", got[envMeterZone]) assert.Equal(t, "csa-pubkey-1", got[envMeterNodeID]) diff --git a/pkg/resourcemanager/meterable.go b/pkg/resourcemanager/meterable.go index 37cfee8831..fcb228e440 100644 --- a/pkg/resourcemanager/meterable.go +++ b/pkg/resourcemanager/meterable.go @@ -14,7 +14,7 @@ import ( // EmitMeterRecord. type Meterable interface { // ResourceIdentity returns the producer's base identity: the coarse - // dimensions (product, tenant, environment, zone, don_identifier, service) plus + // dimensions (product, tenant, numeric_tenant_id, environment, zone, don, service) plus // the service-level resource_pool / resource_pool_id. Per-resource billing // fields (resource_type/resource_id/org_id/event_id/value) are carried by // Utilizations on MeterRecord and MeterSnapshot. @@ -51,8 +51,11 @@ type SnapshotEntry struct { type DeploymentIdentity struct { // Product is the deployment product, e.g. "cre". Product string - // Tenant is the deployment tenant, e.g. "mainline" or "enterprise". + // Tenant is the human-readable deployment tenant name, e.g. "mainline" or + // "enterprise". Tenant string + // NumericTenantID is the numbered tenant identifier as a string. + NumericTenantID string // Environment is the deployment environment, e.g. "production", "staging". Environment string // Zone is the deployment zone, e.g. "wf-zone-a". @@ -64,12 +67,12 @@ type DeploymentIdentity struct { NodeID string } -// DonIdentifier captures DON-specific identity dimensions as one unit. -type DonIdentifier struct { - // DonID is the DON identifier the emitting service belongs to. +// DonIdentity captures DON-specific identity dimensions as one unit. +type DonIdentity struct { + // DonID is the DON ID the emitting service belongs to. DonID string // NodeID is the node's logical name within the scope of the DON, e.g. - // "clp-cre-wf-zone-a-1". It is a human-readable identifier, NOT the CSA + // "clp-cre-wf-zone-a-1". It is a human-readable ID, NOT the CSA // public key. The prefix can be redundant with other fully-qualified // dimensions, but helps readability. The CSA key is emitted separately via // the node_csa_key attribute. @@ -84,9 +87,13 @@ type ResourceIdentity struct { // Product is the deployment product, e.g. "cre". Product string - // Tenant is the deployment tenant, e.g. "mainline" or "enterprise". + // Tenant is the human-readable deployment tenant name, e.g. "mainline" or + // "enterprise". Tenant string + // NumericTenantID is the numbered tenant identifier as a string. + NumericTenantID string + // Environment is the deployment environment, e.g. "production", // "staging". Environment string @@ -94,9 +101,9 @@ type ResourceIdentity struct { // Zone is the deployment zone, e.g. "wf-zone-a". Zone string - // DonIdentifier groups DON-specific identity dimensions so consumers can + // Don groups DON-specific identity dimensions so consumers can // branch on one struct instead of handling don/node permutations. - DonIdentifier *DonIdentifier + Don *DonIdentity // Service is the stable service constant identifying the emitting service, // e.g. "cron-trigger", "http-trigger", "evm-log-trigger", @@ -115,35 +122,36 @@ type ResourceIdentity struct { // toProto converts r to its wire form. Field order mirrors the proto. func (r ResourceIdentity) toProto() *meteringpb.ResourceIdentity { pb := &meteringpb.ResourceIdentity{ - Product: r.Product, - Tenant: r.Tenant, - Environment: r.Environment, - Zone: r.Zone, - Service: r.Service, - ResourcePool: r.ResourcePool, - ResourcePoolId: r.ResourcePoolID, + Product: r.Product, + Tenant: r.Tenant, + NumericTenantId: r.NumericTenantID, + Environment: r.Environment, + Zone: r.Zone, + Service: r.Service, + ResourcePool: r.ResourcePool, + ResourcePoolId: r.ResourcePoolID, } - if r.DonIdentifier != nil { - pb.DonIdentifier = &meteringpb.DonIdentifier{ - DonId: r.DonIdentifier.DonID, - NodeId: r.DonIdentifier.NodeID, + if r.Don != nil { + pb.Don = &meteringpb.DonIdentity{ + DonId: r.Don.DonID, + NodeId: r.Don.NodeID, } } return pb } -// DonID returns the DON identifier when present. +// DonID returns the DON ID when present. func (r ResourceIdentity) DonID() string { - if r.DonIdentifier == nil { + if r.Don == nil { return "" } - return r.DonIdentifier.DonID + return r.Don.DonID } -// NodeID returns the node identifier when present. +// NodeID returns the node ID when present. func (r ResourceIdentity) NodeID() string { - if r.DonIdentifier == nil { + if r.Don == nil { return "" } - return r.DonIdentifier.NodeID + return r.Don.NodeID } diff --git a/pkg/resourcemanager/resourcemanager.go b/pkg/resourcemanager/resourcemanager.go index 170679b87f..8c2ad17440 100644 --- a/pkg/resourcemanager/resourcemanager.go +++ b/pkg/resourcemanager/resourcemanager.go @@ -417,6 +417,7 @@ func (rm *ResourceManager) recordUtilization(ctx context.Context, id ResourceIde rm.utilization.Record(ctx, value, metric.WithAttributes( attribute.String("product", id.Product), attribute.String("tenant", id.Tenant), + attribute.String("numeric_tenant_id", id.NumericTenantID), attribute.String("environment", id.Environment), attribute.String("zone", id.Zone), attribute.String("don_id", id.DonID()), diff --git a/pkg/resourcemanager/resourcemanager_test.go b/pkg/resourcemanager/resourcemanager_test.go index 56377d4497..7b86a2d429 100644 --- a/pkg/resourcemanager/resourcemanager_test.go +++ b/pkg/resourcemanager/resourcemanager_test.go @@ -21,14 +21,15 @@ import ( ) var testIdentity = ResourceIdentity{ - Product: "cre-mainline", - Tenant: "mainline", - Environment: "production", - Zone: "wf-zone-a", - DonIdentifier: &DonIdentifier{DonID: "don-1", NodeID: "node-1"}, - Service: "cron-trigger", - ResourcePool: "trigger_registrations", - ResourcePoolID: "", + Product: "cre-mainline", + Tenant: "mainline", + NumericTenantID: "42", + Environment: "production", + Zone: "wf-zone-a", + Don: &DonIdentity{DonID: "don-1", NodeID: "node-1"}, + Service: "cron-trigger", + ResourcePool: "trigger_registrations", + ResourcePoolID: "", } type emitCall struct { @@ -157,8 +158,9 @@ func TestEmitMeterRecord_Success(t *testing.T) { require.NotNil(t, record.GetIdentity()) assert.Equal(t, testIdentity.Product, record.GetIdentity().GetProduct()) assert.Equal(t, testIdentity.Tenant, record.GetIdentity().GetTenant()) - assert.Equal(t, testIdentity.DonID(), record.GetIdentity().GetDonIdentifier().GetDonId()) - assert.Equal(t, testIdentity.NodeID(), record.GetIdentity().GetDonIdentifier().GetNodeId()) + assert.Equal(t, testIdentity.NumericTenantID, record.GetIdentity().GetNumericTenantId()) + assert.Equal(t, testIdentity.DonID(), record.GetIdentity().GetDon().GetDonId()) + assert.Equal(t, testIdentity.NodeID(), record.GetIdentity().GetDon().GetNodeId()) assert.Equal(t, testIdentity.ResourcePool, record.GetIdentity().GetResourcePool()) assert.Equal(t, meteringpb.MeterAction_METER_ACTION_RESERVE, record.GetAction()) require.Len(t, record.GetUtilizations(), 1) From 13acbbedf964c9955b8eed7951a624558d7f568a Mon Sep 17 00:00:00 2001 From: Patrick Huie Date: Mon, 6 Jul 2026 15:32:58 -0400 Subject: [PATCH 11/11] missing go.sum --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index 3441393f7f..49416dddf2 100644 --- a/go.sum +++ b/go.sum @@ -266,6 +266,8 @@ github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260622152157-c8e129347b8b/go.mod h1:vTFHTCbLui4Vn8fTmAadfE3rdnvfrDwOmMujmW857D0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019 h1:B355/rV/lwpZl3C5iVsFQZU7+LeA+5BTTIzTDYlDOrA= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260706185759-873029fd9019/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b h1:36knUpKHHAZ86K4FGWXtx8i/EQftGdk2bqCoEu/Cha8= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260205130626-db2a2aab956b/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 h1:B7itmjy+CMJ26elVw/cAJqqhBQ3Xa/mBYWK0/rQ5MuI=