diff --git a/sei-db/db_engine/pebbledb/db.go b/sei-db/db_engine/pebbledb/db.go index 686e9bad38..95e723c8af 100644 --- a/sei-db/db_engine/pebbledb/db.go +++ b/sei-db/db_engine/pebbledb/db.go @@ -27,7 +27,8 @@ type pebbleDB struct { var _ types.KeyValueDB = (*pebbleDB)(nil) -// Open opens (or creates) a Pebble-backed DB at path, returning a KeyValueDB +// Open opens (or creates) a Pebble-backed DB at path, returning a KeyValueDB. +// ctx is unused: metrics collection is stopped by Close, not by cancellation. func Open( ctx context.Context, config *PebbleDBConfig, @@ -85,14 +86,14 @@ func Open( return nil, err } - ctx, cancel := context.WithCancel(ctx) + var metricsCancel func() if config.EnableMetrics { - NewPebbleMetrics(ctx, db, filepath.Base(config.DataDir), config.MetricsScrapeInterval) + metricsCancel = NewPebbleMetrics(db, filepath.Base(config.DataDir), config.MetricsScrapeInterval) } return &pebbleDB{ db: db, - metricsCancel: cancel, + metricsCancel: metricsCancel, operationMetrics: NewOperationMetrics(config.EnableReadWriteMetrics, filepath.Base(config.DataDir)), }, nil } diff --git a/sei-db/db_engine/pebbledb/mvcc/db.go b/sei-db/db_engine/pebbledb/mvcc/db.go index b645300944..71799e5e0a 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db.go +++ b/sei-db/db_engine/pebbledb/mvcc/db.go @@ -253,11 +253,8 @@ func OpenDB(dataDir string, config config.StateStoreConfig) (types.StateStore, e database.asyncWriteWG.Add(1) go database.writeAsyncInBackground() - // Start background metrics collection for Pebble-internal stats - // (compaction, flush, sstable, memtable, WAL, cache). - metricsCtx, metricsCancel := context.WithCancel(context.Background()) - database.metricsCancel = metricsCancel - pebbledbmetrics.NewPebbleMetrics(metricsCtx, db, dbName, 10*time.Second) + // Refresh Pebble-internal stats (compaction, flush, sstable, memtable, WAL, cache). + database.metricsCancel = pebbledbmetrics.NewPebbleMetrics(db, dbName, 10*time.Second) return database, nil } diff --git a/sei-db/db_engine/pebbledb/pebble_metrics.go b/sei-db/db_engine/pebbledb/pebble_metrics.go index 54f66c589e..b3d9514e64 100644 --- a/sei-db/db_engine/pebbledb/pebble_metrics.go +++ b/sei-db/db_engine/pebbledb/pebble_metrics.go @@ -2,7 +2,8 @@ package pebbledb import ( "context" - "math" + "sync" + "sync/atomic" "time" "go.opentelemetry.io/otel" @@ -10,1730 +11,410 @@ import ( "go.opentelemetry.io/otel/metric" "github.com/cockroachdb/pebble/v2" - - smetrics "github.com/sei-protocol/sei-chain/sei-db/common/metrics" ) const pebbleMeterName = "seidb_pebble" -// PebbleMetrics scrapes metrics from a Pebble DB and records them via OTel instruments. -// Instrument names match sei-db/db_engine/pebbledb/mvcc for dashboard compatibility. -// The databaseName is used as the "db" attribute on all recorded metrics. -// -// Multiple instances are safe: OTel instrument registration is idempotent, so each -// NewPebbleMetrics call receives references to the same underlying instruments. -// The "db" attribute distinguishes series (e.g. pebble_compaction_count{db="state"}). -type PebbleMetrics struct { - db *pebble.DB - databaseName string - - getLatency metric.Float64Histogram - applyChangesetLatency metric.Float64Histogram - applyChangesetAsyncLatency metric.Float64Histogram - pruneLatency metric.Float64Histogram - importLatency metric.Float64Histogram - batchWriteLatency metric.Float64Histogram - - compactionCount metric.Int64Counter - compactionDuration metric.Float64Histogram - compactionBytesRead metric.Int64Counter - compactionBytesWritten metric.Int64Counter - compactionEstimatedDebt metric.Int64Gauge - compactionInProgressBytes metric.Int64Gauge - compactionNumInProgress metric.Int64Gauge - compactionCancelledCount metric.Int64Counter - compactionCancelledBytes metric.Int64Counter - compactionFailedCount metric.Int64Counter - compactionDefaultCount metric.Int64Counter - compactionDeleteOnlyCount metric.Int64Counter - compactionElisionOnlyCount metric.Int64Counter - compactionCopyCount metric.Int64Counter - compactionMoveCount metric.Int64Counter - compactionReadCount metric.Int64Counter - compactionTombstoneDensityCount metric.Int64Counter - compactionRewriteCount metric.Int64Counter - compactionMultiLevelCount metric.Int64Counter - compactionBlobFileRewriteCount metric.Int64Counter - compactionCounterLevelCount metric.Int64Counter - compactionNumProblemSpans metric.Int64Gauge - compactionMarkedFiles metric.Int64Gauge - - ingestCount metric.Int64Counter - - flushCount metric.Int64Counter - flushDuration metric.Float64Histogram - flushBytesWritten metric.Int64Counter - flushNumInProgress metric.Int64Gauge - flushAsIngestCount metric.Int64Counter - flushAsIngestTableCount metric.Int64Counter - flushAsIngestBytes metric.Int64Counter - flushIdleDuration metric.Float64Gauge - - filterHits metric.Int64Counter - filterMisses metric.Int64Counter - - sstableCount metric.Int64Gauge - sstableTotalSize metric.Int64Gauge - sstableSublevels metric.Int64Gauge - sstableScore metric.Float64Gauge - sstableFillFactor metric.Float64Gauge - sstableVirtualCount metric.Int64Gauge - sstableVirtualSize metric.Int64Gauge - sstableBytesIngested metric.Int64Counter - sstableBytesMoved metric.Int64Counter - sstableBytesRead metric.Int64Counter - sstableBytesFlushed metric.Int64Counter - sstableTablesCompacted metric.Int64Counter - sstableTablesFlushed metric.Int64Counter - sstableTablesIngested metric.Int64Counter - sstableTablesMoved metric.Int64Counter - sstableCompensatedFillFactor metric.Float64Gauge - sstableEstimatedReferencesSize metric.Int64Gauge - sstableTablesDeleted metric.Int64Counter - sstableTablesExcised metric.Int64Counter - sstableBlobBytesReadEstimate metric.Int64Counter - sstableBlobBytesCompacted metric.Int64Counter - sstableBlobBytesFlushed metric.Int64Counter - sstableMultiLevelBytesInTop metric.Int64Counter - sstableMultiLevelBytesIn metric.Int64Counter - sstableMultiLevelBytesRead metric.Int64Counter - sstableValueBlocksSize metric.Int64Gauge - sstableBytesWrittenDataBlocks metric.Int64Counter - sstableBytesWrittenValueBlocks metric.Int64Counter - - memtableCount metric.Int64Gauge - memtableTotalSize metric.Int64Gauge - memtableZombieSize metric.Int64Gauge - memtableZombieCount metric.Int64Gauge - - walSize metric.Int64Gauge - walFiles metric.Int64Gauge - walObsoleteFiles metric.Int64Gauge - walObsoletePhysicalSize metric.Int64Gauge - walPhysicalSize metric.Int64Gauge - walBytesIn metric.Int64Counter - walBytesWritten metric.Int64Counter - - tableObsoleteSize metric.Int64Gauge - tableObsoleteCount metric.Int64Gauge - tableZombieSize metric.Int64Gauge - tableZombieCount metric.Int64Gauge - tableLiveSize metric.Int64Gauge - tableLiveCount metric.Int64Gauge - tableBackingCount metric.Int64Gauge - tableBackingSize metric.Int64Gauge - tableCompressedUnknown metric.Int64Gauge - tableCompressedSnappy metric.Int64Gauge - tableCompressedZstd metric.Int64Gauge - tableCompressedMinLZ metric.Int64Gauge - tableCompressedNone metric.Int64Gauge - tableLocalObsoleteSize metric.Int64Gauge - tableLocalObsoleteCount metric.Int64Gauge - tableLocalZombieSize metric.Int64Gauge - tableLocalZombieCount metric.Int64Gauge - tableGarbagePointDeletionsEstimate metric.Int64Gauge - tableGarbageRangeDeletionsEstimate metric.Int64Gauge - tableInitialStatsComplete metric.Int64Gauge - tablePendingStatsCount metric.Int64Gauge - - blobFilesLiveCount metric.Int64Gauge - blobFilesLiveSize metric.Int64Gauge - blobFilesValueSize metric.Int64Gauge - blobFilesReferencedValueSize metric.Int64Gauge - blobFilesObsoleteCount metric.Int64Gauge - blobFilesObsoleteSize metric.Int64Gauge - blobFilesZombieCount metric.Int64Gauge - blobFilesZombieSize metric.Int64Gauge - blobFilesLocalLiveSize metric.Int64Gauge - blobFilesLocalLiveCount metric.Int64Gauge - blobFilesLocalObsoleteSize metric.Int64Gauge - blobFilesLocalObsoleteCount metric.Int64Gauge - blobFilesLocalZombieSize metric.Int64Gauge - blobFilesLocalZombieCount metric.Int64Gauge - - fileCacheSize metric.Int64Gauge - fileCacheTableCount metric.Int64Gauge - fileCacheBlobFileCount metric.Int64Gauge - fileCacheHits metric.Int64Counter - fileCacheMisses metric.Int64Counter - - // prev* track last scraped cumulative values so we Add(delta) not Add(total). - prevCompactionCount int64 - prevCompactionCancelledCount int64 - prevCompactionCancelledBytes int64 - prevCompactionFailedCount int64 - prevCompactionDefaultCount int64 - prevCompactionDeleteOnlyCount int64 - prevCompactionElisionOnlyCount int64 - prevCompactionCopyCount int64 - prevCompactionMoveCount int64 - prevCompactionReadCount int64 - prevCompactionTombstoneDensityCount int64 - prevCompactionRewriteCount int64 - prevCompactionMultiLevelCount int64 - prevCompactionBlobFileRewriteCount int64 - prevCompactionCounterLevelCount int64 - prevIngestCount int64 - prevFlushCount int64 - prevFlushBytesWritten int64 - prevFlushAsIngestCount int64 - prevFlushAsIngestTableCount int64 - prevFlushAsIngestBytes int64 - prevFilterHits int64 - prevFilterMisses int64 - prevWalBytesIn int64 - prevWalBytesWritten int64 - prevWalFailoverDirSwitchCount int64 - prevKeysMissizedTombstonesCount int64 - prevSnapshotPinnedKeys int64 - prevSnapshotPinnedSize int64 - prevFileCacheHits int64 - prevFileCacheMisses int64 - prevCacheHits int64 - prevCacheMisses int64 +// numLevels is how many LSM levels a Pebble snapshot reports. +const numLevels = len(pebble.Metrics{}.Levels) - // prev*ByLevel hold previous cumulative values per level (index = level). - prevCompactionBytesReadByLevel []int64 - prevCompactionBytesWrittenByLevel []int64 - prevSstableBytesIngestedByLevel []int64 - prevSstableBytesMovedByLevel []int64 - prevSstableBytesReadByLevel []int64 - prevSstableBytesFlushedByLevel []int64 - prevSstableTablesCompactedByLevel []int64 - prevSstableTablesFlushedByLevel []int64 - prevSstableTablesIngestedByLevel []int64 - prevSstableTablesMovedByLevel []int64 - prevSstableTablesDeletedByLevel []int64 - prevSstableTablesExcisedByLevel []int64 - prevSstableBlobBytesReadEstimateByLevel []int64 - prevSstableBlobBytesCompactedByLevel []int64 - prevSstableBlobBytesFlushedByLevel []int64 - prevSstableMultiLevelBytesInTopByLevel []int64 - prevSstableMultiLevelBytesInByLevel []int64 - prevSstableMultiLevelBytesReadByLevel []int64 - prevSstableBytesWrittenDataBlocksByLevel []int64 - prevSstableBytesWrittenValueBlocksByLevel []int64 +type pebbleMetrics struct { + meter metric.Meter + insts []metric.Observable - walFailoverDirSwitchCount metric.Int64Counter - walFailoverPrimaryDuration metric.Float64Gauge - walFailoverSecondaryDuration metric.Float64Gauge + dbAttrs metric.ObserveOption + levelAttrs [numLevels]metric.ObserveOption - numVirtual metric.Int64Gauge - virtualSize metric.Int64Gauge - remoteTablesCount metric.Int64Gauge - remoteTablesSize metric.Int64Gauge - - keysRangeKeySetsCount metric.Int64Gauge - keysTombstoneCount metric.Int64Gauge - keysMissizedTombstonesCount metric.Int64Counter - - snapshotCount metric.Int64Gauge - snapshotPinnedKeys metric.Int64Counter - snapshotPinnedSize metric.Int64Counter - snapshotEarliestSeqNum metric.Int64Gauge - - tableIters metric.Int64Gauge - uptimeSeconds metric.Float64Gauge - readAmp metric.Int64Gauge - diskSpaceUsage metric.Int64Gauge - - cacheHits metric.Int64Counter - cacheMisses metric.Int64Counter - cacheSize metric.Int64Gauge - - batchSize metric.Int64Histogram - pendingChangesQueueDepth metric.Int64Gauge - iteratorIterations metric.Float64Histogram + snapshot atomic.Pointer[pebble.Metrics] + report []func(metric.Observer, *pebble.Metrics) } -// NewPebbleMetrics creates a PebbleMetrics that scrapes metrics from the given Pebble DB -// and records them to OTel. A background goroutine runs every scrapeInterval until -// ctx is cancelled. The databaseName is attached as the "db" attribute to all recorded -// metrics, enabling multi-DB setups to distinguish series in Prometheus/Grafana. -// -// Multiple instances (e.g. one per DB) are safe: OTel returns the same instruments -// for duplicate registrations, and the "db" attribute separates series. -func NewPebbleMetrics( - ctx context.Context, - db *pebble.DB, - databaseName string, - scrapeInterval time.Duration, -) *PebbleMetrics { +// NewPebbleMetrics registers OTel observables over a Pebble metrics snapshot +func NewPebbleMetrics(db *pebble.DB, databaseName string, refreshInterval time.Duration) func() { meter := otel.Meter(pebbleMeterName) - - getLatency, _ := meter.Float64Histogram( - "pebble_get_latency", - metric.WithDescription("Time taken to get a key from PebbleDB"), - metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), - ) - applyChangesetLatency, _ := meter.Float64Histogram( - "pebble_apply_changeset_latency", - metric.WithDescription("Time taken to apply changeset to PebbleDB"), - metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), - ) - applyChangesetAsyncLatency, _ := meter.Float64Histogram( - "pebble_apply_changeset_async_latency", - metric.WithDescription("Time taken to queue changeset for async write"), - metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), - ) - pruneLatency, _ := meter.Float64Histogram( - "pebble_prune_latency", - metric.WithDescription("Time taken to prune old versions from PebbleDB"), - metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), - ) - importLatency, _ := meter.Float64Histogram( - "pebble_import_latency", - metric.WithDescription("Time taken to import snapshot data to PebbleDB"), - metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), - ) - batchWriteLatency, _ := meter.Float64Histogram( - "pebble_batch_write_latency", - metric.WithDescription("Time taken to write a batch to PebbleDB"), - metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), - ) - - compactionCount, _ := meter.Int64Counter( - "pebble_compaction_count", - metric.WithDescription("Total number of compactions"), - metric.WithUnit("{count}"), - ) - compactionDuration, _ := meter.Float64Histogram( - "pebble_compaction_duration", - metric.WithDescription("Duration of compaction operations"), - metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), - ) - compactionBytesRead, _ := meter.Int64Counter( - "pebble_compaction_bytes_read", - metric.WithDescription("Total bytes read during compaction"), - metric.WithUnit("By"), - ) - compactionBytesWritten, _ := meter.Int64Counter( - "pebble_compaction_bytes_written", - metric.WithDescription("Total bytes written during compaction"), - metric.WithUnit("By"), - ) - compactionEstimatedDebt, _ := meter.Int64Gauge( - "pebble_compaction_estimated_debt", - metric.WithDescription("Estimated bytes to compact for LSM to reach stable state"), - metric.WithUnit("By"), - ) - compactionInProgressBytes, _ := meter.Int64Gauge( - "pebble_compaction_in_progress_bytes", - metric.WithDescription("Bytes in sstables being written by in-progress compactions"), - metric.WithUnit("By"), - ) - compactionNumInProgress, _ := meter.Int64Gauge( - "pebble_compaction_num_in_progress", - metric.WithDescription("Number of compactions in progress"), - metric.WithUnit("{count}"), - ) - compactionCancelledCount, _ := meter.Int64Counter( - "pebble_compaction_cancelled_count", - metric.WithDescription("Number of compactions that were cancelled"), - metric.WithUnit("{count}"), - ) - compactionCancelledBytes, _ := meter.Int64Counter( - "pebble_compaction_cancelled_bytes", - metric.WithDescription("Bytes written by cancelled compactions"), - metric.WithUnit("By"), - ) - compactionFailedCount, _ := meter.Int64Counter( - "pebble_compaction_failed_count", - metric.WithDescription("Number of compactions that hit an error"), - metric.WithUnit("{count}"), - ) - compactionDefaultCount, _ := meter.Int64Counter( - "pebble_compaction_default_count", - metric.WithDescription("Default compactions"), - metric.WithUnit("{count}"), - ) - compactionDeleteOnlyCount, _ := meter.Int64Counter( - "pebble_compaction_delete_only_count", - metric.WithDescription("Delete-only compactions"), - metric.WithUnit("{count}"), - ) - compactionElisionOnlyCount, _ := meter.Int64Counter( - "pebble_compaction_elision_only_count", - metric.WithDescription("Elision-only compactions"), - metric.WithUnit("{count}"), - ) - compactionCopyCount, _ := meter.Int64Counter( - "pebble_compaction_copy_count", - metric.WithDescription("Copy compactions"), - metric.WithUnit("{count}"), - ) - compactionMoveCount, _ := meter.Int64Counter( - "pebble_compaction_move_count", - metric.WithDescription("Move compactions"), - metric.WithUnit("{count}"), - ) - compactionReadCount, _ := meter.Int64Counter( - "pebble_compaction_read_count", - metric.WithDescription("Read compactions"), - metric.WithUnit("{count}"), - ) - compactionTombstoneDensityCount, _ := meter.Int64Counter( - "pebble_compaction_tombstone_density_count", - metric.WithDescription("Tombstone-density compactions"), - metric.WithUnit("{count}"), - ) - compactionRewriteCount, _ := meter.Int64Counter( - "pebble_compaction_rewrite_count", - metric.WithDescription("Rewrite compactions"), - metric.WithUnit("{count}"), - ) - compactionMultiLevelCount, _ := meter.Int64Counter( - "pebble_compaction_multi_level_count", - metric.WithDescription("Multi-level compactions"), - metric.WithUnit("{count}"), - ) - compactionBlobFileRewriteCount, _ := meter.Int64Counter( - "pebble_compaction_blob_file_rewrite_count", - metric.WithDescription("Blob file rewrite compactions"), - metric.WithUnit("{count}"), - ) - compactionCounterLevelCount, _ := meter.Int64Counter( - "pebble_compaction_counter_level_count", - metric.WithDescription("Counter-level compactions"), - metric.WithUnit("{count}"), - ) - compactionNumProblemSpans, _ := meter.Int64Gauge( - "pebble_compaction_num_problem_spans", - metric.WithDescription("Problem spans blocking compactions"), - metric.WithUnit("{count}"), - ) - compactionMarkedFiles, _ := meter.Int64Gauge( - "pebble_compaction_marked_files", - metric.WithDescription("Files marked for compaction"), - metric.WithUnit("{count}"), - ) - - ingestCount, _ := meter.Int64Counter( - "pebble_ingest_count", - metric.WithDescription("Total number of ingestions"), - metric.WithUnit("{count}"), - ) - - flushCount, _ := meter.Int64Counter( - "pebble_flush_count", - metric.WithDescription("Total number of memtable flushes"), - metric.WithUnit("{count}"), - ) - flushDuration, _ := meter.Float64Histogram( - "pebble_flush_duration", - metric.WithDescription("Duration of memtable flush operations"), - metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(smetrics.LatencyBuckets...), - ) - flushBytesWritten, _ := meter.Int64Counter( - "pebble_flush_bytes_written", - metric.WithDescription("Total bytes written during memtable flushes"), - metric.WithUnit("By"), - ) - flushNumInProgress, _ := meter.Int64Gauge( - "pebble_flush_num_in_progress", - metric.WithDescription("Number of flushes in progress"), - metric.WithUnit("{count}"), - ) - flushAsIngestCount, _ := meter.Int64Counter( - "pebble_flush_as_ingest_count", - metric.WithDescription("Flush operations handling ingested tables"), - metric.WithUnit("{count}"), - ) - flushAsIngestTableCount, _ := meter.Int64Counter( - "pebble_flush_as_ingest_table_count", - metric.WithDescription("Tables ingested as flushables"), - metric.WithUnit("{count}"), - ) - flushAsIngestBytes, _ := meter.Int64Counter( - "pebble_flush_as_ingest_bytes", - metric.WithDescription("Bytes flushed for flushables from ingestion"), - metric.WithUnit("By"), - ) - flushIdleDuration, _ := meter.Float64Gauge( - "pebble_flush_idle_duration", - metric.WithDescription("Idle duration before memtable flushes"), - metric.WithUnit("s"), - ) - - filterHits, _ := meter.Int64Counter( - "pebble_filter_hits", - metric.WithDescription("Bloom filter hits (block reads avoided)"), - metric.WithUnit("{count}"), - ) - filterMisses, _ := meter.Int64Counter( - "pebble_filter_misses", - metric.WithDescription("Bloom filter misses"), - metric.WithUnit("{count}"), - ) - - sstableCount, _ := meter.Int64Gauge( - "pebble_sstable_count", - metric.WithDescription("Current number of SSTables at each level"), - metric.WithUnit("{count}"), - ) - sstableTotalSize, _ := meter.Int64Gauge( - "pebble_sstable_total_size", - metric.WithDescription("Total size of SSTables at each level"), - metric.WithUnit("By"), - ) - sstableSublevels, _ := meter.Int64Gauge( - "pebble_sstable_sublevels", - metric.WithDescription("Number of sublevels (read amplification); L0 only has non-0/1"), - metric.WithUnit("{count}"), - ) - sstableScore, _ := meter.Float64Gauge( - "pebble_sstable_score", - metric.WithDescription("Level compaction score (0 if no compaction needed)"), - metric.WithUnit("1"), - ) - sstableFillFactor, _ := meter.Float64Gauge( - "pebble_sstable_fill_factor", - metric.WithDescription("Level fill factor (size vs ideal size)"), - metric.WithUnit("1"), - ) - sstableVirtualCount, _ := meter.Int64Gauge( - "pebble_sstable_virtual_count", - metric.WithDescription("Number of virtual sstables at level"), - metric.WithUnit("{count}"), - ) - sstableVirtualSize, _ := meter.Int64Gauge( - "pebble_sstable_virtual_size", - metric.WithDescription("Size of virtual sstables at level"), - metric.WithUnit("By"), - ) - sstableBytesIngested, _ := meter.Int64Counter( - "pebble_sstable_bytes_ingested", - metric.WithDescription("Sstable bytes ingested at level"), - metric.WithUnit("By"), - ) - sstableBytesMoved, _ := meter.Int64Counter( - "pebble_sstable_bytes_moved", - metric.WithDescription("Sstable bytes moved by move compaction at level"), - metric.WithUnit("By"), - ) - sstableBytesRead, _ := meter.Int64Counter( - "pebble_sstable_bytes_read", - metric.WithDescription("Bytes read for compactions at level"), - metric.WithUnit("By"), - ) - sstableBytesFlushed, _ := meter.Int64Counter( - "pebble_sstable_bytes_flushed", - metric.WithDescription("Bytes written to sstables during flushes at level"), - metric.WithUnit("By"), - ) - sstableTablesCompacted, _ := meter.Int64Counter( - "pebble_sstable_tables_compacted", - metric.WithDescription("Sstables compacted to this level"), - metric.WithUnit("{count}"), - ) - sstableTablesFlushed, _ := meter.Int64Counter( - "pebble_sstable_tables_flushed", - metric.WithDescription("Sstables flushed to this level"), - metric.WithUnit("{count}"), - ) - sstableTablesIngested, _ := meter.Int64Counter( - "pebble_sstable_tables_ingested", - metric.WithDescription("Sstables ingested into level"), - metric.WithUnit("{count}"), - ) - sstableTablesMoved, _ := meter.Int64Counter( - "pebble_sstable_tables_moved", - metric.WithDescription("Sstables moved to level by move compaction"), - metric.WithUnit("{count}"), - ) - sstableCompensatedFillFactor, _ := meter.Float64Gauge( - "pebble_sstable_compensated_fill_factor", - metric.WithDescription("Level compensated fill factor"), - metric.WithUnit("1"), - ) - sstableEstimatedReferencesSize, _ := meter.Int64Gauge( - "pebble_sstable_estimated_references_size", - metric.WithDescription("Est. physical size of blob refs at level"), - metric.WithUnit("By"), - ) - sstableTablesDeleted, _ := meter.Int64Counter( - "pebble_sstable_tables_deleted", - metric.WithDescription("Sstables deleted by delete-only compaction at level"), - metric.WithUnit("{count}"), - ) - sstableTablesExcised, _ := meter.Int64Counter( - "pebble_sstable_tables_excised", - metric.WithDescription("Sstables excised by delete-only compaction at level"), - metric.WithUnit("{count}"), - ) - sstableBlobBytesReadEstimate, _ := meter.Int64Counter( - "pebble_sstable_blob_bytes_read_estimate", - metric.WithDescription("Est. physical bytes read for blob refs at level"), - metric.WithUnit("By"), - ) - sstableBlobBytesCompacted, _ := meter.Int64Counter( - "pebble_sstable_blob_bytes_compacted", - metric.WithDescription("Blob bytes written during compaction at level"), - metric.WithUnit("By"), - ) - sstableBlobBytesFlushed, _ := meter.Int64Counter( - "pebble_sstable_blob_bytes_flushed", - metric.WithDescription("Blob bytes written during flush at level"), - metric.WithUnit("By"), - ) - sstableMultiLevelBytesInTop, _ := meter.Int64Counter( - "pebble_sstable_multi_level_bytes_in_top", - metric.WithDescription("Bytes from top level in multilevel compaction"), - metric.WithUnit("By"), - ) - sstableMultiLevelBytesIn, _ := meter.Int64Counter( - "pebble_sstable_multi_level_bytes_in", - metric.WithDescription("Bytes in for multilevel compaction"), - metric.WithUnit("By"), - ) - sstableMultiLevelBytesRead, _ := meter.Int64Counter( - "pebble_sstable_multi_level_bytes_read", - metric.WithDescription("Bytes read for multilevel compaction"), - metric.WithUnit("By"), - ) - sstableValueBlocksSize, _ := meter.Int64Gauge( - "pebble_sstable_value_blocks_size", - metric.WithDescription("Value blocks size at level"), - metric.WithUnit("By"), - ) - sstableBytesWrittenDataBlocks, _ := meter.Int64Counter( - "pebble_sstable_bytes_written_data_blocks", - metric.WithDescription("Bytes written to data blocks at level"), - metric.WithUnit("By"), - ) - sstableBytesWrittenValueBlocks, _ := meter.Int64Counter( - "pebble_sstable_bytes_written_value_blocks", - metric.WithDescription("Bytes written to value blocks at level"), - metric.WithUnit("By"), - ) - - memtableCount, _ := meter.Int64Gauge( - "pebble_memtable_count", - metric.WithDescription("Current number of memtables"), - metric.WithUnit("{count}"), - ) - memtableTotalSize, _ := meter.Int64Gauge( - "pebble_memtable_total_size", - metric.WithDescription("Total size of all memtables"), - metric.WithUnit("By"), - ) - memtableZombieSize, _ := meter.Int64Gauge( - "pebble_memtable_zombie_size", - metric.WithDescription("Bytes in zombie memtables (released but in use by iterators)"), - metric.WithUnit("By"), - ) - memtableZombieCount, _ := meter.Int64Gauge( - "pebble_memtable_zombie_count", - metric.WithDescription("Count of zombie memtables"), - metric.WithUnit("{count}"), - ) - - walSize, _ := meter.Int64Gauge( - "pebble_wal_size", - metric.WithDescription("Current size of Write-Ahead Log"), - metric.WithUnit("By"), - ) - walFiles, _ := meter.Int64Gauge( - "pebble_wal_files", - metric.WithDescription("Number of live WAL files"), - metric.WithUnit("{count}"), - ) - walObsoleteFiles, _ := meter.Int64Gauge( - "pebble_wal_obsolete_files", - metric.WithDescription("Number of obsolete WAL files"), - metric.WithUnit("{count}"), - ) - walObsoletePhysicalSize, _ := meter.Int64Gauge( - "pebble_wal_obsolete_physical_size", - metric.WithDescription("Physical size of obsolete WAL files"), - metric.WithUnit("By"), - ) - walPhysicalSize, _ := meter.Int64Gauge( - "pebble_wal_physical_size", - metric.WithDescription("Physical size of WAL files on disk"), - metric.WithUnit("By"), - ) - walBytesIn, _ := meter.Int64Counter( - "pebble_wal_bytes_in", - metric.WithDescription("Logical bytes written to WAL"), - metric.WithUnit("By"), - ) - walBytesWritten, _ := meter.Int64Counter( - "pebble_wal_bytes_written", - metric.WithDescription("Bytes written to WAL"), - metric.WithUnit("By"), - ) - - tableObsoleteSize, _ := meter.Int64Gauge( - "pebble_table_obsolete_size", - metric.WithDescription("Bytes in obsolete tables no longer referenced"), - metric.WithUnit("By"), - ) - tableObsoleteCount, _ := meter.Int64Gauge( - "pebble_table_obsolete_count", - metric.WithDescription("Count of obsolete tables"), - metric.WithUnit("{count}"), - ) - tableZombieSize, _ := meter.Int64Gauge( - "pebble_table_zombie_size", - metric.WithDescription("Bytes in zombie tables (released but in use by iterators)"), - metric.WithUnit("By"), - ) - tableZombieCount, _ := meter.Int64Gauge( - "pebble_table_zombie_count", - metric.WithDescription("Count of zombie tables"), - metric.WithUnit("{count}"), - ) - tableLiveSize, _ := meter.Int64Gauge( - "pebble_table_live_size", - metric.WithDescription("Bytes in live tables"), - metric.WithUnit("By"), - ) - tableLiveCount, _ := meter.Int64Gauge( - "pebble_table_live_count", - metric.WithDescription("Count of live tables"), - metric.WithUnit("{count}"), - ) - tableBackingCount, _ := meter.Int64Gauge( - "pebble_table_backing_count", - metric.WithDescription("Sstables backing virtual tables"), - metric.WithUnit("{count}"), - ) - tableBackingSize, _ := meter.Int64Gauge( - "pebble_table_backing_size", - metric.WithDescription("Size of sstables backing virtual tables"), - metric.WithUnit("By"), - ) - tableCompressedUnknown, _ := meter.Int64Gauge( - "pebble_table_compressed_unknown", - metric.WithDescription("Sstables with unknown compression"), - metric.WithUnit("{count}"), - ) - tableCompressedSnappy, _ := meter.Int64Gauge( - "pebble_table_compressed_snappy", - metric.WithDescription("Snappy-compressed sstables"), - metric.WithUnit("{count}"), - ) - tableCompressedZstd, _ := meter.Int64Gauge( - "pebble_table_compressed_zstd", - metric.WithDescription("Zstd-compressed sstables"), - metric.WithUnit("{count}"), - ) - tableCompressedMinLZ, _ := meter.Int64Gauge( - "pebble_table_compressed_minlz", - metric.WithDescription("MinLZ-compressed sstables"), - metric.WithUnit("{count}"), - ) - tableCompressedNone, _ := meter.Int64Gauge( - "pebble_table_compressed_none", - metric.WithDescription("Uncompressed sstables"), - metric.WithUnit("{count}"), - ) - tableLocalObsoleteSize, _ := meter.Int64Gauge( - "pebble_table_local_obsolete_size", - metric.WithDescription("Local obsolete table size"), - metric.WithUnit("By"), - ) - tableLocalObsoleteCount, _ := meter.Int64Gauge( - "pebble_table_local_obsolete_count", - metric.WithDescription("Local obsolete table count"), - metric.WithUnit("{count}"), - ) - tableLocalZombieSize, _ := meter.Int64Gauge( - "pebble_table_local_zombie_size", - metric.WithDescription("Local zombie table size"), - metric.WithUnit("By"), - ) - tableLocalZombieCount, _ := meter.Int64Gauge( - "pebble_table_local_zombie_count", - metric.WithDescription("Local zombie table count"), - metric.WithUnit("{count}"), - ) - tableGarbagePointDeletionsEstimate, _ := meter.Int64Gauge( - "pebble_table_garbage_point_deletions_estimate", - metric.WithDescription("Est. bytes reclaimable from point deletes"), - metric.WithUnit("By"), - ) - tableGarbageRangeDeletionsEstimate, _ := meter.Int64Gauge( - "pebble_table_garbage_range_deletions_estimate", - metric.WithDescription("Est. bytes reclaimable from range deletes"), - metric.WithUnit("By"), - ) - tableInitialStatsComplete, _ := meter.Int64Gauge( - "pebble_table_initial_stats_complete", - metric.WithDescription("1 if initial stats collection complete"), - metric.WithUnit("1"), - ) - tablePendingStatsCount, _ := meter.Int64Gauge( - "pebble_table_pending_stats_count", - metric.WithDescription("New sstables awaiting stats collection"), - metric.WithUnit("{count}"), - ) - blobFilesLiveCount, _ := meter.Int64Gauge( - "pebble_blob_files_live_count", - metric.WithDescription("Live blob file count"), - metric.WithUnit("{count}"), - ) - blobFilesLiveSize, _ := meter.Int64Gauge( - "pebble_blob_files_live_size", - metric.WithDescription("Live blob file physical size"), - metric.WithUnit("By"), - ) - blobFilesValueSize, _ := meter.Int64Gauge( - "pebble_blob_files_value_size", - metric.WithDescription("Uncompressed value size in live blobs"), - metric.WithUnit("By"), - ) - blobFilesReferencedValueSize, _ := meter.Int64Gauge( - "pebble_blob_files_referenced_value_size", - metric.WithDescription("Referenced value size in live blobs"), - metric.WithUnit("By"), - ) - blobFilesObsoleteCount, _ := meter.Int64Gauge( - "pebble_blob_files_obsolete_count", - metric.WithDescription("Obsolete blob file count"), - metric.WithUnit("{count}"), - ) - blobFilesObsoleteSize, _ := meter.Int64Gauge( - "pebble_blob_files_obsolete_size", - metric.WithDescription("Obsolete blob file size"), - metric.WithUnit("By"), - ) - blobFilesZombieCount, _ := meter.Int64Gauge( - "pebble_blob_files_zombie_count", - metric.WithDescription("Zombie blob file count"), - metric.WithUnit("{count}"), - ) - blobFilesZombieSize, _ := meter.Int64Gauge( - "pebble_blob_files_zombie_size", - metric.WithDescription("Zombie blob file size"), - metric.WithUnit("By"), - ) - blobFilesLocalLiveSize, _ := meter.Int64Gauge( - "pebble_blob_files_local_live_size", - metric.WithDescription("Local live blob file size"), - metric.WithUnit("By"), - ) - blobFilesLocalLiveCount, _ := meter.Int64Gauge( - "pebble_blob_files_local_live_count", - metric.WithDescription("Local live blob file count"), - metric.WithUnit("{count}"), - ) - blobFilesLocalObsoleteSize, _ := meter.Int64Gauge( - "pebble_blob_files_local_obsolete_size", - metric.WithDescription("Local obsolete blob file size"), - metric.WithUnit("By"), - ) - blobFilesLocalObsoleteCount, _ := meter.Int64Gauge( - "pebble_blob_files_local_obsolete_count", - metric.WithDescription("Local obsolete blob file count"), - metric.WithUnit("{count}"), - ) - blobFilesLocalZombieSize, _ := meter.Int64Gauge( - "pebble_blob_files_local_zombie_size", - metric.WithDescription("Local zombie blob file size"), - metric.WithUnit("By"), - ) - blobFilesLocalZombieCount, _ := meter.Int64Gauge( - "pebble_blob_files_local_zombie_count", - metric.WithDescription("Local zombie blob file count"), - metric.WithUnit("{count}"), - ) - fileCacheSize, _ := meter.Int64Gauge( - "pebble_file_cache_size", - metric.WithDescription("Bytes in file cache"), - metric.WithUnit("By"), - ) - fileCacheTableCount, _ := meter.Int64Gauge( - "pebble_file_cache_table_count", - metric.WithDescription("Tables in file cache"), - metric.WithUnit("{count}"), - ) - fileCacheBlobFileCount, _ := meter.Int64Gauge( - "pebble_file_cache_blob_file_count", - metric.WithDescription("Blob files in file cache"), - metric.WithUnit("{count}"), - ) - fileCacheHits, _ := meter.Int64Counter( - "pebble_file_cache_hits", - metric.WithDescription("File cache hits"), - metric.WithUnit("{count}"), - ) - fileCacheMisses, _ := meter.Int64Counter( - "pebble_file_cache_misses", - metric.WithDescription("File cache misses"), - metric.WithUnit("{count}"), - ) - walFailoverDirSwitchCount, _ := meter.Int64Counter( - "pebble_wal_failover_dir_switch_count", - metric.WithDescription("WAL directory switches (failover/failback)"), - metric.WithUnit("{count}"), - ) - walFailoverPrimaryDuration, _ := meter.Float64Gauge( - "pebble_wal_failover_primary_duration", - metric.WithDescription("Cumulative WAL write duration on primary"), - metric.WithUnit("s"), - ) - walFailoverSecondaryDuration, _ := meter.Float64Gauge( - "pebble_wal_failover_secondary_duration", - metric.WithDescription("Cumulative WAL write duration on secondary"), - metric.WithUnit("s"), - ) - numVirtual, _ := meter.Int64Gauge( - "pebble_num_virtual", - metric.WithDescription("Total virtual sstable count"), - metric.WithUnit("{count}"), - ) - virtualSize, _ := meter.Int64Gauge( - "pebble_virtual_size", - metric.WithDescription("Total virtual sstable size"), - metric.WithUnit("By"), - ) - remoteTablesCount, _ := meter.Int64Gauge( - "pebble_remote_tables_count", - metric.WithDescription("Remote tables count"), - metric.WithUnit("{count}"), - ) - remoteTablesSize, _ := meter.Int64Gauge( - "pebble_remote_tables_size", - metric.WithDescription("Remote tables size"), - metric.WithUnit("By"), - ) - - keysRangeKeySetsCount, _ := meter.Int64Gauge( - "pebble_keys_range_key_sets_count", - metric.WithDescription("Approximate count of internal range key set keys"), - metric.WithUnit("{count}"), - ) - keysTombstoneCount, _ := meter.Int64Gauge( - "pebble_keys_tombstone_count", - metric.WithDescription("Approximate count of internal tombstones"), - metric.WithUnit("{count}"), - ) - keysMissizedTombstonesCount, _ := meter.Int64Counter( - "pebble_keys_missized_tombstones_count", - metric.WithDescription("Missized DELSIZED keys encountered by compactions"), - metric.WithUnit("{count}"), - ) - - snapshotCount, _ := meter.Int64Gauge( - "pebble_snapshot_count", - metric.WithDescription("Number of currently open snapshots"), - metric.WithUnit("{count}"), - ) - snapshotPinnedKeys, _ := meter.Int64Counter( - "pebble_snapshot_pinned_keys", - metric.WithDescription("Keys written that would've been elided without open snapshots"), - metric.WithUnit("{count}"), - ) - snapshotPinnedSize, _ := meter.Int64Counter( - "pebble_snapshot_pinned_size", - metric.WithDescription("Size of keys/values written due to open snapshots"), - metric.WithUnit("By"), - ) - snapshotEarliestSeqNum, _ := meter.Int64Gauge( - "pebble_snapshot_earliest_seq_num", - metric.WithDescription("Sequence number of earliest open snapshot"), - metric.WithUnit("{count}"), - ) - - tableIters, _ := meter.Int64Gauge( - "pebble_table_iters", - metric.WithDescription("Count of open sstable iterators"), - metric.WithUnit("{count}"), - ) - uptimeSeconds, _ := meter.Float64Gauge( - "pebble_uptime_seconds", - metric.WithDescription("Seconds since DB was opened"), - metric.WithUnit("s"), - ) - readAmp, _ := meter.Int64Gauge( - "pebble_read_amp", - metric.WithDescription("Read amplification"), - metric.WithUnit("{count}"), - ) - diskSpaceUsage, _ := meter.Int64Gauge( - "pebble_disk_space_usage", - metric.WithDescription("Total disk space used by the DB"), - metric.WithUnit("By"), - ) - - cacheHits, _ := meter.Int64Counter( - "pebble_cache_hits", - metric.WithDescription("Total number of cache hits"), - metric.WithUnit("{count}"), - ) - cacheMisses, _ := meter.Int64Counter( - "pebble_cache_misses", - metric.WithDescription("Total number of cache misses"), - metric.WithUnit("{count}"), - ) - cacheSize, _ := meter.Int64Gauge( - "pebble_cache_size", - metric.WithDescription("Current cache size"), - metric.WithUnit("By"), - ) - - batchSize, _ := meter.Int64Histogram( - "pebble_batch_size", - metric.WithDescription("Size of batches written to PebbleDB"), - metric.WithUnit("By"), - metric.WithExplicitBucketBoundaries(smetrics.ByteSizeBuckets...), - ) - pendingChangesQueueDepth, _ := meter.Int64Gauge( - "pebble_pending_changes_queue_depth", - metric.WithDescription("Number of pending changesets in async write queue"), - metric.WithUnit("{count}"), - ) - iteratorIterations, _ := meter.Float64Histogram( - "pebble_iterator_iterations", - metric.WithDescription("Number of iterations per iterator"), - metric.WithUnit("{count}"), - metric.WithExplicitBucketBoundaries(smetrics.CountBuckets...), - ) - - pm := &PebbleMetrics{ - db: db, databaseName: databaseName, - - getLatency: getLatency, - applyChangesetLatency: applyChangesetLatency, - applyChangesetAsyncLatency: applyChangesetAsyncLatency, - pruneLatency: pruneLatency, - importLatency: importLatency, - batchWriteLatency: batchWriteLatency, - - compactionCount: compactionCount, - compactionDuration: compactionDuration, - compactionBytesRead: compactionBytesRead, - compactionBytesWritten: compactionBytesWritten, - compactionEstimatedDebt: compactionEstimatedDebt, - compactionInProgressBytes: compactionInProgressBytes, - compactionNumInProgress: compactionNumInProgress, - compactionCancelledCount: compactionCancelledCount, - compactionCancelledBytes: compactionCancelledBytes, - compactionFailedCount: compactionFailedCount, - compactionDefaultCount: compactionDefaultCount, - compactionDeleteOnlyCount: compactionDeleteOnlyCount, - compactionElisionOnlyCount: compactionElisionOnlyCount, - compactionCopyCount: compactionCopyCount, - compactionMoveCount: compactionMoveCount, - compactionReadCount: compactionReadCount, - compactionTombstoneDensityCount: compactionTombstoneDensityCount, - compactionRewriteCount: compactionRewriteCount, - compactionMultiLevelCount: compactionMultiLevelCount, - compactionBlobFileRewriteCount: compactionBlobFileRewriteCount, - compactionCounterLevelCount: compactionCounterLevelCount, - compactionNumProblemSpans: compactionNumProblemSpans, - compactionMarkedFiles: compactionMarkedFiles, - - ingestCount: ingestCount, - - flushCount: flushCount, - flushDuration: flushDuration, - flushBytesWritten: flushBytesWritten, - flushNumInProgress: flushNumInProgress, - flushAsIngestCount: flushAsIngestCount, - flushAsIngestTableCount: flushAsIngestTableCount, - flushAsIngestBytes: flushAsIngestBytes, - flushIdleDuration: flushIdleDuration, - - filterHits: filterHits, - filterMisses: filterMisses, - - sstableCount: sstableCount, - sstableTotalSize: sstableTotalSize, - sstableSublevels: sstableSublevels, - sstableScore: sstableScore, - sstableFillFactor: sstableFillFactor, - sstableVirtualCount: sstableVirtualCount, - sstableVirtualSize: sstableVirtualSize, - sstableBytesIngested: sstableBytesIngested, - sstableBytesMoved: sstableBytesMoved, - sstableBytesRead: sstableBytesRead, - sstableBytesFlushed: sstableBytesFlushed, - sstableTablesCompacted: sstableTablesCompacted, - sstableTablesFlushed: sstableTablesFlushed, - sstableTablesIngested: sstableTablesIngested, - sstableTablesMoved: sstableTablesMoved, - sstableCompensatedFillFactor: sstableCompensatedFillFactor, - sstableEstimatedReferencesSize: sstableEstimatedReferencesSize, - sstableTablesDeleted: sstableTablesDeleted, - sstableTablesExcised: sstableTablesExcised, - sstableBlobBytesReadEstimate: sstableBlobBytesReadEstimate, - sstableBlobBytesCompacted: sstableBlobBytesCompacted, - sstableBlobBytesFlushed: sstableBlobBytesFlushed, - sstableMultiLevelBytesInTop: sstableMultiLevelBytesInTop, - sstableMultiLevelBytesIn: sstableMultiLevelBytesIn, - sstableMultiLevelBytesRead: sstableMultiLevelBytesRead, - sstableValueBlocksSize: sstableValueBlocksSize, - sstableBytesWrittenDataBlocks: sstableBytesWrittenDataBlocks, - sstableBytesWrittenValueBlocks: sstableBytesWrittenValueBlocks, - - memtableCount: memtableCount, - memtableTotalSize: memtableTotalSize, - memtableZombieSize: memtableZombieSize, - memtableZombieCount: memtableZombieCount, - - walSize: walSize, - walFiles: walFiles, - walObsoleteFiles: walObsoleteFiles, - walObsoletePhysicalSize: walObsoletePhysicalSize, - walPhysicalSize: walPhysicalSize, - walBytesIn: walBytesIn, - walBytesWritten: walBytesWritten, - - tableObsoleteSize: tableObsoleteSize, - tableObsoleteCount: tableObsoleteCount, - tableZombieSize: tableZombieSize, - tableZombieCount: tableZombieCount, - tableLiveSize: tableLiveSize, - tableLiveCount: tableLiveCount, - tableBackingCount: tableBackingCount, - tableBackingSize: tableBackingSize, - tableCompressedUnknown: tableCompressedUnknown, - tableCompressedSnappy: tableCompressedSnappy, - tableCompressedZstd: tableCompressedZstd, - tableCompressedMinLZ: tableCompressedMinLZ, - tableCompressedNone: tableCompressedNone, - tableLocalObsoleteSize: tableLocalObsoleteSize, - tableLocalObsoleteCount: tableLocalObsoleteCount, - tableLocalZombieSize: tableLocalZombieSize, - tableLocalZombieCount: tableLocalZombieCount, - tableGarbagePointDeletionsEstimate: tableGarbagePointDeletionsEstimate, - tableGarbageRangeDeletionsEstimate: tableGarbageRangeDeletionsEstimate, - tableInitialStatsComplete: tableInitialStatsComplete, - tablePendingStatsCount: tablePendingStatsCount, - blobFilesLiveCount: blobFilesLiveCount, - blobFilesLiveSize: blobFilesLiveSize, - blobFilesValueSize: blobFilesValueSize, - blobFilesReferencedValueSize: blobFilesReferencedValueSize, - blobFilesObsoleteCount: blobFilesObsoleteCount, - blobFilesObsoleteSize: blobFilesObsoleteSize, - blobFilesZombieCount: blobFilesZombieCount, - blobFilesZombieSize: blobFilesZombieSize, - blobFilesLocalLiveSize: blobFilesLocalLiveSize, - blobFilesLocalLiveCount: blobFilesLocalLiveCount, - blobFilesLocalObsoleteSize: blobFilesLocalObsoleteSize, - blobFilesLocalObsoleteCount: blobFilesLocalObsoleteCount, - blobFilesLocalZombieSize: blobFilesLocalZombieSize, - blobFilesLocalZombieCount: blobFilesLocalZombieCount, - fileCacheSize: fileCacheSize, - fileCacheTableCount: fileCacheTableCount, - fileCacheBlobFileCount: fileCacheBlobFileCount, - fileCacheHits: fileCacheHits, - fileCacheMisses: fileCacheMisses, - walFailoverDirSwitchCount: walFailoverDirSwitchCount, - walFailoverPrimaryDuration: walFailoverPrimaryDuration, - walFailoverSecondaryDuration: walFailoverSecondaryDuration, - numVirtual: numVirtual, - virtualSize: virtualSize, - remoteTablesCount: remoteTablesCount, - remoteTablesSize: remoteTablesSize, - - keysRangeKeySetsCount: keysRangeKeySetsCount, - keysTombstoneCount: keysTombstoneCount, - keysMissizedTombstonesCount: keysMissizedTombstonesCount, - - snapshotCount: snapshotCount, - snapshotPinnedKeys: snapshotPinnedKeys, - snapshotPinnedSize: snapshotPinnedSize, - snapshotEarliestSeqNum: snapshotEarliestSeqNum, - - tableIters: tableIters, - uptimeSeconds: uptimeSeconds, - readAmp: readAmp, - diskSpaceUsage: diskSpaceUsage, - - cacheHits: cacheHits, - cacheMisses: cacheMisses, - cacheSize: cacheSize, - - batchSize: batchSize, - pendingChangesQueueDepth: pendingChangesQueueDepth, - iteratorIterations: iteratorIterations, - } - - go pm.collectLoop(ctx, scrapeInterval) - return pm + dbAttr := attribute.String("db", databaseName) + p := &pebbleMetrics{meter: meter, dbAttrs: metric.WithAttributes(dbAttr)} + for level := range p.levelAttrs { + p.levelAttrs[level] = metric.WithAttributes(dbAttr, attribute.Int("level", level)) + } + p.declareDB() + p.declareLevels() + + p.snapshot.Store(db.Metrics()) + observe := func(_ context.Context, o metric.Observer) error { + m := p.snapshot.Load() + for _, report := range p.report { + report(o, m) + } + return nil + } + + reg, err := meter.RegisterCallback(observe, p.insts...) + if err != nil { + otel.Handle(err) + return func() {} + } + + ticker := time.NewTicker(refreshInterval) + stop, stopped := make(chan struct{}), make(chan struct{}) + go func() { + defer close(stopped) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + p.snapshot.Store(db.Metrics()) + } + } + }() + // Waiting for the refresher to exit before unregistering is what lets the + // caller close db as soon as this returns: no observation can be in flight. + return sync.OnceFunc(func() { + close(stop) + <-stopped + _ = reg.Unregister() + }) } -// collectLoop runs a ticker that periodically calls recordFromPebble. It exits when ctx is cancelled. -func (pm *PebbleMetrics) collectLoop(ctx context.Context, interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - pm.recordFromPebble(ctx) - } - } +// counter declares a whole-DB series whose value Pebble only ever raises. +func (p *pebbleMetrics) counter(name, unit, desc string, val func(*pebble.Metrics) float64) { + inst, _ := p.meter.Float64ObservableCounter(name, metric.WithDescription(desc), metric.WithUnit(unit)) + p.insts = append(p.insts, inst) + p.report = append(p.report, func(o metric.Observer, m *pebble.Metrics) { + o.ObserveFloat64(inst, val(m), p.dbAttrs) + }) } -func uint64ToInt64Clamped(v uint64) int64 { - if v > math.MaxInt64 { - return math.MaxInt64 - } - return int64(v) +// gauge declares a whole-DB series whose value can fall as well as rise. +func (p *pebbleMetrics) gauge(name, unit, desc string, val func(*pebble.Metrics) float64) { + inst, _ := p.meter.Float64ObservableGauge(name, metric.WithDescription(desc), metric.WithUnit(unit)) + p.insts = append(p.insts, inst) + p.report = append(p.report, func(o metric.Observer, m *pebble.Metrics) { + o.ObserveFloat64(inst, val(m), p.dbAttrs) + }) } -// addDelta computes the difference between current and prev, updates prev to current, -// and adds the positive delta to the counter. Used to convert cumulative scraped -// values into rate/counter increments. -func addDelta(ctx context.Context, counter metric.Int64Counter, current int64, prev *int64, opts ...metric.AddOption) { - delta := current - *prev - *prev = current - if delta > 0 { - counter.Add(ctx, delta, opts...) - } -} - -// recordFromPebble fetches the current metrics from the Pebble DB via Metrics(), then -// records compaction, flush, level, memtable, WAL, and cache metrics with the configured -// database name as the "db" attribute. -func (pm *PebbleMetrics) recordFromPebble(ctx context.Context) { - if pm.db == nil { - return - } - m := pm.db.Metrics() - dbAttr := attribute.String("db", pm.databaseName) - - if pm.compactionCount != nil { - addDelta(ctx, pm.compactionCount, m.Compact.Count, - &pm.prevCompactionCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionDuration != nil { - pm.compactionDuration.Record(ctx, m.Compact.Duration.Seconds(), metric.WithAttributes(dbAttr)) - } - if pm.compactionEstimatedDebt != nil { - pm.compactionEstimatedDebt.Record(ctx, - uint64ToInt64Clamped(m.Compact.EstimatedDebt), metric.WithAttributes(dbAttr)) - } - if pm.compactionInProgressBytes != nil { - pm.compactionInProgressBytes.Record(ctx, m.Compact.InProgressBytes, metric.WithAttributes(dbAttr)) - } - if pm.compactionNumInProgress != nil { - pm.compactionNumInProgress.Record(ctx, m.Compact.NumInProgress, metric.WithAttributes(dbAttr)) - } - if pm.compactionCancelledCount != nil { - addDelta(ctx, pm.compactionCancelledCount, m.Compact.CancelledCount, - &pm.prevCompactionCancelledCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionCancelledBytes != nil { - addDelta(ctx, pm.compactionCancelledBytes, m.Compact.CancelledBytes, - &pm.prevCompactionCancelledBytes, metric.WithAttributes(dbAttr)) - } - if pm.compactionFailedCount != nil { - addDelta(ctx, pm.compactionFailedCount, m.Compact.FailedCount, - &pm.prevCompactionFailedCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionDefaultCount != nil { - addDelta(ctx, pm.compactionDefaultCount, m.Compact.DefaultCount, - &pm.prevCompactionDefaultCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionDeleteOnlyCount != nil { - addDelta(ctx, pm.compactionDeleteOnlyCount, m.Compact.DeleteOnlyCount, - &pm.prevCompactionDeleteOnlyCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionElisionOnlyCount != nil { - addDelta(ctx, pm.compactionElisionOnlyCount, m.Compact.ElisionOnlyCount, - &pm.prevCompactionElisionOnlyCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionCopyCount != nil { - addDelta(ctx, pm.compactionCopyCount, m.Compact.CopyCount, - &pm.prevCompactionCopyCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionMoveCount != nil { - addDelta(ctx, pm.compactionMoveCount, m.Compact.MoveCount, - &pm.prevCompactionMoveCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionReadCount != nil { - addDelta(ctx, pm.compactionReadCount, m.Compact.ReadCount, - &pm.prevCompactionReadCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionTombstoneDensityCount != nil { - addDelta(ctx, pm.compactionTombstoneDensityCount, m.Compact.TombstoneDensityCount, - &pm.prevCompactionTombstoneDensityCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionRewriteCount != nil { - addDelta(ctx, pm.compactionRewriteCount, m.Compact.RewriteCount, - &pm.prevCompactionRewriteCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionMultiLevelCount != nil { - addDelta(ctx, pm.compactionMultiLevelCount, m.Compact.MultiLevelCount, - &pm.prevCompactionMultiLevelCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionBlobFileRewriteCount != nil { - addDelta(ctx, pm.compactionBlobFileRewriteCount, m.Compact.BlobFileRewriteCount, - &pm.prevCompactionBlobFileRewriteCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionCounterLevelCount != nil { - addDelta(ctx, pm.compactionCounterLevelCount, m.Compact.CounterLevelCount, - &pm.prevCompactionCounterLevelCount, metric.WithAttributes(dbAttr)) - } - if pm.compactionNumProblemSpans != nil { - pm.compactionNumProblemSpans.Record(ctx, int64(m.Compact.NumProblemSpans), metric.WithAttributes(dbAttr)) - } - if pm.compactionMarkedFiles != nil { - pm.compactionMarkedFiles.Record(ctx, int64(m.Compact.MarkedFiles), metric.WithAttributes(dbAttr)) - } - - if pm.ingestCount != nil { - addDelta(ctx, pm.ingestCount, uint64ToInt64Clamped(m.Ingest.Count), - &pm.prevIngestCount, metric.WithAttributes(dbAttr)) - } - - if pm.flushCount != nil { - addDelta(ctx, pm.flushCount, m.Flush.Count, &pm.prevFlushCount, metric.WithAttributes(dbAttr)) - } - if pm.flushDuration != nil { - pm.flushDuration.Record(ctx, - m.Flush.WriteThroughput.WorkDuration.Seconds(), metric.WithAttributes(dbAttr)) - } - if pm.flushBytesWritten != nil { - addDelta(ctx, pm.flushBytesWritten, m.Flush.WriteThroughput.Bytes, - &pm.prevFlushBytesWritten, metric.WithAttributes(dbAttr)) - } - if pm.flushNumInProgress != nil { - pm.flushNumInProgress.Record(ctx, m.Flush.NumInProgress, metric.WithAttributes(dbAttr)) - } - if pm.flushAsIngestCount != nil { - addDelta(ctx, pm.flushAsIngestCount, uint64ToInt64Clamped(m.Flush.AsIngestCount), - &pm.prevFlushAsIngestCount, metric.WithAttributes(dbAttr)) - } - if pm.flushAsIngestTableCount != nil { - addDelta(ctx, pm.flushAsIngestTableCount, uint64ToInt64Clamped(m.Flush.AsIngestTableCount), - &pm.prevFlushAsIngestTableCount, metric.WithAttributes(dbAttr)) - } - if pm.flushAsIngestBytes != nil { - addDelta(ctx, pm.flushAsIngestBytes, uint64ToInt64Clamped(m.Flush.AsIngestBytes), - &pm.prevFlushAsIngestBytes, metric.WithAttributes(dbAttr)) - } - if pm.flushIdleDuration != nil { - pm.flushIdleDuration.Record(ctx, - m.Flush.WriteThroughput.IdleDuration.Seconds(), metric.WithAttributes(dbAttr)) - } - - if pm.filterHits != nil { - addDelta(ctx, pm.filterHits, m.Filter.Hits, &pm.prevFilterHits, metric.WithAttributes(dbAttr)) - } - if pm.filterMisses != nil { - addDelta(ctx, pm.filterMisses, m.Filter.Misses, &pm.prevFilterMisses, metric.WithAttributes(dbAttr)) - } - - for level := 0; level < len(m.Levels); level++ { - lm := m.Levels[level] - levelAttr := attribute.Int("level", level) - attrs := metric.WithAttributes(dbAttr, levelAttr) - - // Grow prev slices if needed. - for level >= len(pm.prevCompactionBytesReadByLevel) { - pm.prevCompactionBytesReadByLevel = append(pm.prevCompactionBytesReadByLevel, 0) - pm.prevCompactionBytesWrittenByLevel = append(pm.prevCompactionBytesWrittenByLevel, 0) - pm.prevSstableBytesIngestedByLevel = append(pm.prevSstableBytesIngestedByLevel, 0) - pm.prevSstableBytesMovedByLevel = append(pm.prevSstableBytesMovedByLevel, 0) - pm.prevSstableBytesReadByLevel = append(pm.prevSstableBytesReadByLevel, 0) - pm.prevSstableBytesFlushedByLevel = append(pm.prevSstableBytesFlushedByLevel, 0) - pm.prevSstableTablesCompactedByLevel = append(pm.prevSstableTablesCompactedByLevel, 0) - pm.prevSstableTablesFlushedByLevel = append(pm.prevSstableTablesFlushedByLevel, 0) - pm.prevSstableTablesIngestedByLevel = append(pm.prevSstableTablesIngestedByLevel, 0) - pm.prevSstableTablesMovedByLevel = append(pm.prevSstableTablesMovedByLevel, 0) - pm.prevSstableTablesDeletedByLevel = append(pm.prevSstableTablesDeletedByLevel, 0) - pm.prevSstableTablesExcisedByLevel = append(pm.prevSstableTablesExcisedByLevel, 0) - pm.prevSstableBlobBytesReadEstimateByLevel = append(pm.prevSstableBlobBytesReadEstimateByLevel, 0) - pm.prevSstableBlobBytesCompactedByLevel = append(pm.prevSstableBlobBytesCompactedByLevel, 0) - pm.prevSstableBlobBytesFlushedByLevel = append(pm.prevSstableBlobBytesFlushedByLevel, 0) - pm.prevSstableMultiLevelBytesInTopByLevel = append(pm.prevSstableMultiLevelBytesInTopByLevel, 0) - pm.prevSstableMultiLevelBytesInByLevel = append(pm.prevSstableMultiLevelBytesInByLevel, 0) - pm.prevSstableMultiLevelBytesReadByLevel = append(pm.prevSstableMultiLevelBytesReadByLevel, 0) - pm.prevSstableBytesWrittenDataBlocksByLevel = append(pm.prevSstableBytesWrittenDataBlocksByLevel, 0) - pm.prevSstableBytesWrittenValueBlocksByLevel = append(pm.prevSstableBytesWrittenValueBlocksByLevel, 0) - } - - if pm.sstableCount != nil { - pm.sstableCount.Record(ctx, lm.TablesCount, attrs) - } - if pm.sstableTotalSize != nil { - pm.sstableTotalSize.Record(ctx, lm.TablesSize, attrs) - } - if pm.sstableSublevels != nil { - pm.sstableSublevels.Record(ctx, int64(lm.Sublevels), attrs) - } - if pm.sstableScore != nil { - pm.sstableScore.Record(ctx, lm.Score, attrs) - } - if pm.sstableFillFactor != nil { - pm.sstableFillFactor.Record(ctx, lm.FillFactor, attrs) - } - if pm.sstableVirtualCount != nil { - pm.sstableVirtualCount.Record(ctx, uint64ToInt64Clamped(lm.VirtualTablesCount), attrs) - } - if pm.sstableVirtualSize != nil { - pm.sstableVirtualSize.Record(ctx, uint64ToInt64Clamped(lm.VirtualTablesSize), attrs) - } - if pm.compactionBytesRead != nil { - addDelta(ctx, pm.compactionBytesRead, uint64ToInt64Clamped(lm.TableBytesIn), - &pm.prevCompactionBytesReadByLevel[level], attrs) +// levelCounter declares a per-level series whose value Pebble only ever raises. +func (p *pebbleMetrics) levelCounter(name, unit, desc string, val func(*pebble.LevelMetrics) float64) { + inst, _ := p.meter.Float64ObservableCounter(name, metric.WithDescription(desc), metric.WithUnit(unit)) + p.insts = append(p.insts, inst) + p.report = append(p.report, func(o metric.Observer, m *pebble.Metrics) { + for level := range m.Levels { + o.ObserveFloat64(inst, val(&m.Levels[level]), p.levelAttrs[level]) } - if pm.compactionBytesWritten != nil { - addDelta(ctx, pm.compactionBytesWritten, uint64ToInt64Clamped(lm.TableBytesCompacted), - &pm.prevCompactionBytesWrittenByLevel[level], attrs) - } - if pm.sstableBytesIngested != nil { - addDelta(ctx, pm.sstableBytesIngested, uint64ToInt64Clamped(lm.TableBytesIngested), - &pm.prevSstableBytesIngestedByLevel[level], attrs) - } - if pm.sstableBytesMoved != nil { - addDelta(ctx, pm.sstableBytesMoved, uint64ToInt64Clamped(lm.TableBytesMoved), - &pm.prevSstableBytesMovedByLevel[level], attrs) - } - if pm.sstableBytesRead != nil { - addDelta(ctx, pm.sstableBytesRead, uint64ToInt64Clamped(lm.TableBytesRead), - &pm.prevSstableBytesReadByLevel[level], attrs) - } - if pm.sstableBytesFlushed != nil { - addDelta(ctx, pm.sstableBytesFlushed, uint64ToInt64Clamped(lm.TableBytesFlushed), - &pm.prevSstableBytesFlushedByLevel[level], attrs) - } - if pm.sstableTablesCompacted != nil { - addDelta(ctx, pm.sstableTablesCompacted, uint64ToInt64Clamped(lm.TablesCompacted), - &pm.prevSstableTablesCompactedByLevel[level], attrs) - } - if pm.sstableTablesFlushed != nil { - addDelta(ctx, pm.sstableTablesFlushed, uint64ToInt64Clamped(lm.TablesFlushed), - &pm.prevSstableTablesFlushedByLevel[level], attrs) - } - if pm.sstableTablesIngested != nil { - addDelta(ctx, pm.sstableTablesIngested, uint64ToInt64Clamped(lm.TablesIngested), - &pm.prevSstableTablesIngestedByLevel[level], attrs) - } - if pm.sstableTablesMoved != nil { - addDelta(ctx, pm.sstableTablesMoved, uint64ToInt64Clamped(lm.TablesMoved), - &pm.prevSstableTablesMovedByLevel[level], attrs) - } - if pm.sstableCompensatedFillFactor != nil { - pm.sstableCompensatedFillFactor.Record(ctx, lm.CompensatedFillFactor, attrs) - } - if pm.sstableEstimatedReferencesSize != nil { - pm.sstableEstimatedReferencesSize.Record(ctx, uint64ToInt64Clamped(lm.EstimatedReferencesSize), attrs) - } - if pm.sstableTablesDeleted != nil { - addDelta(ctx, pm.sstableTablesDeleted, uint64ToInt64Clamped(lm.TablesDeleted), - &pm.prevSstableTablesDeletedByLevel[level], attrs) - } - if pm.sstableTablesExcised != nil { - addDelta(ctx, pm.sstableTablesExcised, uint64ToInt64Clamped(lm.TablesExcised), - &pm.prevSstableTablesExcisedByLevel[level], attrs) - } - if pm.sstableBlobBytesReadEstimate != nil { - addDelta(ctx, pm.sstableBlobBytesReadEstimate, uint64ToInt64Clamped(lm.BlobBytesReadEstimate), - &pm.prevSstableBlobBytesReadEstimateByLevel[level], attrs) - } - if pm.sstableBlobBytesCompacted != nil { - addDelta(ctx, pm.sstableBlobBytesCompacted, uint64ToInt64Clamped(lm.BlobBytesCompacted), - &pm.prevSstableBlobBytesCompactedByLevel[level], attrs) - } - if pm.sstableBlobBytesFlushed != nil { - addDelta(ctx, pm.sstableBlobBytesFlushed, uint64ToInt64Clamped(lm.BlobBytesFlushed), - &pm.prevSstableBlobBytesFlushedByLevel[level], attrs) - } - if pm.sstableMultiLevelBytesInTop != nil { - addDelta(ctx, pm.sstableMultiLevelBytesInTop, uint64ToInt64Clamped(lm.MultiLevel.TableBytesInTop), - &pm.prevSstableMultiLevelBytesInTopByLevel[level], attrs) - } - if pm.sstableMultiLevelBytesIn != nil { - addDelta(ctx, pm.sstableMultiLevelBytesIn, uint64ToInt64Clamped(lm.MultiLevel.TableBytesIn), - &pm.prevSstableMultiLevelBytesInByLevel[level], attrs) - } - if pm.sstableMultiLevelBytesRead != nil { - addDelta(ctx, pm.sstableMultiLevelBytesRead, uint64ToInt64Clamped(lm.MultiLevel.TableBytesRead), - &pm.prevSstableMultiLevelBytesReadByLevel[level], attrs) - } - if pm.sstableValueBlocksSize != nil { - pm.sstableValueBlocksSize.Record(ctx, uint64ToInt64Clamped(lm.Additional.ValueBlocksSize), attrs) - } - if pm.sstableBytesWrittenDataBlocks != nil { - addDelta(ctx, pm.sstableBytesWrittenDataBlocks, - uint64ToInt64Clamped(lm.Additional.BytesWrittenDataBlocks), - &pm.prevSstableBytesWrittenDataBlocksByLevel[level], attrs) - } - if pm.sstableBytesWrittenValueBlocks != nil { - addDelta(ctx, pm.sstableBytesWrittenValueBlocks, - uint64ToInt64Clamped(lm.Additional.BytesWrittenValueBlocks), - &pm.prevSstableBytesWrittenValueBlocksByLevel[level], attrs) - } - } - - if pm.memtableCount != nil { - pm.memtableCount.Record(ctx, m.MemTable.Count, metric.WithAttributes(dbAttr)) - } - if pm.memtableTotalSize != nil { - pm.memtableTotalSize.Record(ctx, uint64ToInt64Clamped(m.MemTable.Size), metric.WithAttributes(dbAttr)) - } - if pm.memtableZombieSize != nil { - pm.memtableZombieSize.Record(ctx, - uint64ToInt64Clamped(m.MemTable.ZombieSize), metric.WithAttributes(dbAttr)) - } - if pm.memtableZombieCount != nil { - pm.memtableZombieCount.Record(ctx, m.MemTable.ZombieCount, metric.WithAttributes(dbAttr)) - } - - if pm.walSize != nil { - pm.walSize.Record(ctx, uint64ToInt64Clamped(m.WAL.Size), metric.WithAttributes(dbAttr)) - } - if pm.walFiles != nil { - pm.walFiles.Record(ctx, m.WAL.Files, metric.WithAttributes(dbAttr)) - } - if pm.walObsoleteFiles != nil { - pm.walObsoleteFiles.Record(ctx, m.WAL.ObsoleteFiles, metric.WithAttributes(dbAttr)) - } - if pm.walObsoletePhysicalSize != nil { - pm.walObsoletePhysicalSize.Record(ctx, - uint64ToInt64Clamped(m.WAL.ObsoletePhysicalSize), metric.WithAttributes(dbAttr)) - } - if pm.walPhysicalSize != nil { - pm.walPhysicalSize.Record(ctx, uint64ToInt64Clamped(m.WAL.PhysicalSize), metric.WithAttributes(dbAttr)) - } - if pm.walBytesIn != nil { - addDelta(ctx, pm.walBytesIn, - uint64ToInt64Clamped(m.WAL.BytesIn), &pm.prevWalBytesIn, metric.WithAttributes(dbAttr)) - } - if pm.walBytesWritten != nil { - addDelta(ctx, pm.walBytesWritten, uint64ToInt64Clamped(m.WAL.BytesWritten), - &pm.prevWalBytesWritten, metric.WithAttributes(dbAttr)) - } + }) +} - if pm.tableObsoleteSize != nil { - pm.tableObsoleteSize.Record(ctx, - uint64ToInt64Clamped(m.Table.ObsoleteSize), metric.WithAttributes(dbAttr)) - } - if pm.tableObsoleteCount != nil { - pm.tableObsoleteCount.Record(ctx, m.Table.ObsoleteCount, metric.WithAttributes(dbAttr)) - } - if pm.tableZombieSize != nil { - pm.tableZombieSize.Record(ctx, - uint64ToInt64Clamped(m.Table.ZombieSize), metric.WithAttributes(dbAttr)) - } - if pm.tableZombieCount != nil { - pm.tableZombieCount.Record(ctx, m.Table.ZombieCount, metric.WithAttributes(dbAttr)) - } - if pm.tableLiveSize != nil { - pm.tableLiveSize.Record(ctx, - uint64ToInt64Clamped(m.Table.Local.LiveSize), metric.WithAttributes(dbAttr)) - } - if pm.tableLiveCount != nil { - pm.tableLiveCount.Record(ctx, - uint64ToInt64Clamped(m.Table.Local.LiveCount), metric.WithAttributes(dbAttr)) - } - if pm.tableBackingCount != nil { - pm.tableBackingCount.Record(ctx, - uint64ToInt64Clamped(m.Table.BackingTableCount), metric.WithAttributes(dbAttr)) - } - if pm.tableBackingSize != nil { - pm.tableBackingSize.Record(ctx, uint64ToInt64Clamped(m.Table.BackingTableSize), metric.WithAttributes(dbAttr)) - } - if pm.tableCompressedUnknown != nil { - pm.tableCompressedUnknown.Record(ctx, m.Table.CompressedCountUnknown, metric.WithAttributes(dbAttr)) - } - if pm.tableCompressedSnappy != nil { - pm.tableCompressedSnappy.Record(ctx, m.Table.CompressedCountSnappy, metric.WithAttributes(dbAttr)) - } - if pm.tableCompressedZstd != nil { - pm.tableCompressedZstd.Record(ctx, m.Table.CompressedCountZstd, metric.WithAttributes(dbAttr)) - } - if pm.tableCompressedMinLZ != nil { - pm.tableCompressedMinLZ.Record(ctx, m.Table.CompressedCountMinLZ, metric.WithAttributes(dbAttr)) - } - if pm.tableCompressedNone != nil { - pm.tableCompressedNone.Record(ctx, m.Table.CompressedCountNone, metric.WithAttributes(dbAttr)) - } - if pm.tableLocalObsoleteSize != nil { - pm.tableLocalObsoleteSize.Record(ctx, - uint64ToInt64Clamped(m.Table.Local.ObsoleteSize), metric.WithAttributes(dbAttr)) - } - if pm.tableLocalObsoleteCount != nil { - pm.tableLocalObsoleteCount.Record(ctx, - uint64ToInt64Clamped(m.Table.Local.ObsoleteCount), metric.WithAttributes(dbAttr)) - } - if pm.tableLocalZombieSize != nil { - pm.tableLocalZombieSize.Record(ctx, - uint64ToInt64Clamped(m.Table.Local.ZombieSize), metric.WithAttributes(dbAttr)) - } - if pm.tableLocalZombieCount != nil { - pm.tableLocalZombieCount.Record(ctx, - uint64ToInt64Clamped(m.Table.Local.ZombieCount), metric.WithAttributes(dbAttr)) - } - if pm.tableGarbagePointDeletionsEstimate != nil { - pm.tableGarbagePointDeletionsEstimate.Record(ctx, - uint64ToInt64Clamped(m.Table.Garbage.PointDeletionsBytesEstimate), metric.WithAttributes(dbAttr)) - } - if pm.tableGarbageRangeDeletionsEstimate != nil { - pm.tableGarbageRangeDeletionsEstimate.Record(ctx, - uint64ToInt64Clamped(m.Table.Garbage.RangeDeletionsBytesEstimate), metric.WithAttributes(dbAttr)) - } - if pm.tableInitialStatsComplete != nil { - v := int64(0) - if m.Table.InitialStatsCollectionComplete { - v = 1 +// levelGauge declares a per-level series whose value can fall as well as rise. +func (p *pebbleMetrics) levelGauge(name, unit, desc string, val func(*pebble.LevelMetrics) float64) { + inst, _ := p.meter.Float64ObservableGauge(name, metric.WithDescription(desc), metric.WithUnit(unit)) + p.insts = append(p.insts, inst) + p.report = append(p.report, func(o metric.Observer, m *pebble.Metrics) { + for level := range m.Levels { + o.ObserveFloat64(inst, val(&m.Levels[level]), p.levelAttrs[level]) } - pm.tableInitialStatsComplete.Record(ctx, v, metric.WithAttributes(dbAttr)) - } - if pm.tablePendingStatsCount != nil { - pm.tablePendingStatsCount.Record(ctx, m.Table.PendingStatsCollectionCount, metric.WithAttributes(dbAttr)) - } - if pm.blobFilesLiveCount != nil { - pm.blobFilesLiveCount.Record(ctx, uint64ToInt64Clamped(m.BlobFiles.LiveCount), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesLiveSize != nil { - pm.blobFilesLiveSize.Record(ctx, uint64ToInt64Clamped(m.BlobFiles.LiveSize), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesValueSize != nil { - pm.blobFilesValueSize.Record(ctx, uint64ToInt64Clamped(m.BlobFiles.ValueSize), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesReferencedValueSize != nil { - pm.blobFilesReferencedValueSize.Record(ctx, - uint64ToInt64Clamped(m.BlobFiles.ReferencedValueSize), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesObsoleteCount != nil { - pm.blobFilesObsoleteCount.Record(ctx, - uint64ToInt64Clamped(m.BlobFiles.ObsoleteCount), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesObsoleteSize != nil { - pm.blobFilesObsoleteSize.Record(ctx, - uint64ToInt64Clamped(m.BlobFiles.ObsoleteSize), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesZombieCount != nil { - pm.blobFilesZombieCount.Record(ctx, - uint64ToInt64Clamped(m.BlobFiles.ZombieCount), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesZombieSize != nil { - pm.blobFilesZombieSize.Record(ctx, - uint64ToInt64Clamped(m.BlobFiles.ZombieSize), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesLocalLiveSize != nil { - pm.blobFilesLocalLiveSize.Record(ctx, - uint64ToInt64Clamped(m.BlobFiles.Local.LiveSize), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesLocalLiveCount != nil { - pm.blobFilesLocalLiveCount.Record(ctx, - uint64ToInt64Clamped(m.BlobFiles.Local.LiveCount), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesLocalObsoleteSize != nil { - pm.blobFilesLocalObsoleteSize.Record(ctx, - uint64ToInt64Clamped(m.BlobFiles.Local.ObsoleteSize), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesLocalObsoleteCount != nil { - pm.blobFilesLocalObsoleteCount.Record(ctx, - uint64ToInt64Clamped(m.BlobFiles.Local.ObsoleteCount), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesLocalZombieSize != nil { - pm.blobFilesLocalZombieSize.Record(ctx, - uint64ToInt64Clamped(m.BlobFiles.Local.ZombieSize), metric.WithAttributes(dbAttr)) - } - if pm.blobFilesLocalZombieCount != nil { - pm.blobFilesLocalZombieCount.Record(ctx, - uint64ToInt64Clamped(m.BlobFiles.Local.ZombieCount), metric.WithAttributes(dbAttr)) - } - if pm.fileCacheSize != nil { - pm.fileCacheSize.Record(ctx, m.FileCache.Size, metric.WithAttributes(dbAttr)) - } - if pm.fileCacheTableCount != nil { - pm.fileCacheTableCount.Record(ctx, m.FileCache.TableCount, metric.WithAttributes(dbAttr)) - } - if pm.fileCacheBlobFileCount != nil { - pm.fileCacheBlobFileCount.Record(ctx, m.FileCache.BlobFileCount, metric.WithAttributes(dbAttr)) - } - if pm.fileCacheHits != nil { - addDelta(ctx, pm.fileCacheHits, m.FileCache.Hits, &pm.prevFileCacheHits, metric.WithAttributes(dbAttr)) - } - if pm.fileCacheMisses != nil { - addDelta(ctx, pm.fileCacheMisses, - m.FileCache.Misses, &pm.prevFileCacheMisses, metric.WithAttributes(dbAttr)) - } - if pm.walFailoverDirSwitchCount != nil { - addDelta(ctx, pm.walFailoverDirSwitchCount, m.WAL.Failover.DirSwitchCount, - &pm.prevWalFailoverDirSwitchCount, metric.WithAttributes(dbAttr)) - } - if pm.walFailoverPrimaryDuration != nil { - pm.walFailoverPrimaryDuration.Record(ctx, - m.WAL.Failover.PrimaryWriteDuration.Seconds(), metric.WithAttributes(dbAttr)) - } - if pm.walFailoverSecondaryDuration != nil { - pm.walFailoverSecondaryDuration.Record(ctx, - m.WAL.Failover.SecondaryWriteDuration.Seconds(), metric.WithAttributes(dbAttr)) - } - if pm.numVirtual != nil { - pm.numVirtual.Record(ctx, uint64ToInt64Clamped(m.NumVirtual()), metric.WithAttributes(dbAttr)) - } - if pm.virtualSize != nil { - pm.virtualSize.Record(ctx, uint64ToInt64Clamped(m.VirtualSize()), metric.WithAttributes(dbAttr)) - } - rtCount, rtSize := m.RemoteTablesTotal() - if pm.remoteTablesCount != nil { - pm.remoteTablesCount.Record(ctx, uint64ToInt64Clamped(rtCount), metric.WithAttributes(dbAttr)) - } - if pm.remoteTablesSize != nil { - pm.remoteTablesSize.Record(ctx, uint64ToInt64Clamped(rtSize), metric.WithAttributes(dbAttr)) - } - - if pm.keysRangeKeySetsCount != nil { - pm.keysRangeKeySetsCount.Record(ctx, - uint64ToInt64Clamped(m.Keys.RangeKeySetsCount), metric.WithAttributes(dbAttr)) - } - if pm.keysTombstoneCount != nil { - pm.keysTombstoneCount.Record(ctx, - uint64ToInt64Clamped(m.Keys.TombstoneCount), metric.WithAttributes(dbAttr)) - } - if pm.keysMissizedTombstonesCount != nil { - addDelta(ctx, pm.keysMissizedTombstonesCount, uint64ToInt64Clamped(m.Keys.MissizedTombstonesCount), - &pm.prevKeysMissizedTombstonesCount, metric.WithAttributes(dbAttr)) - } - - if pm.snapshotCount != nil { - pm.snapshotCount.Record(ctx, int64(m.Snapshots.Count), metric.WithAttributes(dbAttr)) - } - if pm.snapshotPinnedKeys != nil { - addDelta(ctx, pm.snapshotPinnedKeys, uint64ToInt64Clamped(m.Snapshots.PinnedKeys), - &pm.prevSnapshotPinnedKeys, metric.WithAttributes(dbAttr)) - } - if pm.snapshotPinnedSize != nil { - addDelta(ctx, pm.snapshotPinnedSize, uint64ToInt64Clamped(m.Snapshots.PinnedSize), - &pm.prevSnapshotPinnedSize, metric.WithAttributes(dbAttr)) - } - if pm.snapshotEarliestSeqNum != nil { - pm.snapshotEarliestSeqNum.Record(ctx, - uint64ToInt64Clamped(uint64(m.Snapshots.EarliestSeqNum)), metric.WithAttributes(dbAttr)) - } + }) +} - if pm.tableIters != nil { - pm.tableIters.Record(ctx, m.TableIters, metric.WithAttributes(dbAttr)) - } - if pm.uptimeSeconds != nil { - pm.uptimeSeconds.Record(ctx, m.Uptime.Seconds(), metric.WithAttributes(dbAttr)) - } - if pm.readAmp != nil { - pm.readAmp.Record(ctx, int64(m.ReadAmp()), metric.WithAttributes(dbAttr)) - } - if pm.diskSpaceUsage != nil { - pm.diskSpaceUsage.Record(ctx, uint64ToInt64Clamped(m.DiskSpaceUsage()), metric.WithAttributes(dbAttr)) - } +// declareDB declares the series read from a whole-DB snapshot. +func (p *pebbleMetrics) declareDB() { + p.counter("pebble_compaction_count", "{count}", "Total number of compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.Count) }) + p.counter("pebble_compaction_duration", "s", "Cumulative compaction duration since DB open", + func(m *pebble.Metrics) float64 { return m.Compact.Duration.Seconds() }) + p.gauge("pebble_compaction_estimated_debt", "By", "Estimated bytes to compact for LSM to reach stable state", + func(m *pebble.Metrics) float64 { return float64(m.Compact.EstimatedDebt) }) + p.gauge("pebble_compaction_in_progress_bytes", "By", "Bytes in sstables being written by in-progress compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.InProgressBytes) }) + p.gauge("pebble_compaction_num_in_progress", "{count}", "Number of compactions in progress", + func(m *pebble.Metrics) float64 { return float64(m.Compact.NumInProgress) }) + p.counter("pebble_compaction_cancelled_count", "{count}", "Number of compactions that were cancelled", + func(m *pebble.Metrics) float64 { return float64(m.Compact.CancelledCount) }) + p.counter("pebble_compaction_cancelled_bytes", "By", "Bytes written by cancelled compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.CancelledBytes) }) + p.counter("pebble_compaction_failed_count", "{count}", "Number of compactions that hit an error", + func(m *pebble.Metrics) float64 { return float64(m.Compact.FailedCount) }) + p.counter("pebble_compaction_default_count", "{count}", "Default compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.DefaultCount) }) + p.counter("pebble_compaction_delete_only_count", "{count}", "Delete-only compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.DeleteOnlyCount) }) + p.counter("pebble_compaction_elision_only_count", "{count}", "Elision-only compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.ElisionOnlyCount) }) + p.counter("pebble_compaction_copy_count", "{count}", "Copy compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.CopyCount) }) + p.counter("pebble_compaction_move_count", "{count}", "Move compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.MoveCount) }) + p.counter("pebble_compaction_read_count", "{count}", "Read compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.ReadCount) }) + p.counter("pebble_compaction_tombstone_density_count", "{count}", "Tombstone-density compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.TombstoneDensityCount) }) + p.counter("pebble_compaction_rewrite_count", "{count}", "Rewrite compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.RewriteCount) }) + p.counter("pebble_compaction_multi_level_count", "{count}", "Multi-level compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.MultiLevelCount) }) + p.counter("pebble_compaction_blob_file_rewrite_count", "{count}", "Blob file rewrite compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.BlobFileRewriteCount) }) + p.counter("pebble_compaction_counter_level_count", "{count}", "Counter-level compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.CounterLevelCount) }) + p.gauge("pebble_compaction_num_problem_spans", "{count}", "Problem spans blocking compactions", + func(m *pebble.Metrics) float64 { return float64(m.Compact.NumProblemSpans) }) + p.gauge("pebble_compaction_marked_files", "{count}", "Files marked for compaction", + func(m *pebble.Metrics) float64 { return float64(m.Compact.MarkedFiles) }) + + p.counter("pebble_ingest_count", "{count}", "Total number of ingestions", + func(m *pebble.Metrics) float64 { return float64(m.Ingest.Count) }) + + p.counter("pebble_flush_count", "{count}", "Total number of memtable flushes", + func(m *pebble.Metrics) float64 { return float64(m.Flush.Count) }) + p.counter("pebble_flush_duration", "s", "Cumulative memtable flush work duration since DB open", + func(m *pebble.Metrics) float64 { return m.Flush.WriteThroughput.WorkDuration.Seconds() }) + p.counter("pebble_flush_bytes_written", "By", "Total bytes written during memtable flushes", + func(m *pebble.Metrics) float64 { return float64(m.Flush.WriteThroughput.Bytes) }) + p.gauge("pebble_flush_num_in_progress", "{count}", "Number of flushes in progress", + func(m *pebble.Metrics) float64 { return float64(m.Flush.NumInProgress) }) + p.counter("pebble_flush_as_ingest_count", "{count}", "Flush operations handling ingested tables", + func(m *pebble.Metrics) float64 { return float64(m.Flush.AsIngestCount) }) + p.counter("pebble_flush_as_ingest_table_count", "{count}", "Tables ingested as flushables", + func(m *pebble.Metrics) float64 { return float64(m.Flush.AsIngestTableCount) }) + p.counter("pebble_flush_as_ingest_bytes", "By", "Bytes flushed for flushables from ingestion", + func(m *pebble.Metrics) float64 { return float64(m.Flush.AsIngestBytes) }) + p.gauge("pebble_flush_idle_duration", "s", "Idle duration before memtable flushes", + func(m *pebble.Metrics) float64 { return m.Flush.WriteThroughput.IdleDuration.Seconds() }) + + p.counter("pebble_filter_hits", "{count}", "Bloom filter hits (block reads avoided)", + func(m *pebble.Metrics) float64 { return float64(m.Filter.Hits) }) + p.counter("pebble_filter_misses", "{count}", "Bloom filter misses", + func(m *pebble.Metrics) float64 { return float64(m.Filter.Misses) }) + + p.gauge("pebble_memtable_count", "{count}", "Current number of memtables", + func(m *pebble.Metrics) float64 { return float64(m.MemTable.Count) }) + p.gauge("pebble_memtable_total_size", "By", "Total size of all memtables", + func(m *pebble.Metrics) float64 { return float64(m.MemTable.Size) }) + p.gauge("pebble_memtable_zombie_size", "By", "Bytes in zombie memtables (released but in use by iterators)", + func(m *pebble.Metrics) float64 { return float64(m.MemTable.ZombieSize) }) + p.gauge("pebble_memtable_zombie_count", "{count}", "Count of zombie memtables", + func(m *pebble.Metrics) float64 { return float64(m.MemTable.ZombieCount) }) + + p.gauge("pebble_wal_size", "By", "Current size of Write-Ahead Log", + func(m *pebble.Metrics) float64 { return float64(m.WAL.Size) }) + p.gauge("pebble_wal_files", "{count}", "Number of live WAL files", + func(m *pebble.Metrics) float64 { return float64(m.WAL.Files) }) + p.gauge("pebble_wal_obsolete_files", "{count}", "Number of obsolete WAL files", + func(m *pebble.Metrics) float64 { return float64(m.WAL.ObsoleteFiles) }) + p.gauge("pebble_wal_obsolete_physical_size", "By", "Physical size of obsolete WAL files", + func(m *pebble.Metrics) float64 { return float64(m.WAL.ObsoletePhysicalSize) }) + p.gauge("pebble_wal_physical_size", "By", "Physical size of WAL files on disk", + func(m *pebble.Metrics) float64 { return float64(m.WAL.PhysicalSize) }) + p.counter("pebble_wal_bytes_in", "By", "Logical bytes written to WAL", + func(m *pebble.Metrics) float64 { return float64(m.WAL.BytesIn) }) + p.counter("pebble_wal_bytes_written", "By", "Bytes written to WAL", + func(m *pebble.Metrics) float64 { return float64(m.WAL.BytesWritten) }) + p.counter("pebble_wal_failover_dir_switch_count", "{count}", "WAL directory switches (failover/failback)", + func(m *pebble.Metrics) float64 { return float64(m.WAL.Failover.DirSwitchCount) }) + p.gauge("pebble_wal_failover_primary_duration", "s", "Cumulative WAL write duration on primary", + func(m *pebble.Metrics) float64 { return m.WAL.Failover.PrimaryWriteDuration.Seconds() }) + p.gauge("pebble_wal_failover_secondary_duration", "s", "Cumulative WAL write duration on secondary", + func(m *pebble.Metrics) float64 { return m.WAL.Failover.SecondaryWriteDuration.Seconds() }) + + p.gauge("pebble_table_obsolete_size", "By", "Bytes in obsolete tables no longer referenced", + func(m *pebble.Metrics) float64 { return float64(m.Table.ObsoleteSize) }) + p.gauge("pebble_table_obsolete_count", "{count}", "Count of obsolete tables", + func(m *pebble.Metrics) float64 { return float64(m.Table.ObsoleteCount) }) + p.gauge("pebble_table_zombie_size", "By", "Bytes in zombie tables (released but in use by iterators)", + func(m *pebble.Metrics) float64 { return float64(m.Table.ZombieSize) }) + p.gauge("pebble_table_zombie_count", "{count}", "Count of zombie tables", + func(m *pebble.Metrics) float64 { return float64(m.Table.ZombieCount) }) + p.gauge("pebble_table_live_size", "By", "Bytes in live tables", + func(m *pebble.Metrics) float64 { return float64(m.Table.Local.LiveSize) }) + p.gauge("pebble_table_live_count", "{count}", "Count of live tables", + func(m *pebble.Metrics) float64 { return float64(m.Table.Local.LiveCount) }) + p.gauge("pebble_table_backing_count", "{count}", "Sstables backing virtual tables", + func(m *pebble.Metrics) float64 { return float64(m.Table.BackingTableCount) }) + p.gauge("pebble_table_backing_size", "By", "Size of sstables backing virtual tables", + func(m *pebble.Metrics) float64 { return float64(m.Table.BackingTableSize) }) + p.gauge("pebble_table_compressed_unknown", "{count}", "Sstables with unknown compression", + func(m *pebble.Metrics) float64 { return float64(m.Table.CompressedCountUnknown) }) + p.gauge("pebble_table_compressed_snappy", "{count}", "Snappy-compressed sstables", + func(m *pebble.Metrics) float64 { return float64(m.Table.CompressedCountSnappy) }) + p.gauge("pebble_table_compressed_zstd", "{count}", "Zstd-compressed sstables", + func(m *pebble.Metrics) float64 { return float64(m.Table.CompressedCountZstd) }) + p.gauge("pebble_table_compressed_minlz", "{count}", "MinLZ-compressed sstables", + func(m *pebble.Metrics) float64 { return float64(m.Table.CompressedCountMinLZ) }) + p.gauge("pebble_table_compressed_none", "{count}", "Uncompressed sstables", + func(m *pebble.Metrics) float64 { return float64(m.Table.CompressedCountNone) }) + p.gauge("pebble_table_local_obsolete_size", "By", "Local obsolete table size", + func(m *pebble.Metrics) float64 { return float64(m.Table.Local.ObsoleteSize) }) + p.gauge("pebble_table_local_obsolete_count", "{count}", "Local obsolete table count", + func(m *pebble.Metrics) float64 { return float64(m.Table.Local.ObsoleteCount) }) + p.gauge("pebble_table_local_zombie_size", "By", "Local zombie table size", + func(m *pebble.Metrics) float64 { return float64(m.Table.Local.ZombieSize) }) + p.gauge("pebble_table_local_zombie_count", "{count}", "Local zombie table count", + func(m *pebble.Metrics) float64 { return float64(m.Table.Local.ZombieCount) }) + p.gauge("pebble_table_garbage_point_deletions_estimate", "By", "Est. bytes reclaimable from point deletes", + func(m *pebble.Metrics) float64 { return float64(m.Table.Garbage.PointDeletionsBytesEstimate) }) + p.gauge("pebble_table_garbage_range_deletions_estimate", "By", "Est. bytes reclaimable from range deletes", + func(m *pebble.Metrics) float64 { return float64(m.Table.Garbage.RangeDeletionsBytesEstimate) }) + p.gauge("pebble_table_initial_stats_complete", "1", "1 if initial stats collection complete", + func(m *pebble.Metrics) float64 { + if m.Table.InitialStatsCollectionComplete { + return 1 + } + return 0 + }) + p.gauge("pebble_table_pending_stats_count", "{count}", "New sstables awaiting stats collection", + func(m *pebble.Metrics) float64 { return float64(m.Table.PendingStatsCollectionCount) }) + + p.gauge("pebble_blob_files_live_count", "{count}", "Live blob file count", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.LiveCount) }) + p.gauge("pebble_blob_files_live_size", "By", "Live blob file physical size", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.LiveSize) }) + p.gauge("pebble_blob_files_value_size", "By", "Uncompressed value size in live blobs", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.ValueSize) }) + p.gauge("pebble_blob_files_referenced_value_size", "By", "Referenced value size in live blobs", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.ReferencedValueSize) }) + p.gauge("pebble_blob_files_obsolete_count", "{count}", "Obsolete blob file count", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.ObsoleteCount) }) + p.gauge("pebble_blob_files_obsolete_size", "By", "Obsolete blob file size", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.ObsoleteSize) }) + p.gauge("pebble_blob_files_zombie_count", "{count}", "Zombie blob file count", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.ZombieCount) }) + p.gauge("pebble_blob_files_zombie_size", "By", "Zombie blob file size", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.ZombieSize) }) + p.gauge("pebble_blob_files_local_live_size", "By", "Local live blob file size", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.Local.LiveSize) }) + p.gauge("pebble_blob_files_local_live_count", "{count}", "Local live blob file count", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.Local.LiveCount) }) + p.gauge("pebble_blob_files_local_obsolete_size", "By", "Local obsolete blob file size", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.Local.ObsoleteSize) }) + p.gauge("pebble_blob_files_local_obsolete_count", "{count}", "Local obsolete blob file count", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.Local.ObsoleteCount) }) + p.gauge("pebble_blob_files_local_zombie_size", "By", "Local zombie blob file size", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.Local.ZombieSize) }) + p.gauge("pebble_blob_files_local_zombie_count", "{count}", "Local zombie blob file count", + func(m *pebble.Metrics) float64 { return float64(m.BlobFiles.Local.ZombieCount) }) + + p.gauge("pebble_file_cache_size", "By", "Bytes in file cache", + func(m *pebble.Metrics) float64 { return float64(m.FileCache.Size) }) + p.gauge("pebble_file_cache_table_count", "{count}", "Tables in file cache", + func(m *pebble.Metrics) float64 { return float64(m.FileCache.TableCount) }) + p.gauge("pebble_file_cache_blob_file_count", "{count}", "Blob files in file cache", + func(m *pebble.Metrics) float64 { return float64(m.FileCache.BlobFileCount) }) + p.counter("pebble_file_cache_hits", "{count}", "File cache hits", + func(m *pebble.Metrics) float64 { return float64(m.FileCache.Hits) }) + p.counter("pebble_file_cache_misses", "{count}", "File cache misses", + func(m *pebble.Metrics) float64 { return float64(m.FileCache.Misses) }) + + p.gauge("pebble_num_virtual", "{count}", "Total virtual sstable count", + func(m *pebble.Metrics) float64 { return float64(m.NumVirtual()) }) + p.gauge("pebble_virtual_size", "By", "Total virtual sstable size", + func(m *pebble.Metrics) float64 { return float64(m.VirtualSize()) }) + p.gauge("pebble_remote_tables_count", "{count}", "Remote tables count", + func(m *pebble.Metrics) float64 { + count, _ := m.RemoteTablesTotal() + return float64(count) + }) + p.gauge("pebble_remote_tables_size", "By", "Remote tables size", + func(m *pebble.Metrics) float64 { + _, size := m.RemoteTablesTotal() + return float64(size) + }) + + p.gauge("pebble_keys_range_key_sets_count", "{count}", "Approximate count of internal range key set keys", + func(m *pebble.Metrics) float64 { return float64(m.Keys.RangeKeySetsCount) }) + p.gauge("pebble_keys_tombstone_count", "{count}", "Approximate count of internal tombstones", + func(m *pebble.Metrics) float64 { return float64(m.Keys.TombstoneCount) }) + p.counter("pebble_keys_missized_tombstones_count", "{count}", "Missized DELSIZED keys encountered by compactions", + func(m *pebble.Metrics) float64 { return float64(m.Keys.MissizedTombstonesCount) }) + + p.gauge("pebble_snapshot_count", "{count}", "Number of currently open snapshots", + func(m *pebble.Metrics) float64 { return float64(m.Snapshots.Count) }) + p.counter("pebble_snapshot_pinned_keys", "{count}", "Keys written that would've been elided without open snapshots", + func(m *pebble.Metrics) float64 { return float64(m.Snapshots.PinnedKeys) }) + p.counter("pebble_snapshot_pinned_size", "By", "Size of keys/values written due to open snapshots", + func(m *pebble.Metrics) float64 { return float64(m.Snapshots.PinnedSize) }) + p.gauge("pebble_snapshot_earliest_seq_num", "{count}", "Sequence number of earliest open snapshot", + func(m *pebble.Metrics) float64 { return float64(m.Snapshots.EarliestSeqNum) }) + + p.gauge("pebble_table_iters", "{count}", "Count of open sstable iterators", + func(m *pebble.Metrics) float64 { return float64(m.TableIters) }) + p.gauge("pebble_uptime_seconds", "s", "Seconds since DB was opened", + func(m *pebble.Metrics) float64 { return m.Uptime.Seconds() }) + p.gauge("pebble_read_amp", "{count}", "Read amplification", + func(m *pebble.Metrics) float64 { return float64(m.ReadAmp()) }) + p.gauge("pebble_disk_space_usage", "By", "Total disk space used by the DB", + func(m *pebble.Metrics) float64 { return float64(m.DiskSpaceUsage()) }) + + p.counter("pebble_cache_hits", "{count}", "Total number of cache hits", + func(m *pebble.Metrics) float64 { return float64(m.BlockCache.Hits) }) + p.counter("pebble_cache_misses", "{count}", "Total number of cache misses", + func(m *pebble.Metrics) float64 { return float64(m.BlockCache.Misses) }) + p.gauge("pebble_cache_size", "By", "Current cache size", + func(m *pebble.Metrics) float64 { return float64(m.BlockCache.Size) }) +} - if pm.cacheHits != nil { - addDelta(ctx, pm.cacheHits, m.BlockCache.Hits, &pm.prevCacheHits, metric.WithAttributes(dbAttr)) - } - if pm.cacheMisses != nil { - addDelta(ctx, pm.cacheMisses, m.BlockCache.Misses, &pm.prevCacheMisses, metric.WithAttributes(dbAttr)) - } - if pm.cacheSize != nil { - pm.cacheSize.Record(ctx, m.BlockCache.Size, metric.WithAttributes(dbAttr)) - } +// declareLevels declares the series read per LSM level, each reported with a +// "level" attribute. +func (p *pebbleMetrics) declareLevels() { + p.levelGauge("pebble_sstable_count", "{count}", "Current number of SSTables at each level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TablesCount) }) + p.levelGauge("pebble_sstable_total_size", "By", "Total size of SSTables at each level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TablesSize) }) + p.levelGauge("pebble_sstable_sublevels", "{count}", "Number of sublevels (read amplification); L0 only has non-0/1", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.Sublevels) }) + p.levelGauge("pebble_sstable_score", "1", "Level compaction score (0 if no compaction needed)", + func(lm *pebble.LevelMetrics) float64 { return lm.Score }) + p.levelGauge("pebble_sstable_fill_factor", "1", "Level fill factor (size vs ideal size)", + func(lm *pebble.LevelMetrics) float64 { return lm.FillFactor }) + p.levelGauge("pebble_sstable_virtual_count", "{count}", "Number of virtual sstables at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.VirtualTablesCount) }) + p.levelGauge("pebble_sstable_virtual_size", "By", "Size of virtual sstables at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.VirtualTablesSize) }) + p.levelCounter("pebble_compaction_bytes_read", "By", "Total bytes read during compaction", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TableBytesIn) }) + p.levelCounter("pebble_compaction_bytes_written", "By", "Total bytes written during compaction", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TableBytesCompacted) }) + p.levelCounter("pebble_sstable_bytes_ingested", "By", "Sstable bytes ingested at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TableBytesIngested) }) + p.levelCounter("pebble_sstable_bytes_moved", "By", "Sstable bytes moved by move compaction at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TableBytesMoved) }) + p.levelCounter("pebble_sstable_bytes_read", "By", "Bytes read for compactions at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TableBytesRead) }) + p.levelCounter("pebble_sstable_bytes_flushed", "By", "Bytes written to sstables during flushes at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TableBytesFlushed) }) + p.levelCounter("pebble_sstable_tables_compacted", "{count}", "Sstables compacted to this level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TablesCompacted) }) + p.levelCounter("pebble_sstable_tables_flushed", "{count}", "Sstables flushed to this level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TablesFlushed) }) + p.levelCounter("pebble_sstable_tables_ingested", "{count}", "Sstables ingested into level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TablesIngested) }) + p.levelCounter("pebble_sstable_tables_moved", "{count}", "Sstables moved to level by move compaction", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TablesMoved) }) + p.levelGauge("pebble_sstable_compensated_fill_factor", "1", "Level compensated fill factor", + func(lm *pebble.LevelMetrics) float64 { return lm.CompensatedFillFactor }) + p.levelGauge("pebble_sstable_estimated_references_size", "By", "Est. physical size of blob refs at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.EstimatedReferencesSize) }) + p.levelCounter("pebble_sstable_tables_deleted", "{count}", "Sstables deleted by delete-only compaction at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TablesDeleted) }) + p.levelCounter("pebble_sstable_tables_excised", "{count}", "Sstables excised by delete-only compaction at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.TablesExcised) }) + p.levelCounter("pebble_sstable_blob_bytes_read_estimate", "By", "Est. physical bytes read for blob refs at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.BlobBytesReadEstimate) }) + p.levelCounter("pebble_sstable_blob_bytes_compacted", "By", "Blob bytes written during compaction at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.BlobBytesCompacted) }) + p.levelCounter("pebble_sstable_blob_bytes_flushed", "By", "Blob bytes written during flush at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.BlobBytesFlushed) }) + p.levelCounter("pebble_sstable_multi_level_bytes_in_top", "By", "Bytes from top level in multilevel compaction", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.MultiLevel.TableBytesInTop) }) + p.levelCounter("pebble_sstable_multi_level_bytes_in", "By", "Bytes in for multilevel compaction", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.MultiLevel.TableBytesIn) }) + p.levelCounter("pebble_sstable_multi_level_bytes_read", "By", "Bytes read for multilevel compaction", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.MultiLevel.TableBytesRead) }) + p.levelGauge("pebble_sstable_value_blocks_size", "By", "Value blocks size at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.Additional.ValueBlocksSize) }) + p.levelCounter("pebble_sstable_bytes_written_data_blocks", "By", "Bytes written to data blocks at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.Additional.BytesWrittenDataBlocks) }) + p.levelCounter("pebble_sstable_bytes_written_value_blocks", "By", "Bytes written to value blocks at level", + func(lm *pebble.LevelMetrics) float64 { return float64(lm.Additional.BytesWrittenValueBlocks) }) }