From b81206dc4251dd18151e7bc3ed149f971726a606 Mon Sep 17 00:00:00 2001 From: Dickson Date: Tue, 11 Aug 2026 23:35:34 -0400 Subject: [PATCH] fix(storage): make pruning crash-consistent --- sei-db/db_engine/pebbledb/batch.go | 4 + .../block/littblock/litt_block_db.go | 43 ++++-- .../block/littblock/litt_block_gc.go | 7 +- .../block/littblock/litt_block_gc_test.go | 46 +++++- sei-db/ledger_db/receipt/export_test.go | 11 +- sei-db/ledger_db/receipt/litt_receipt_gc.go | 5 +- .../ledger_db/receipt/litt_receipt_gc_test.go | 52 +++++++ .../litt_receipt_pruner_internal_test.go | 140 ++++++++++++++++++ .../ledger_db/receipt/litt_receipt_store.go | 106 +++++++++---- sei-db/ledger_db/receipt/litt_tag_index.go | 2 +- sei-db/management/gc/prunable_store.go | 4 +- sei-db/state_db/statewal/state_wal_gc.go | 13 +- sei-db/state_db/statewal/state_wal_gc_test.go | 47 +++++- sei-db/state_db/statewal/state_wal_impl.go | 13 +- sei-tendermint/autobahn/types/block_db.go | 5 +- 15 files changed, 432 insertions(+), 66 deletions(-) diff --git a/sei-db/db_engine/pebbledb/batch.go b/sei-db/db_engine/pebbledb/batch.go index 7b4fd3b77f..ccc59448fc 100644 --- a/sei-db/db_engine/pebbledb/batch.go +++ b/sei-db/db_engine/pebbledb/batch.go @@ -29,6 +29,10 @@ func (pb *pebbleBatch) Delete(key []byte) error { return pb.b.Delete(key, nil) } +func (pb *pebbleBatch) DeleteRange(start, end []byte) error { + return pb.b.DeleteRange(start, end, nil) +} + func (pb *pebbleBatch) Commit(opts types.WriteOptions) error { writeCount := int64(pb.b.Count()) err := pb.b.Commit(toPebbleWriteOpts(opts)) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 68cf7af46a..8826dfbc6d 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -52,11 +52,12 @@ type blockDB struct { mu sync.Mutex hasBlocks bool lastBlockNumber types.GlobalBlockNumber - hasQC bool - lastQCNext types.GlobalBlockNumber - - // latestQCStartBlock is the most recently written QC's starting block number. - latestQCStartBlock types.GlobalBlockNumber + // The newest block covered by a successful Flush (or recovered on open). GC measures retention + // from this cursor so it never prunes durable history against a volatile write suffix. + hasDurableBlocks bool + lastDurableBlockNumber types.GlobalBlockNumber + hasQC bool + lastQCNext types.GlobalBlockNumber // firstBlockNumber is the lowest block number this handle has seen. Iterator clamps its // start up to it so a scan always opens on a block that exists: the first block may be @@ -196,11 +197,15 @@ func (s *blockDB) recoverCursors() error { if err != nil { return fmt.Errorf("failed to unmarshal newest qc: %w", err) } - s.latestQCStartBlock, s.lastQCNext = coveredRange(qc) + _, s.lastQCNext = coveredRange(qc) s.hasQC = true } } } + if s.hasBlocks { + s.hasDurableBlocks = true + s.lastDurableBlockNumber = s.lastBlockNumber + } return nil } @@ -329,7 +334,6 @@ func (s *blockDB) WriteQC(qc *types.FullCommitQC) error { // discovering it by scanning; a reopen re-derives the same value. s.oldestQCStart = first } - s.latestQCStartBlock = first s.lastQCNext = next s.hasQC = true return nil @@ -345,8 +349,14 @@ func (s *blockDB) PruneBefore(blockHeight types.GlobalBlockNumber) error { return nil } - if ceiling := min(s.latestQCStartBlock, s.lastBlockNumber); blockHeight > ceiling { - blockHeight = ceiling + // Never make the store depend on an unflushed suffix to remain non-empty. GC and callers may + // prune concurrently with the persistence loop; a process crash can lose every write above this + // cursor, so the newest durable block's whole QC cohort is the highest safe prune boundary. + if !s.hasDurableBlocks { + return nil + } + if blockHeight > s.lastDurableBlockNumber { + blockHeight = s.lastDurableBlockNumber } // Round the watermark down to the start of a QC's range, to avoid pruning a QC before its blocks. @@ -405,9 +415,24 @@ func (s *blockDB) gcFilter(key []byte, _ bool) (bool, error) { } func (s *blockDB) Flush() error { + // Snapshot the write cursor before flushing. Table.Flush guarantees every Put that completed + // before it began, while overlapping writes may or may not be durable; recording this snapshot + // is therefore conservative without blocking the persistence loop for the duration of the I/O. + s.mu.Lock() + hasBlocks := s.hasBlocks + lastBlockNumber := s.lastBlockNumber + s.mu.Unlock() + if err := s.table.Flush(); err != nil { return fmt.Errorf("failed to flush ledger table: %w", err) } + + s.mu.Lock() + defer s.mu.Unlock() + if hasBlocks && (!s.hasDurableBlocks || lastBlockNumber > s.lastDurableBlockNumber) { + s.hasDurableBlocks = true + s.lastDurableBlockNumber = lastBlockNumber + } return nil } diff --git a/sei-db/ledger_db/block/littblock/litt_block_gc.go b/sei-db/ledger_db/block/littblock/litt_block_gc.go index ff48ba9c32..68e871dc4a 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_gc.go +++ b/sei-db/ledger_db/block/littblock/litt_block_gc.go @@ -41,16 +41,15 @@ func (s *blockDB) GetRollbackFloor(rollbackWindow uint64) uint64 { return head - rollbackWindow } -// GetLatestBlock returns the newest block number written, or 0 when none has been. It reports the -// written cursor rather than the flushed one, so a block a crash would lose still counts as ingested. +// GetLatestBlock returns the newest crash-recoverable block number, or 0 when none has been. // // Global block numbers start at genesis block 0, so a store holding only that block is // indistinguishable from an empty one. func (s *blockDB) GetLatestBlock() (uint64, error) { s.mu.Lock() defer s.mu.Unlock() - if !s.hasBlocks { + if !s.hasDurableBlocks { return 0, nil } - return uint64(s.lastBlockNumber), nil + return uint64(s.lastDurableBlockNumber), nil } diff --git a/sei-db/ledger_db/block/littblock/litt_block_gc_test.go b/sei-db/ledger_db/block/littblock/litt_block_gc_test.go index 73f8382247..d5bdd43417 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_gc_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_gc_test.go @@ -36,11 +36,10 @@ func openForGC(t *testing.T, dir string) (types.BlockDB, gc.PrunableStore) { return db, store } -// The head the collector reads is the newest block written — not the newest QC's coverage, since -// a QC is written before the blocks it covers — and 0 while nothing has been ingested, so an -// empty store drops out of the head minimum instead of dragging the lookback floor to 0. The reopen -// at the end covers recovery: a store that reported 0 after a restart would let the other stores -// prune past a height this one still holds. +// The head the collector reads is the newest crash-recoverable block — not the newest written +// block and not the newest QC's coverage. Counting an unflushed suffix would let the collector +// prune durable history against records a process crash can lose, leaving less than the configured +// rollback window. The reopen at the end covers recovery. func TestGCLatestBlock(t *testing.T) { dir := t.TempDir() rng := utils.TestRngFromSeed(1) @@ -53,6 +52,11 @@ func TestGCLatestBlock(t *testing.T) { writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19, QCs [0,5)..[15,20) latest, err = store.GetLatestBlock() require.NoError(t, err) + require.Equal(t, uint64(0), latest, "written but unflushed blocks must not advance the pruning head") + + require.NoError(t, db.Flush()) + latest, err = store.GetLatestBlock() + require.NoError(t, err) require.Equal(t, uint64(19), latest) // A QC covering 20..24 is written but none of its blocks are, so the head must not move. @@ -62,7 +66,14 @@ func TestGCLatestBlock(t *testing.T) { require.Equal(t, uint64(19), latest, "a QC ahead of its blocks must not advance the head") require.NoError(t, db.WriteBlock(20, types.GenBlock(rng))) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(19), latest, "an unflushed block must not advance the pruning head") + require.NoError(t, db.Flush()) + latest, err = store.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(20), latest) require.NoError(t, db.Close()) _, reopened := openForGC(t, dir) @@ -107,6 +118,7 @@ func TestGCRollbackFloorAndPruneHistory(t *testing.T) { require.Equal(t, uint64(0), impl.watermark.Load()) writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19, QCs [0,5),[5,10),[10,15),[15,20) + require.NoError(t, db.Flush()) require.Equal(t, uint64(19), store.GetRollbackFloor(0), "the whole store is inside a window of 0") require.Equal(t, uint64(7), store.GetRollbackFloor(12)) // A window deeper than the store's own head is a rollback promise reaching past genesis, so @@ -132,6 +144,7 @@ func TestGCPruneHistoryAboveHeadIsCapped(t *testing.T) { rng := utils.TestRngFromSeed(3) writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19, newest cohort QC[15,20) + require.NoError(t, db.Flush()) require.NoError(t, store.PruneHistory(1_000)) require.Equal(t, uint64(15), db.(*blockDB).watermark.Load(), "a prune past the head is capped to the newest cohort") @@ -142,6 +155,29 @@ func TestGCPruneHistoryAboveHeadIsCapped(t *testing.T) { } } +// The never-empty cap is measured from the durable tip, not the newest write. Otherwise a concurrent +// unflushed suffix could let pruning reclaim every crash-recoverable block; a process crash would +// then lose the suffix and reopen without the history the cap promised to preserve. +func TestGCPruneHistoryAboveHeadIsCappedToDurableCohort(t *testing.T) { + db, store := openForGC(t, t.TempDir()) + rng := utils.TestRngFromSeed(5) + + writeSyntheticBatches(t, db, rng, 2, 5) // durable blocks 0..9, newest durable cohort QC[5,10) + require.NoError(t, db.Flush()) + for i := 2; i < 4; i++ { + first := types.GlobalBlockNumber(i * 5) + next := first + 5 + require.NoError(t, db.WriteQC(types.GenFullCommitQCRange(rng, first, next))) + for n := first; n < next; n++ { + require.NoError(t, db.WriteBlock(n, types.GenBlock(rng))) + } + } + + require.NoError(t, store.PruneHistory(1_000)) + require.Equal(t, uint64(5), db.(*blockDB).watermark.Load(), + "the store must retain the newest durable cohort, not rely on an unflushed suffix") +} + func TestConfigValidateRetentionTime(t *testing.T) { cfg, err := DefaultConfig(t.TempDir()) require.NoError(t, err) diff --git a/sei-db/ledger_db/receipt/export_test.go b/sei-db/ledger_db/receipt/export_test.go index 04650ae381..ebbf03ce61 100644 --- a/sei-db/ledger_db/receipt/export_test.go +++ b/sei-db/ledger_db/receipt/export_test.go @@ -20,5 +20,14 @@ func GetLogsForTx(receipt *types.Receipt, logStartIndex uint) []*ethtypes.Log { // block below cutoff. Test-only hook so prune behavior can be asserted without // waiting on the background interval. func PruneLittIdx(store ReceiptStore, cutoff uint64) error { - return store.(*littReceiptStore).pruneBlocksBelow(cutoff) + s := store.(*littReceiptStore) + if err := s.flushReceipts(); err != nil { + return err + } + return s.pruneBlocksBelow(cutoff) +} + +// FlushLittIdx makes every receipt body written before the call crash recoverable. +func FlushLittIdx(store ReceiptStore) error { + return store.(*littReceiptStore).flushReceipts() } diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc.go b/sei-db/ledger_db/receipt/litt_receipt_gc.go index 9ba6dee74a..54bac99d5d 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_gc.go +++ b/sei-db/ledger_db/receipt/litt_receipt_gc.go @@ -43,9 +43,10 @@ func (s *littReceiptStore) GetRollbackFloor(rollbackWindow uint64) uint64 { return head - rollbackWindow } -// GetLatestBlock returns the newest block whose receipts have been written, or 0 when none have. +// GetLatestBlock returns the newest block whose receipt bodies are crash recoverable, or 0 when none +// have been. func (s *littReceiptStore) GetLatestBlock() (uint64, error) { - latest := s.latestVersion.Load() + latest := s.latestDurableVersion.Load() if latest <= 0 { return 0, nil } diff --git a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go index 0cb19d07a4..ef8bf52d69 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_gc_test.go +++ b/sei-db/ledger_db/receipt/litt_receipt_gc_test.go @@ -120,6 +120,7 @@ func TestReceiptGCAnswersDoNotDependOnKeepRecent(t *testing.T) { for block := uint64(1); block <= 10; block++ { writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) } + require.NoError(t, receipt.FlushLittIdx(store)) require.Equal(t, uint64(7), prunable.GetRollbackFloor(3), "keepRecent %d must not change the floor (head 10 - window 3)", keepRecent) } @@ -158,12 +159,42 @@ func TestReceiptGCLatestBlock(t *testing.T) { writeLitBlock(t, store, ctx, 2, litReceipt(2, 0, addr, topic)) writeLitBlock(t, store, ctx, 3, litReceipt(3, 0, addr, topic)) + require.NoError(t, receipt.FlushLittIdx(store)) latest, err = prunable.GetLatestBlock() require.NoError(t, err) require.Equal(t, uint64(3), latest) require.Equal(t, int64(3), store.LatestVersion(), "the head reported to the collector must agree with the store's own version") } +func TestReceiptGCLatestBlockSurvivesReopen(t *testing.T) { + dir := t.TempDir() + storeKey := storetypes.NewKVStoreKey("evm") + tkey := storetypes.NewTransientStoreKey("evm_transient") + ctx := testutil.DefaultContext(storeKey, tkey).WithBlockHeight(1) + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.Backend = "littidx" + cfg.DBDirectory = dir + cfg.ExternalPruning = true + + store, err := receipt.NewReceiptStore(cfg, storeKey) + require.NoError(t, err) + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + for block := uint64(1); block <= 3; block++ { + writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) + } + require.NoError(t, receipt.FlushLittIdx(store)) + require.NoError(t, store.Close()) + + store, err = receipt.NewReceiptStore(cfg, storeKey) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + prunable := store.(gc.PrunableStore) + latest, err := prunable.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(3), latest, "the durable head must be recovered with the receipt bodies") +} + // A contiguous store resolves the window against its own head, and the prune that follows moves the // retention floor to the height it is given — which is what makes the receipts below it stop being // served. Reclaiming their bodies lags that, since litt also waits for the TTL, but it can no @@ -180,6 +211,7 @@ func TestReceiptGCRollbackFloorAndPruneHistory(t *testing.T) { for block := uint64(1); block <= 3; block++ { writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) } + require.NoError(t, receipt.FlushLittIdx(store)) require.Equal(t, uint64(1), prunable.GetRollbackFloor(2)) // A window deeper than its own head is a rollback promise reaching past genesis, so nothing here // is eligible for pruning yet. @@ -219,6 +251,7 @@ func TestReceiptGCPruneHistoryAboveHead(t *testing.T) { for block := uint64(1); block <= 3; block++ { writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) } + require.NoError(t, receipt.FlushLittIdx(store)) require.NoError(t, prunable.PruneHistory(1_000)) require.Equal(t, int64(3), store.EarliestVersion(), "the floor stops at the head, not the request") @@ -228,3 +261,22 @@ func TestReceiptGCPruneHistoryAboveHead(t *testing.T) { require.Equal(t, uint64(3), kept.BlockNumber) }) } + +func TestReceiptGCPruneHistoryCapsAtDurableHead(t *testing.T) { + store, prunable, ctx := setupLittIdxForGC(t, 0) + addr := common.HexToAddress("0xabcd") + topic := common.HexToHash("0x1111") + for block := uint64(1); block <= 3; block++ { + writeLitBlock(t, store, ctx, block, litReceipt(block, 0, addr, topic)) + } + require.NoError(t, receipt.FlushLittIdx(store)) + writeLitBlock(t, store, ctx, 4, litReceipt(4, 0, addr, topic)) + + require.NoError(t, prunable.PruneHistory(1_000)) + require.Equal(t, int64(3), store.EarliestVersion(), + "the floor must not rely on the unflushed block remaining after a crash") + + kept, err := store.GetReceiptFromStore(ctx, litTxHash(3, 0)) + require.NoError(t, err) + require.Equal(t, uint64(3), kept.BlockNumber) +} diff --git a/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go b/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go index f684e41a7a..d847c7789a 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go +++ b/sei-db/ledger_db/receipt/litt_receipt_pruner_internal_test.go @@ -3,9 +3,64 @@ package receipt import ( "testing" + storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" + dbconfig "github.com/sei-protocol/sei-chain/sei-db/config" + dbtypes "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" "github.com/stretchr/testify/require" ) +type pruneIndexSpy struct { + dbtypes.KeyValueDB + + directSets int + directSetOptions []dbtypes.WriteOptions + directRangeDeletes int + batchSets int + batchRangeDeletes int + batchCommits int + batchCommitOptions []dbtypes.WriteOptions +} + +func (s *pruneIndexSpy) Set(key, value []byte, opts dbtypes.WriteOptions) error { + s.directSets++ + s.directSetOptions = append(s.directSetOptions, opts) + return s.KeyValueDB.Set(key, value, opts) +} + +func (s *pruneIndexSpy) DeleteRange(start, end []byte, opts dbtypes.WriteOptions) error { + s.directRangeDeletes++ + return s.KeyValueDB.(interface { + DeleteRange(start, end []byte, opts dbtypes.WriteOptions) error + }).DeleteRange(start, end, opts) +} + +func (s *pruneIndexSpy) NewBatch() dbtypes.Batch { + return &pruneBatchSpy{Batch: s.KeyValueDB.NewBatch(), parent: s} +} + +type pruneBatchSpy struct { + dbtypes.Batch + parent *pruneIndexSpy +} + +func (b *pruneBatchSpy) Set(key, value []byte) error { + b.parent.batchSets++ + return b.Batch.Set(key, value) +} + +func (b *pruneBatchSpy) DeleteRange(start, end []byte) error { + b.parent.batchRangeDeletes++ + return b.Batch.(interface { + DeleteRange(start, end []byte) error + }).DeleteRange(start, end) +} + +func (b *pruneBatchSpy) Commit(opts dbtypes.WriteOptions) error { + b.parent.batchCommits++ + b.parent.batchCommitOptions = append(b.parent.batchCommitOptions, opts) + return b.Batch.Commit(opts) +} + // Which driver enforces retention is a four-way decision, and getting it wrong in either direction // is a production bug rather than a test nicety: two pruners race to different floors, and none // lets the tag index grow without bound. Enumerated here rather than observed through the jittered @@ -68,3 +123,88 @@ func TestRunsLocalPrunerIsTheNegationOfExternalPruning(t *testing.T) { "exactly one of the collector and the local pruner may enforce retention") } } + +// The range tombstone and the earliest-version metadata are one logical retention-floor update. +// If they are separate Pebble writes, a crash can preserve the tombstone but lose the metadata: +// after restart RPC reports old blocks as available even though their tag index is gone. One batch +// makes the update all-or-nothing under the KeyValueDB crash contract. +func TestPruneBlocksBelowAtomicallyMovesIndexAndFloor(t *testing.T) { + s, closeFn := setupLittCtxStore(t) + defer closeFn() + + s.latestVersion.Store(3) + s.latestDurableVersion.Store(3) + spy := &pruneIndexSpy{KeyValueDB: s.index} + s.index = spy + + require.NoError(t, s.pruneBlocksBelow(3)) + require.Zero(t, spy.directRangeDeletes, "the range tombstone must not commit before the floor metadata") + require.Zero(t, spy.directSets, "the floor metadata must not be a separate write") + require.Equal(t, 1, spy.batchRangeDeletes) + require.Equal(t, 1, spy.batchSets) + require.Equal(t, 1, spy.batchCommits) + require.Equal(t, []dbtypes.WriteOptions{{Sync: true}}, spy.batchCommitOptions) + require.Equal(t, int64(3), s.EarliestVersion()) +} + +func TestSetEarliestVersionPersistsBeforePublishing(t *testing.T) { + s, closeFn := setupLittCtxStore(t) + defer closeFn() + + spy := &pruneIndexSpy{KeyValueDB: s.index} + s.index = spy + + require.NoError(t, s.SetEarliestVersion(3)) + require.Equal(t, []dbtypes.WriteOptions{{Sync: true}}, spy.directSetOptions) + require.Equal(t, int64(3), s.EarliestVersion()) +} + +func TestReceiptGCFlushDoesNotPromoteRecoveredIndexOnlySuffix(t *testing.T) { + dir := t.TempDir() + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.Backend = receiptBackendLittIdx + cfg.DBDirectory = dir + cfg.ExternalPruning = true + storeKey := storetypes.NewKVStoreKey("evm") + + store, err := newLittReceiptStore(cfg, storeKey) + require.NoError(t, err) + s := store.(*littReceiptStore) + stopLittIdxBackground(s) + require.NoError(t, s.index.Set( + receiptLatestVersionKey, + encodeBlockNumber(4), + dbtypes.WriteOptions{Sync: true}, + )) + require.NoError(t, s.index.Set( + receiptLatestDurableVersionKey, + encodeBlockNumber(3), + dbtypes.WriteOptions{Sync: true}, + )) + require.NoError(t, s.Close()) + + store, err = newLittReceiptStore(cfg, storeKey) + require.NoError(t, err) + s = store.(*littReceiptStore) + stopLittIdxBackground(s) + defer func() { require.NoError(t, s.Close()) }() + + require.Equal(t, int64(4), s.LatestVersion()) + require.NoError(t, s.flushReceipts()) + latest, err := s.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(3), latest, + "flushing after reopen must not certify an index-only suffix left by a prior crash") + + require.NoError(t, s.SetLatestVersion(5)) + require.NoError(t, s.flushReceipts()) + latest, err = s.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(5), latest, "new writes must still advance the durable marker") +} + +func stopLittIdxBackground(s *littReceiptStore) { + close(s.stopBackground) + s.backgroundWg.Wait() + s.stopBackground = make(chan struct{}) +} diff --git a/sei-db/ledger_db/receipt/litt_receipt_store.go b/sei-db/ledger_db/receipt/litt_receipt_store.go index 7ff821ddd7..4c185f866e 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_store.go +++ b/sei-db/ledger_db/receipt/litt_receipt_store.go @@ -41,7 +41,7 @@ import ( // appending a new immutable part. // // The pebble index holds the tag keys (litt_tag_index.go) plus version -// metadata (m:latest / m:earliest). +// metadata (m:latest / m:latest_durable / m:earliest). // // Durability: a background flusher bounds litt durability lag to // littFlushInterval (~one block) without putting fsync on the commit path; @@ -66,8 +66,15 @@ type littReceiptStore struct { index dbtypes.KeyValueDB storeKey sdk.StoreKey - latestVersion atomic.Int64 - earliestVersion atomic.Int64 + latestVersion atomic.Int64 + // latestDurableVersion is the highest version covered by a successful Litt flush. Retention + // cannot advance from latestVersion directly because receipt bodies are flushed asynchronously. + latestDurableVersion atomic.Int64 + // latestFlushCandidate is the newest version written in this process that a subsequent Litt + // flush may publish as durable. It starts at the recovered durable marker, not latestVersion: + // after an unclean shutdown the Pebble index may be ahead of the receipt bodies. + latestFlushCandidate atomic.Int64 + earliestVersion atomic.Int64 keepRecent int64 pruneInterval int64 @@ -81,8 +88,9 @@ type littReceiptStore struct { var _ ReceiptStore = (*littReceiptStore)(nil) var ( - receiptLatestVersionKey = []byte("m:latest") - receiptEarliestVersionKey = []byte("m:earliest") + receiptLatestVersionKey = []byte("m:latest") + receiptLatestDurableVersionKey = []byte("m:latest_durable") + receiptEarliestVersionKey = []byte("m:earliest") ) const ( @@ -198,6 +206,9 @@ func newLittReceiptStore(cfg dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) s.index = index s.latestVersion.Store(s.readMeta(receiptLatestVersionKey)) + durableVersion := s.readMeta(receiptLatestDurableVersionKey) + s.latestDurableVersion.Store(durableVersion) + s.latestFlushCandidate.Store(durableVersion) s.earliestVersion.Store(s.readMeta(receiptEarliestVersionKey)) s.startPruning() s.startFlusher() @@ -224,6 +235,7 @@ func (s *littReceiptStore) SetLatestVersion(version int64) error { return err } s.latestVersion.Store(version) + s.latestFlushCandidate.Store(version) return nil } @@ -232,7 +244,11 @@ func (s *littReceiptStore) SetEarliestVersion(version int64) error { if version <= s.earliestVersion.Load() { return nil } - if err := s.index.Set(receiptEarliestVersionKey, encodeBlockNumber(uint64(version)), dbtypes.WriteOptions{}); err != nil { //nolint:gosec // block heights fit within uint64 + if err := s.index.Set( + receiptEarliestVersionKey, + encodeBlockNumber(uint64(version)), //nolint:gosec // block heights fit within uint64 + dbtypes.WriteOptions{Sync: true}, + ); err != nil { return err } s.earliestVersion.Store(version) @@ -301,8 +317,9 @@ func (s *littReceiptStore) SetReceipts(ctx sdk.Context, receipts []ReceiptRecord } maxBlock := blockNumbers[len(blockNumbers)-1] - newLatest := s.latestVersion.Load() - if int64(maxBlock) > newLatest { //nolint:gosec // block heights fit within int64 + currentLatest := s.latestVersion.Load() + newLatest := currentLatest + if int64(maxBlock) > currentLatest { //nolint:gosec // block heights fit within int64 newLatest = int64(maxBlock) //nolint:gosec // block heights fit within int64 if err := batch.Set(receiptLatestVersionKey, encodeBlockNumber(maxBlock)); err != nil { return err @@ -311,7 +328,10 @@ func (s *littReceiptStore) SetReceipts(ctx sdk.Context, receipts []ReceiptRecord if err := batch.Commit(dbtypes.WriteOptions{}); err != nil { return err } - s.latestVersion.Store(newLatest) + if newLatest > currentLatest { + s.latestVersion.Store(newLatest) + s.latestFlushCandidate.Store(newLatest) + } return nil } @@ -407,7 +427,7 @@ func (s *littReceiptStore) startFlusher() { case <-s.stopBackground: return case <-ticker.C: - if err := s.receipts.Flush(); err != nil { + if err := s.flushReceipts(); err != nil { logger.Error("failed to flush littdb receipts", "err", err) } } @@ -415,13 +435,38 @@ func (s *littReceiptStore) startFlusher() { }() } +func (s *littReceiptStore) flushReceipts() error { + latest := s.latestFlushCandidate.Load() + if err := s.receipts.Flush(); err != nil { + return err + } + if latest <= s.latestDurableVersion.Load() { + return nil + } + if err := s.index.Set( + receiptLatestDurableVersionKey, + encodeBlockNumber(uint64(latest)), //nolint:gosec // guarded positive below + // The Litt flush above is the safety boundary. Losing this marker can only make retention + // more conservative; forcing an additional Pebble fsync every 5 ms would put I/O on the + // background fast path without improving receipt-body durability. + dbtypes.WriteOptions{}, + ); err != nil { + return err + } + s.latestDurableVersion.Store(latest) + return nil +} + func (s *littReceiptStore) Close() error { var err error s.closeOnce.Do(func() { close(s.stopBackground) s.backgroundWg.Wait() - // litt's Close flushes, so the last sub-interval of writes is durable. - err = s.values.Close() + // Flush explicitly so the durable receipt head advances before the table closes. + err = s.flushReceipts() + if closeErr := s.values.Close(); err == nil { + err = closeErr + } if indexErr := s.index.Close(); err == nil { err = indexErr } @@ -471,7 +516,7 @@ func (s *littReceiptStore) startPruning() { // It is shared by both retention drivers, startPruning and the collector's PruneHistory. A cutoff // above this store's head is capped at the head rather than honored. func (s *littReceiptStore) pruneBlocksBelow(cutoff uint64) error { - head := s.latestVersion.Load() + head := s.latestDurableVersion.Load() if head <= 0 { return nil // nothing ingested, so nothing to drop } @@ -487,33 +532,30 @@ func (s *littReceiptStore) pruneBlocksBelow(cutoff uint64) error { return nil } - if err := s.deleteIndexRange(littTagBlockKey(floor), littTagBlockKey(cutoff)); err != nil { + rawBatch := s.index.NewBatch() + defer func() { _ = rawBatch.Close() }() + + batch, ok := rawBatch.(rangeDeleteBatch) + if !ok { + return fmt.Errorf("receipt index batch %T does not support range delete", rawBatch) + } + if err := batch.DeleteRange(littTagBlockKey(floor), littTagBlockKey(cutoff)); err != nil { + return err + } + if err := batch.Set(receiptEarliestVersionKey, encodeBlockNumber(cutoff)); err != nil { return err } - if err := s.index.Set(receiptEarliestVersionKey, encodeBlockNumber(cutoff), dbtypes.WriteOptions{}); err != nil { + // The index floor must survive before the in-memory floor can release receipt bodies to Litt GC. + if err := batch.Commit(dbtypes.WriteOptions{Sync: true}); err != nil { return err } s.earliestVersion.Store(int64(cutoff)) //nolint:gosec // block heights fit within int64 return nil } -// rangeDeleter is implemented by index DBs that can drop a whole key range with -// one range tombstone instead of per-key deletes (pebble implements it). -type rangeDeleter interface { - DeleteRange(start, end []byte, opts dbtypes.WriteOptions) error -} - -// deleteIndexRange removes every index key in [lower, upper) with one O(1) range -// tombstone — essential for the tag index, which writes thousands of keys per -// block, so per-key deletes would scan and delete millions of keys per prune -// pass. The index is always pebble (which supports range delete); the assertion -// guards against a future backend that does not. -func (s *littReceiptStore) deleteIndexRange(lower, upper []byte) error { - rd, ok := s.index.(rangeDeleter) - if !ok { - return fmt.Errorf("receipt index %T does not support range delete", s.index) - } - return rd.DeleteRange(lower, upper, dbtypes.WriteOptions{}) +type rangeDeleteBatch interface { + dbtypes.Batch + DeleteRange(start, end []byte) error } // groupReceiptRecordsByBlock splits records by block number (dropping entries diff --git a/sei-db/ledger_db/receipt/litt_tag_index.go b/sei-db/ledger_db/receipt/litt_tag_index.go index c60d7fc861..14074a6b30 100644 --- a/sei-db/ledger_db/receipt/litt_tag_index.go +++ b/sei-db/ledger_db/receipt/litt_tag_index.go @@ -28,7 +28,7 @@ import ( // topic position in disjoint keyspaces (criteria are positional). // firstLogIndex is the receipt's block-wide first log index, stored so reads // can number logs without decoding the receipts before it. Pruning a block -// range is a single range tombstone (see deleteIndexRange). +// range is a single range tombstone in the same batch as the retention-floor metadata. const ( littTagKeyPrefix = 't' diff --git a/sei-db/management/gc/prunable_store.go b/sei-db/management/gc/prunable_store.go index cbf0c81f30..7a66c80444 100644 --- a/sei-db/management/gc/prunable_store.go +++ b/sei-db/management/gc/prunable_store.go @@ -50,7 +50,7 @@ type PrunableStore interface { // nothing clamps this answer. GetRollbackFloor(rollbackWindow uint64) uint64 - // GetLatestBlock returns the highest block this store has ingested, 0 when it has ingested - // nothing. It is the head GetRollbackFloor measures rollbackWindow against. + // GetLatestBlock returns the highest crash-recoverable block this store has ingested, or 0 when + // it has ingested nothing. It is the head GetRollbackFloor measures rollbackWindow against. GetLatestBlock() (uint64, error) } diff --git a/sei-db/state_db/statewal/state_wal_gc.go b/sei-db/state_db/statewal/state_wal_gc.go index 5fc16eab86..bf564f99da 100644 --- a/sei-db/state_db/statewal/state_wal_gc.go +++ b/sei-db/state_db/statewal/state_wal_gc.go @@ -30,6 +30,13 @@ func (w *stateWALImpl) ExternalPruning() bool { // WAL. Prune is the equivalent for the WAL's own owner; this one is safe to call from the collector's // goroutine, and leaves the WAL usable on failure rather than bricking it. func (w *stateWALImpl) PruneHistory(blockNumber uint64) error { + head := w.lastDurableBlock.Load() + if head == 0 { + return nil + } + if blockNumber > head { + blockNumber = head + } if err := w.wal.PruneBefore(blockNumber); err != nil { return fmt.Errorf("failed to prune state WAL below block %d: %w", blockNumber, err) } @@ -55,8 +62,8 @@ func (w *stateWALImpl) GetRollbackFloor(rollbackWindow uint64) uint64 { return head - rollbackWindow } -// GetLatestBlock returns the highest block ended by SignalEndOfBlock, or 0 when none has been. A -// block written but not yet ended is excluded: it is still buffered rather than a record. +// GetLatestBlock returns the highest block made crash recoverable by Flush or Close, or 0 when none +// has been. func (w *stateWALImpl) GetLatestBlock() (uint64, error) { - return w.lastCompletedBlock.Load(), nil + return w.lastDurableBlock.Load(), nil } diff --git a/sei-db/state_db/statewal/state_wal_gc_test.go b/sei-db/state_db/statewal/state_wal_gc_test.go index 4e4acad450..01f790d10b 100644 --- a/sei-db/state_db/statewal/state_wal_gc_test.go +++ b/sei-db/state_db/statewal/state_wal_gc_test.go @@ -19,10 +19,10 @@ func openWALForGC(t *testing.T, cfg *Config) (StateWAL, gc.PrunableStore) { return w, store } -// The head is the last block ended by SignalEndOfBlock. A block that has been written but not ended -// is still buffered rather than a record, so counting it would put this store's head — and with it -// the floor it reports — one block above what the WAL can replay. -func TestGCLatestBlockCountsOnlyCompletedBlocks(t *testing.T) { +// The head is the last crash-recoverable block. SignalEndOfBlock only queues a record, so counting +// it before Flush would let the collector prune durable history against a volatile suffix. A process +// crash could then lose that suffix and leave less than the configured rollback window. +func TestGCLatestBlockCountsOnlyFlushedBlocks(t *testing.T) { w, store := openWALForGC(t, testConfig(t.TempDir())) latest, err := store.GetLatestBlock() @@ -33,6 +33,11 @@ func TestGCLatestBlockCountsOnlyCompletedBlocks(t *testing.T) { writeBlock(t, w, 2) latest, err = store.GetLatestBlock() require.NoError(t, err) + require.Equal(t, uint64(0), latest, "completed but unflushed blocks must not advance the pruning head") + + require.NoError(t, w.Flush()) + latest, err = store.GetLatestBlock() + require.NoError(t, err) require.Equal(t, uint64(2), latest) // Block 3 is written but not ended, so it is not yet in the WAL. @@ -44,6 +49,11 @@ func TestGCLatestBlockCountsOnlyCompletedBlocks(t *testing.T) { require.NoError(t, w.SignalEndOfBlock()) latest, err = store.GetLatestBlock() require.NoError(t, err) + require.Equal(t, uint64(2), latest, "ending a block does not make it crash recoverable") + + require.NoError(t, w.Flush()) + latest, err = store.GetLatestBlock() + require.NoError(t, err) require.Equal(t, uint64(3), latest) } @@ -100,6 +110,7 @@ func TestGCRollbackFloorComesFromOwnHead(t *testing.T) { for block := uint64(1); block <= 5; block++ { writeBlock(t, w, block) } + require.NoError(t, w.Flush()) require.Equal(t, uint64(5), store.GetRollbackFloor(0)) require.Equal(t, uint64(2), store.GetRollbackFloor(3)) require.Equal(t, uint64(0), store.GetRollbackFloor(1_000), @@ -155,6 +166,30 @@ func TestGCPruneHistoryIgnoresALowerFloor(t *testing.T) { require.Equal(t, uint64(8), first, "a later, lower floor must not undo the higher one") } +// A request above this store's own durable head can arrive from a stale or misconfigured caller. +// It must retain the newest crash-recoverable block rather than pruning all durable history while +// only an unflushed suffix remains. +func TestGCPruneHistoryAboveDurableHeadIsCapped(t *testing.T) { + cfg := testConfig(t.TempDir()) + cfg.TargetFileSize = 1 + w, store := openWALForGC(t, cfg) + + for block := uint64(1); block <= 5; block++ { + writeBlock(t, w, block) + } + require.NoError(t, w.Flush()) + writeBlock(t, w, 6) // completed, but not crash recoverable yet + + require.NoError(t, store.PruneHistory(1_000)) + require.NoError(t, w.Flush()) // order behind the prune and make block 6 durable + + ok, first, last, err := w.GetStoredRange() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, uint64(5), first, "the newest durable block at prune time must survive") + require.Equal(t, uint64(6), last) +} + // The collector runs on its own goroutine while the WAL's owner writes blocks on another, which is // the whole reason this surface is separate from the writer-facing one. Only the race detector can // judge it: stateWALImpl keeps its state in plain fields on the assumption of a single caller, and @@ -177,6 +212,9 @@ func TestGCConcurrentWithWriter(t *testing.T) { if err := w.SignalEndOfBlock(); err != nil { panic(err) } + if err := w.Flush(); err != nil { + panic(err) + } } }() @@ -208,6 +246,7 @@ func TestGCPruneHistoryBeforeClose(t *testing.T) { for block := uint64(1); block <= 5; block++ { writeBlock(t, w, block) } + require.NoError(t, w.Flush()) require.NoError(t, w.(gc.PrunableStore).PruneHistory(4)) require.NoError(t, w.Close()) diff --git a/sei-db/state_db/statewal/state_wal_impl.go b/sei-db/state_db/statewal/state_wal_impl.go index 416ec606dd..ba32688c19 100644 --- a/sei-db/state_db/statewal/state_wal_impl.go +++ b/sei-db/state_db/statewal/state_wal_impl.go @@ -15,7 +15,7 @@ var _ StateWAL = (*stateWALImpl)(nil) // // Not safe for concurrent use; see the StateWAL interface doc. The gc.PrunableStore surface in // state_wal_gc.go is the one exception: it runs on the collector's goroutine, and touches only -// lastCompletedBlock and the WAL underneath. +// lastDurableBlock and the WAL underneath. type stateWALImpl struct { // The underlying generic WAL, keyed by block number, whose payload is a block's changesets. wal seiwal.WAL[[]*proto.NamedChangeSet] @@ -50,6 +50,11 @@ type stateWALImpl struct { // 0 also means no block has completed yet, so a WAL whose only completed block is block 0 is // indistinguishable from an empty one. lastCompletedBlock atomic.Uint64 + + // The highest completed block made crash recoverable by Flush or Close. The garbage collector + // must measure its rollback window from this cursor rather than lastCompletedBlock: appends are + // asynchronous, so a completed but unflushed suffix can disappear on a process crash. + lastDurableBlock atomic.Uint64 } // New opens (or creates) a state WAL in the configured directory, recovering any files left behind by a @@ -125,6 +130,7 @@ func newStateWAL(wal seiwal.WAL[[]*proto.NamedChangeSet]) (StateWAL, error) { w.currentBlockEnded = true w.hasCurrentBlock = true w.lastCompletedBlock.Store(last) + w.lastDurableBlock.Store(last) } return w, nil } @@ -216,9 +222,13 @@ func (w *stateWALImpl) Flush() error { if w.fatalErr != nil { return fmt.Errorf("state WAL failed: %w", w.fatalErr) } + // Snapshot before Flush: an Append that overlaps the underlying flush is not guaranteed durable + // when it returns, so publishing lastCompletedBlock afterward could overstate the durable suffix. + lastCompletedBlock := w.lastCompletedBlock.Load() if err := w.wal.Flush(); err != nil { return w.fail(fmt.Errorf("failed to flush state WAL: %w", err)) } + w.lastDurableBlock.Store(lastCompletedBlock) return nil } @@ -281,6 +291,7 @@ func (w *stateWALImpl) Close() error { if err := w.wal.Close(); err != nil { return fmt.Errorf("failed to close state WAL: %w", err) } + w.lastDurableBlock.Store(w.lastCompletedBlock.Load()) return nil } diff --git a/sei-tendermint/autobahn/types/block_db.go b/sei-tendermint/autobahn/types/block_db.go index b2f62b0936..65af594402 100644 --- a/sei-tendermint/autobahn/types/block_db.go +++ b/sei-tendermint/autobahn/types/block_db.go @@ -126,8 +126,9 @@ type BlockDB interface { // // Pruning never empties the store. Once a block has been written, at // least one block (and a QC covering it) always remains readable — a - // request that would remove every block is capped to retain the most - // recently written block (and the QC covering it). + // request that would remove every block is capped to retain the newest + // crash-recoverable block (and the QC covering it). A newer unflushed + // suffix cannot be the only retained history because a crash may lose it. // // Pruning is asynchronous and MAY BE DELAYED. PruneBefore records the // watermark and returns; reclamation happens later, on the