From c5a5e34fd79f1600cd393c3298433d5a44ddc0bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:11:03 -0300 Subject: [PATCH 1/3] fix(startup): resume from existing DB without a checkpoint URL `fetch_initial_state` gated the on-disk state lookup on `--checkpoint-sync-url`, so restarting a node without that flag wrote a slot-0 genesis anchor over a populated RocksDB and discarded a current chain. Operators had to pass a checkpoint URL purely as a fallback trigger, even on a fresh DB where it was never used. Try the persisted state first and treat the URL as a fallback for when nothing on disk is resumable. When the DB is stale and no URL was given, resume anyway instead of re-initializing: P2P forward-sync can close the gap, a genesis re-init cannot. The staleness threshold now only decides whether a checkpoint is preferable to what we already have. Also drop the `Starting checkpoint sync` line emitted before the DB was consulted; it fired on every successful resume, so grepping the boot log for it falsely reported a resync. --- bin/ethlambda/src/cli.rs | 16 +++-- bin/ethlambda/src/main.rs | 146 +++++++++++++++++++++++++++++++++----- docs/checkpoint_sync.md | 28 +++++++- 3 files changed, 164 insertions(+), 26 deletions(-) diff --git a/bin/ethlambda/src/cli.rs b/bin/ethlambda/src/cli.rs index 4bbe1683..81208b67 100644 --- a/bin/ethlambda/src/cli.rs +++ b/bin/ethlambda/src/cli.rs @@ -37,11 +37,17 @@ pub(crate) struct CliOptions { #[arg(long)] pub(crate) node_id: String, /// Base URL(s) of checkpoint-sync peer API servers (e.g., http://peer:5052). - /// When set, skips genesis initialization and fetches the finalized state - /// and block from each peer's `/lean/v0/states/finalized` and - /// `/lean/v0/blocks/finalized` endpoints. For backward compatibility, a - /// URL ending in `/lean/v0/states/finalized` is accepted and the trailing - /// path is stripped. + /// When set, fetches the finalized state and block from each peer's + /// `/lean/v0/states/finalized` and `/lean/v0/blocks/finalized` endpoints. + /// For backward compatibility, a URL ending in + /// `/lean/v0/states/finalized` is accepted and the trailing path is + /// stripped. + /// + /// This is a fallback, not a precedence: state already in the data + /// directory always wins, so these URLs are only used when there is no + /// resumable state on disk (or it has fallen too far behind the current + /// slot). With neither resumable state nor URLs, the node starts from + /// genesis. /// /// Multiple URLs may be supplied for redundancy, either comma-separated /// (`--checkpoint-sync-url u1,u2`) or by repeating the flag diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index df4893e6..fef90945 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -641,11 +641,19 @@ fn read_hex_file_bytes(path: impl AsRef) -> eyre::Result> { /// Fetch the initial state for the node. /// -/// If `checkpoint_urls` is empty, creates a genesis state from the local -/// genesis configuration. Otherwise performs checkpoint sync by downloading -/// and verifying the finalized state AND signed block from a peer. URLs are -/// tried in order: the first peer that succeeds wins, and failures fall over -/// to the next URL. Startup only aborts if every URL fails. +/// State already on disk wins: a previous run's DB is resumed from whenever it +/// exists and belongs to this network, whether or not `checkpoint_urls` is +/// supplied. `checkpoint_urls` is the fallback for when there is nothing +/// resumable on disk, or when what is there has fallen too far behind the +/// current slot to be worth catching up over P2P +/// ([`MAX_RESUMABLE_DB_STATE_AGE`]). +/// +/// With no resumable DB state, a non-empty `checkpoint_urls` performs checkpoint +/// sync by downloading and verifying the finalized state AND signed block from a +/// peer. URLs are tried in order: the first peer that succeeds wins, and +/// failures fall over to the next URL. Startup only aborts if every URL fails. +/// An empty `checkpoint_urls` creates a genesis state from the local genesis +/// configuration. /// /// Fetching the matching signed block lets the local store serve a valid /// anchor via the `BlocksByRoot` req-resp protocol; without it, peers @@ -669,20 +677,10 @@ async fn fetch_initial_state( ) -> Result { let validators = genesis.validators(); - if checkpoint_urls.is_empty() { - info!("No checkpoint sync URL provided, initializing from genesis state"); - let genesis_state = State::from_genesis(genesis.genesis_time, validators); - return Ok(Store::from_anchor_state(backend, genesis_state)); - }; - - // Checkpoint sync path: try URLs in order, fail over to the next on error. - info!( - url_count = checkpoint_urls.len(), - "Starting checkpoint sync" - ); - // Checkpoint sync path - - // Prefer resuming from a fresh on-disk state to avoid re-downloading what we already have. + // Prefer resuming from on-disk state to avoid re-downloading what we already + // have. Tried before the checkpoint-sync and genesis paths so that a restart + // without `--checkpoint-sync-url` keeps the chain instead of writing a + // slot-0 anchor over it. if let Ok(Some(store)) = Store::from_db_state(backend.clone(), genesis.genesis_time) { let now_ms = SystemTime::UNIX_EPOCH .elapsed() @@ -699,12 +697,34 @@ async fn fetch_initial_state( ); return Ok(store); } + // Stale, but with no checkpoint URL resuming is the only + // non-destructive option: re-initializing from genesis would discard + // this history, so close the gap over P2P instead. That only works + // while peers can still serve the range; beyond their block-signature + // pruning horizon the node cannot catch up and needs a checkpoint URL. + if checkpoint_urls.is_empty() { + warn!( + head_slot, + current_slot, + gap, + "Existing DB state is stale and no checkpoint sync URL was provided; \ + resuming anyway and relying on P2P sync to catch up" + ); + return Ok(store); + } warn!( head_slot, current_slot, gap, "Existing DB state is stale; falling through to checkpoint sync" ); } + if checkpoint_urls.is_empty() { + info!("No checkpoint sync URL provided, initializing from genesis state"); + let genesis_state = State::from_genesis(genesis.genesis_time, validators); + return Ok(Store::from_anchor_state(backend, genesis_state)); + } + + // Checkpoint sync path: try URLs in order, fail over to the next on error. info!(?checkpoint_urls, "Starting checkpoint sync"); let (state, signed_block) = checkpoint_sync::fetch_anchor_with_retry( @@ -740,6 +760,8 @@ async fn fetch_initial_state( #[cfg(test)] mod tests { use super::*; + use ethlambda_storage::backend::InMemoryBackend; + use ethlambda_types::genesis::GenesisValidatorEntry; /// Validator-config snippet matching `lean-quickstart`'s ansible-devnet /// where networks share a non-default committee count. @@ -828,4 +850,90 @@ validators: .unwrap_or(1); assert_eq!(resolved, 1); } + + /// Slot of the anchor seeded into the test DB. Any non-zero slot works: a + /// genesis re-initialization always anchors at slot 0, so a non-zero head + /// slot is what distinguishes "resumed from disk" from "started over". + const SEEDED_HEAD_SLOT: u64 = 12; + + fn now_secs() -> u64 { + SystemTime::UNIX_EPOCH + .elapsed() + .expect("already past the unix epoch") + .as_secs() + } + + /// Single-validator genesis config. The pubkeys are placeholders; none of + /// the paths under test verify signatures. + fn test_genesis(genesis_time: u64) -> GenesisConfig { + GenesisConfig { + genesis_time, + genesis_validators: vec![GenesisValidatorEntry { + attestation_pubkey: [1u8; 52], + proposal_pubkey: [2u8; 52], + }], + } + } + + /// Write an anchor at [`SEEDED_HEAD_SLOT`] into `backend`, standing in for a + /// previous run's persisted chain state. + fn seed_db(backend: Arc, genesis: &GenesisConfig) { + let mut anchor = State::from_genesis(genesis.genesis_time, genesis.validators()); + anchor.slot = SEEDED_HEAD_SLOT; + anchor.latest_block_header.slot = SEEDED_HEAD_SLOT; + Store::from_anchor_state(backend, anchor); + } + + #[tokio::test] + async fn initializes_from_genesis_when_db_is_empty() { + let genesis = test_genesis(now_secs()); + let backend = Arc::new(InMemoryBackend::default()); + + let store = fetch_initial_state(&[], &genesis, backend).await.unwrap(); + + assert_eq!(store.head_slot(), 0); + } + + #[tokio::test] + async fn resumes_from_fresh_db_without_checkpoint_url() { + let genesis = test_genesis(now_secs()); + let backend = Arc::new(InMemoryBackend::default()); + seed_db(backend.clone(), &genesis); + + let store = fetch_initial_state(&[], &genesis, backend).await.unwrap(); + + assert_eq!(store.head_slot(), SEEDED_HEAD_SLOT); + } + + /// With no checkpoint URL to fall back to, resuming a stale DB beats + /// clobbering it with a slot-0 genesis anchor: P2P forward-sync can close + /// the gap, a genesis re-init cannot. + #[tokio::test] + async fn resumes_from_stale_db_without_checkpoint_url() { + let seconds_per_slot = MILLISECONDS_PER_SLOT / 1_000; + let stale_slots = MAX_RESUMABLE_DB_STATE_AGE + 100; + let genesis = test_genesis(now_secs() - stale_slots * seconds_per_slot); + let backend = Arc::new(InMemoryBackend::default()); + seed_db(backend.clone(), &genesis); + + let store = fetch_initial_state(&[], &genesis, backend).await.unwrap(); + + assert_eq!(store.head_slot(), SEEDED_HEAD_SLOT); + } + + /// A DB from another network is not resumable, so the no-URL path still + /// falls back to genesis. + #[tokio::test] + async fn initializes_from_genesis_when_db_genesis_time_differs() { + let seeded_genesis = test_genesis(now_secs()); + let backend = Arc::new(InMemoryBackend::default()); + seed_db(backend.clone(), &seeded_genesis); + + let other_genesis = test_genesis(seeded_genesis.genesis_time + 1); + let store = fetch_initial_state(&[], &other_genesis, backend) + .await + .unwrap(); + + assert_eq!(store.head_slot(), 0); + } } diff --git a/docs/checkpoint_sync.md b/docs/checkpoint_sync.md index 79d2f12a..ee2428a9 100644 --- a/docs/checkpoint_sync.md +++ b/docs/checkpoint_sync.md @@ -24,7 +24,7 @@ ethlambda \ Where `` is the address of a checkpoint source (see [Checkpoint Sources](#checkpoint-sources) below). -When `--checkpoint-sync-url` is omitted, the node initializes from genesis. +State already on disk takes precedence over both checkpoint sync and genesis: if the data directory holds a previous run's chain state for this network, the node resumes from it. `--checkpoint-sync-url` is the fallback for when there is nothing resumable on disk, or when what is there has fallen too far behind (see [Restarts and Existing State](#restarts-and-existing-state)). With no resumable state and no URL, the node initializes from genesis. ## Checkpoint Sources @@ -56,7 +56,31 @@ If any step fails (network error, decoding error, verification failure), the nod After successful initialization, the node starts normally: it connects to the P2P network and begins participating from the checkpoint slot. -If the data directory (`./data`) already contains state from a previous run, checkpoint sync writes the new anchor state on top without clearing existing data. For a clean checkpoint sync, remove the data directory first. +## Restarts and Existing State + +A node restarted against a populated data directory resumes from disk rather than re-initializing, so no flag is needed to preserve the chain across a redeploy. The decision is made before any download: + +| State in data directory | `--checkpoint-sync-url` | Result | +| ------------------------- | ------------------------- | -------- | +| None, or from another network (`GENESIS_TIME` differs) | omitted | Initialize from genesis | +| None, or from another network | set | Checkpoint sync | +| Present, head within the resume window | either | Resume from disk (no download) | +| Present, head beyond the resume window | set | Checkpoint sync | +| Present, head beyond the resume window | omitted | Resume from disk anyway, with a warning | + +The resume window is `MAX_RESUMABLE_DB_STATE_AGE` (450 slots, ~30 minutes at 4-second slots) measured as `current_slot - head_slot`. Staleness is measured against the head, not the finalized checkpoint, so a node whose head is current still resumes during a finality stall. + +Beyond that window the node prefers a checkpoint when one is offered, since catching up over P2P costs more than downloading a recent state. With no URL to fall back on it resumes regardless and relies on P2P sync, because re-initializing from genesis would discard the existing history. That recovery only works while peers can still serve the missing range; past their block-signature pruning horizon (`SIGNATURE_PRUNING_RANGE`, 21600 slots, ~1 day) the node cannot catch up and needs a checkpoint URL. The warning logs the gap so this is visible in the boot log. + +To deliberately discard existing state and start over from genesis or from a checkpoint, remove the data directory first. Checkpoint sync itself writes its anchor state on top without clearing existing data. + +Each outcome has one unambiguous boot log line: + +| Log line | Meaning | +| ---------- | --------- | +| `Resuming from existing DB state head_slot=… current_slot=… gap=…` | Resumed from disk, nothing downloaded | +| `Starting checkpoint sync checkpoint_urls=[…]` then `Checkpoint sync complete …` | Downloaded a checkpoint | +| `No checkpoint sync URL provided, initializing from genesis state` | Started from genesis | ## Verification Checks From 994ae8202587edb404d7830ab1949a6ea9cd7447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:40:19 -0300 Subject: [PATCH 2/3] chore: simplify logs --- bin/ethlambda/src/main.rs | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index fef90945..2e353b4d 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -691,31 +691,15 @@ async fn fetch_initial_state( let head_slot = store.head_slot(); let gap = current_slot.saturating_sub(head_slot); if gap <= MAX_RESUMABLE_DB_STATE_AGE { - info!( - head_slot, - current_slot, gap, "Resuming from existing DB state" - ); + info!(head_slot, current_slot, gap, "Resuming from existing DB"); return Ok(store); } - // Stale, but with no checkpoint URL resuming is the only - // non-destructive option: re-initializing from genesis would discard - // this history, so close the gap over P2P instead. That only works - // while peers can still serve the range; beyond their block-signature - // pruning horizon the node cannot catch up and needs a checkpoint URL. + warn!(head_slot, current_slot, gap, "Existing DB state is stale"); if checkpoint_urls.is_empty() { - warn!( - head_slot, - current_slot, - gap, - "Existing DB state is stale and no checkpoint sync URL was provided; \ - resuming anyway and relying on P2P sync to catch up" - ); + warn!("No checkpoint sync URL provided, resuming from existing stale DB"); return Ok(store); } - warn!( - head_slot, - current_slot, gap, "Existing DB state is stale; falling through to checkpoint sync" - ); + warn!("Falling through to checkpoint sync"); } if checkpoint_urls.is_empty() { From 8d0375d035392bbd70de129e184f62a804cd9580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:45:21 -0300 Subject: [PATCH 3/3] docs: fix docs --- CLAUDE.md | 5 ++++- docs/checkpoint_sync.md | 8 -------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5eb367d2..3e092d5e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -288,7 +288,10 @@ actual_slot = finalized_slot + 1 + relative_index ## HTTP Servers (API + Metrics) -The RPC crate runs two independent Axum servers (API on `:5052`, metrics/debug on `:5054`). See [`docs/rpc.md`](docs/rpc.md) for the full reference: CLI flags and defaults, the API endpoints (health, finalized state/block, justified checkpoint, blocks by root/slot, fork-choice tree + D3.js UI, runtime aggregator toggle), the metrics/debug endpoints (Prometheus `/metrics`, jemalloc heap profiling), the Hive test-driver endpoints, plus request/response shapes, status codes, and content types. +The RPC crate serves the API router (`--api-port`, default 5052) and the metrics/debug routers +(`--metrics-port`, default 5054). When the two ports differ it binds two independent Axum servers; +when they are equal it merges all three routers onto a single listener, so pointing both flags at +one port is supported and not a misconfiguration. See [`docs/rpc.md`](docs/rpc.md) for the full reference: CLI flags and defaults, the API endpoints (health, finalized state/block, justified checkpoint, blocks by root/slot, fork-choice tree + D3.js UI, runtime aggregator toggle), the metrics/debug endpoints (Prometheus `/metrics`, jemalloc heap profiling), the Hive test-driver endpoints, plus request/response shapes, status codes, and content types. ## Configuration Files diff --git a/docs/checkpoint_sync.md b/docs/checkpoint_sync.md index ee2428a9..46d32446 100644 --- a/docs/checkpoint_sync.md +++ b/docs/checkpoint_sync.md @@ -74,14 +74,6 @@ Beyond that window the node prefers a checkpoint when one is offered, since catc To deliberately discard existing state and start over from genesis or from a checkpoint, remove the data directory first. Checkpoint sync itself writes its anchor state on top without clearing existing data. -Each outcome has one unambiguous boot log line: - -| Log line | Meaning | -| ---------- | --------- | -| `Resuming from existing DB state head_slot=… current_slot=… gap=…` | Resumed from disk, nothing downloaded | -| `Starting checkpoint sync checkpoint_urls=[…]` then `Checkpoint sync complete …` | Downloaded a checkpoint | -| `No checkpoint sync URL provided, initializing from genesis state` | Started from genesis | - ## Verification Checks All checks are performed before the state is accepted: