From 1a432be9e487bbfcfa3c523a8727ce17b7d8af8c Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 9 Apr 2026 16:34:08 -0400 Subject: [PATCH 1/8] branch to check for chain progression during quiet period (fault injection off, system allowed time to recover --- .github/workflows/run_antithesis_test.yml | 6 + docker-compose.yaml | 3 + workload/cmd/stress-engine/main.go | 2 + .../stress-engine/quiet_recovery_vectors.go | 199 ++++++++++++++++++ 4 files changed, 210 insertions(+) create mode 100644 workload/cmd/stress-engine/quiet_recovery_vectors.go diff --git a/.github/workflows/run_antithesis_test.yml b/.github/workflows/run_antithesis_test.yml index 6222f01e..d76dd460 100644 --- a/.github/workflows/run_antithesis_test.yml +++ b/.github/workflows/run_antithesis_test.yml @@ -100,6 +100,11 @@ on: required: false default: false description: Enable to run an smoke test. + quiet_recovery: + type: boolean + required: false + default: false + description: Enable quiet recovery checks (pauses fault injection to verify chain self-healing) jobs: manual_run: @@ -153,6 +158,7 @@ jobs: custom.lotus_miner_1_tag=${{ inputs.lotus_miner_1_tag }} custom.forest_tag=${{ inputs.forest }} custom.forest_0_tag=${{ inputs.forest_0_tag }} + custom.quiet_recovery=${{ inputs.quiet_recovery }} scheduled_run_a: diff --git a/docker-compose.yaml b/docker-compose.yaml index 99dc8a18..c4f9f151 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -156,6 +156,9 @@ services: # Power / reorg - STRESS_WEIGHT_REORG=${STRESS_WEIGHT_REORG:-0} - STRESS_WEIGHT_POWER_SLASH=${STRESS_WEIGHT_POWER_SLASH:-2} + # Quiet recovery: pauses all fault injection to verify chain self-healing (off by default) + - QUIET_RECOVERY_ENABLED=${QUIET_RECOVERY_ENABLED:-0} + - STRESS_WEIGHT_QUIET_RECOVERY=${STRESS_WEIGHT_QUIET_RECOVERY:-0} # Protocol fuzzer: 0=off, 1=on (fuzzer uses its own Go-code defaults for weights) - FUZZER_ENABLED=${FUZZER_ENABLED:-0} # diff --git a/workload/cmd/stress-engine/main.go b/workload/cmd/stress-engine/main.go index e88d0b96..62652958 100644 --- a/workload/cmd/stress-engine/main.go +++ b/workload/cmd/stress-engine/main.go @@ -255,6 +255,8 @@ func buildDeck() { {"DoF3FinalityMonitor", "STRESS_WEIGHT_F3_MONITOR", DoF3FinalityMonitor, 2}, {"DoF3FinalityAgreement", "STRESS_WEIGHT_F3_AGREEMENT", DoF3FinalityAgreement, 3}, {"DoDrandBeaconAudit", "STRESS_WEIGHT_DRAND_BEACON_AUDIT", DoDrandBeaconAudit, 3}, + // Quiet recovery: pauses all faults, checks self-healing (off by default, enable via QUIET_RECOVERY_ENABLED=1) + {"DoQuietRecovery", "STRESS_WEIGHT_QUIET_RECOVERY", DoQuietRecovery, 0}, } // Non-FOC stress vectors — skipped when FOC profile is active. diff --git a/workload/cmd/stress-engine/quiet_recovery_vectors.go b/workload/cmd/stress-engine/quiet_recovery_vectors.go new file mode 100644 index 00000000..df99870c --- /dev/null +++ b/workload/cmd/stress-engine/quiet_recovery_vectors.go @@ -0,0 +1,199 @@ +package main + +import ( + "log" + "os" + "os/exec" + "time" + + "github.com/antithesishq/antithesis-sdk-go/assert" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" +) + +// =========================================================================== +// Quiet Recovery Vector +// +// Pauses Antithesis fault injection for a configurable duration, then verifies +// that the Filecoin network self-heals: chain advances, nodes converge, and +// all nodes agree on the tipset at finalized height. +// +// Gated by QUIET_RECOVERY_ENABLED=1 (off by default — pausing faults is +// disruptive to the entire devnet). Enable via the notebook or GH Action toggle. +// +// Requires the ANTITHESIS_STOP_FAULTS binary injected by the Antithesis runtime. +// =========================================================================== + +const ( + quietDuration = "45" // seconds to pause faults (string for exec arg) + quietStabilizeSecs = 15 // seconds to wait for gossip/reconnection after faults pause + quietDriftThreshold = 3 // max block drift to consider nodes "converged" +) + +// DoQuietRecovery requests a fault-free window and verifies chain self-healing. +func DoQuietRecovery() { + if os.Getenv("QUIET_RECOVERY_ENABLED") != "1" { + return + } + + stopBin := os.Getenv("ANTITHESIS_STOP_FAULTS") + if stopBin == "" { + debugLog("[quiet-recovery] ANTITHESIS_STOP_FAULTS not set, skipping") + return + } + + if len(nodeKeys) < 2 { + return + } + + // ── Step 1: Record pre-recovery heights ────────────────────────────────── + preHeights := queryNodeHeights() + preMax := maxEpoch(preHeights) + if preMax == 0 { + log.Println("[quiet-recovery] no responsive nodes, skipping") + return + } + log.Printf("[quiet-recovery] pre-recovery max height: %d", preMax) + + // ── Step 2: Pause fault injection ──────────────────────────────────────── + log.Printf("[quiet-recovery] requesting %ss quiet period", quietDuration) + cmd := exec.CommandContext(ctx, stopBin, quietDuration) + if err := cmd.Run(); err != nil { + log.Printf("[quiet-recovery] ANTITHESIS_STOP_FAULTS failed: %v", err) + return + } + + // ── Step 3: Wait for stabilization ─────────────────────────────────────── + time.Sleep(time.Duration(quietStabilizeSecs) * time.Second) + + // ── Step 4: Record post-recovery heights ───────────────────────────────── + postHeights := queryNodeHeights() + postMax := maxEpoch(postHeights) + postMin := minEpoch(postHeights) + log.Printf("[quiet-recovery] post-recovery max height: %d, min height: %d", postMax, postMin) + + // ── Step 5: Assert chain advanced ──────────────────────────────────────── + advanced := postMax > preMax + assert.Sometimes(advanced, "Chain advanced during quiet period", map[string]any{ + "pre_max_height": preMax, + "post_max_height": postMax, + }) + if advanced { + log.Printf("[quiet-recovery] chain advanced from %d to %d", preMax, postMax) + } else { + log.Printf("[quiet-recovery] chain did NOT advance (pre=%d post=%d)", preMax, postMax) + } + + // ── Step 6: Assert consensus recovery (drift check) ────────────────────── + if len(postHeights) < 2 { + log.Println("[quiet-recovery] fewer than 2 responsive nodes post-recovery, skipping convergence check") + return + } + + drift := int(postMax - postMin) + converged := drift <= quietDriftThreshold + assert.Sometimes(converged, "Consensus recovered during quiet period", map[string]any{ + "drift": drift, + "threshold": quietDriftThreshold, + "nodes": len(postHeights), + }) + + if converged { + log.Printf("[quiet-recovery] consensus recovered (drift=%d <= %d)", drift, quietDriftThreshold) + } else { + log.Printf("[quiet-recovery] consensus NOT recovered (drift=%d > %d)", drift, quietDriftThreshold) + return // don't check tipset agreement when nodes are diverged + } + + // ── Step 7: Assert tipset agreement at finalized height ────────────────── + // Use the minimum post-recovery height minus a small finality buffer as the + // comparison point. All converged nodes should agree on this tipset. + const finalityBuffer = 5 + checkHeight := postMin - abi.ChainEpoch(finalityBuffer) + if checkHeight <= 0 { + log.Println("[quiet-recovery] chain too short for finalized tipset check") + return + } + + var cidStrings []string + var respondents int + for _, name := range nodeKeys { + h, ok := postHeights[name] + if !ok || h == 0 { + continue + } + ts, err := nodes[name].ChainGetTipSetByHeight(ctx, checkHeight, types.EmptyTSK) + if err != nil { + debugLog("[quiet-recovery] ChainGetTipSetByHeight(%d) failed on %s: %v", checkHeight, name, err) + continue + } + cids := "" + for _, c := range ts.Cids() { + cids += c.String() + "," + } + cidStrings = append(cidStrings, cids) + respondents++ + } + + if respondents < 2 { + log.Printf("[quiet-recovery] only %d nodes returned tipsets at height %d, skipping agreement check", respondents, checkHeight) + return + } + + allAgree := true + for i := 1; i < len(cidStrings); i++ { + if cidStrings[i] != cidStrings[0] { + allAgree = false + break + } + } + + assert.Always(allAgree, "State consistent after quiet period recovery", map[string]any{ + "check_height": checkHeight, + "respondents": respondents, + "drift": drift, + }) + + if allAgree { + log.Printf("[quiet-recovery] all %d nodes agree on tipset at height %d", respondents, checkHeight) + } else { + log.Printf("[quiet-recovery] TIPSET DISAGREEMENT at height %d among %d nodes", checkHeight, respondents) + } +} + +// queryNodeHeights returns the chain head height for each connected node. +func queryNodeHeights() map[string]abi.ChainEpoch { + heights := make(map[string]abi.ChainEpoch, len(nodeKeys)) + for _, name := range nodeKeys { + head, err := nodes[name].ChainHead(ctx) + if err != nil { + debugLog("[quiet-recovery] ChainHead failed on %s: %v", name, err) + continue + } + heights[name] = head.Height() + } + return heights +} + +// maxEpoch returns the maximum height from a height map. +func maxEpoch(heights map[string]abi.ChainEpoch) abi.ChainEpoch { + var max abi.ChainEpoch + for _, h := range heights { + if h > max { + max = h + } + } + return max +} + +// minEpoch returns the minimum height from a height map (ignoring zeros). +func minEpoch(heights map[string]abi.ChainEpoch) abi.ChainEpoch { + var min abi.ChainEpoch + for _, h := range heights { + if h > 0 && (min == 0 || h < min) { + min = h + } + } + return min +} From 78b2bf4244e0f97a944c37ccbf798d6935c3f65c Mon Sep 17 00:00:00 2001 From: asgharmusani-antithesis-dev Date: Tue, 21 Apr 2026 11:01:32 -0400 Subject: [PATCH 2/8] added logs to see stopBin Signed-off-by: asgharmusani-antithesis-dev --- workload/cmd/stress-engine/quiet_recovery_vectors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workload/cmd/stress-engine/quiet_recovery_vectors.go b/workload/cmd/stress-engine/quiet_recovery_vectors.go index df99870c..4b4fadee 100644 --- a/workload/cmd/stress-engine/quiet_recovery_vectors.go +++ b/workload/cmd/stress-engine/quiet_recovery_vectors.go @@ -57,7 +57,7 @@ func DoQuietRecovery() { log.Printf("[quiet-recovery] pre-recovery max height: %d", preMax) // ── Step 2: Pause fault injection ──────────────────────────────────────── - log.Printf("[quiet-recovery] requesting %ss quiet period", quietDuration) + log.Printf("[quiet-recovery] requesting %ss quiet period via: %s", quietDuration, stopBin) cmd := exec.CommandContext(ctx, stopBin, quietDuration) if err := cmd.Run(); err != nil { log.Printf("[quiet-recovery] ANTITHESIS_STOP_FAULTS failed: %v", err) From 2963028f643190553f52e4e968442d49c596375f Mon Sep 17 00:00:00 2001 From: asgharmusani-antithesis-dev Date: Tue, 21 Apr 2026 12:26:44 -0400 Subject: [PATCH 3/8] changed finalityBuffer to 10 and add some logging Signed-off-by: asgharmusani-antithesis-dev --- .../stress-engine/quiet_recovery_vectors.go | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/workload/cmd/stress-engine/quiet_recovery_vectors.go b/workload/cmd/stress-engine/quiet_recovery_vectors.go index 4b4fadee..e80b6c9b 100644 --- a/workload/cmd/stress-engine/quiet_recovery_vectors.go +++ b/workload/cmd/stress-engine/quiet_recovery_vectors.go @@ -29,6 +29,7 @@ const ( quietDuration = "45" // seconds to pause faults (string for exec arg) quietStabilizeSecs = 15 // seconds to wait for gossip/reconnection after faults pause quietDriftThreshold = 3 // max block drift to consider nodes "converged" + quietFinalityBuffer = 10 // epochs below min post-recovery height to check tipset agreement ) // DoQuietRecovery requests a fault-free window and verifies chain self-healing. @@ -54,6 +55,9 @@ func DoQuietRecovery() { log.Println("[quiet-recovery] no responsive nodes, skipping") return } + for name, h := range preHeights { + log.Printf("[quiet-recovery] pre-height %s: %d", name, h) + } log.Printf("[quiet-recovery] pre-recovery max height: %d", preMax) // ── Step 2: Pause fault injection ──────────────────────────────────────── @@ -71,6 +75,9 @@ func DoQuietRecovery() { postHeights := queryNodeHeights() postMax := maxEpoch(postHeights) postMin := minEpoch(postHeights) + for name, h := range postHeights { + log.Printf("[quiet-recovery] post-height %s: %d", name, h) + } log.Printf("[quiet-recovery] post-recovery max height: %d, min height: %d", postMax, postMin) // ── Step 5: Assert chain advanced ──────────────────────────────────────── @@ -109,7 +116,6 @@ func DoQuietRecovery() { // ── Step 7: Assert tipset agreement at finalized height ────────────────── // Use the minimum post-recovery height minus a small finality buffer as the // comparison point. All converged nodes should agree on this tipset. - const finalityBuffer = 5 checkHeight := postMin - abi.ChainEpoch(finalityBuffer) if checkHeight <= 0 { log.Println("[quiet-recovery] chain too short for finalized tipset check") @@ -117,6 +123,7 @@ func DoQuietRecovery() { } var cidStrings []string + var cidByNode []string // "node=CIDs" for logging var respondents int for _, name := range nodeKeys { h, ok := postHeights[name] @@ -133,8 +140,11 @@ func DoQuietRecovery() { cids += c.String() + "," } cidStrings = append(cidStrings, cids) + cidByNode = append(cidByNode, name+"="+cids) respondents++ } + log.Printf("[quiet-recovery] tipsets at height %d: %v", checkHeight, cidByNode) + if respondents < 2 { log.Printf("[quiet-recovery] only %d nodes returned tipsets at height %d, skipping agreement check", respondents, checkHeight) @@ -159,6 +169,21 @@ func DoQuietRecovery() { log.Printf("[quiet-recovery] all %d nodes agree on tipset at height %d", respondents, checkHeight) } else { log.Printf("[quiet-recovery] TIPSET DISAGREEMENT at height %d among %d nodes", checkHeight, respondents) + for _, entry := range cidByNode { + log.Printf("[quiet-recovery] %s", entry) + } + // Also check head tipset to see if disagreement extends to the tip + for _, name := range nodeKeys { + head, err := nodes[name].ChainHead(ctx) + if err != nil { + continue + } + headCids := "" + for _, c := range head.Cids() { + headCids += c.String() + "," + } + log.Printf("[quiet-recovery] head-tipset %s (height=%d): %s", name, head.Height(), headCids) + } } } From f609a9cf6cd765755a4a6bea184698aaa6fa8a26 Mon Sep 17 00:00:00 2001 From: asgharmusani-antithesis-dev Date: Tue, 21 Apr 2026 12:33:31 -0400 Subject: [PATCH 4/8] Update quiet_recovery_vectors.go Signed-off-by: asgharmusani-antithesis-dev --- workload/cmd/stress-engine/quiet_recovery_vectors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workload/cmd/stress-engine/quiet_recovery_vectors.go b/workload/cmd/stress-engine/quiet_recovery_vectors.go index e80b6c9b..a8e4f5b7 100644 --- a/workload/cmd/stress-engine/quiet_recovery_vectors.go +++ b/workload/cmd/stress-engine/quiet_recovery_vectors.go @@ -116,7 +116,7 @@ func DoQuietRecovery() { // ── Step 7: Assert tipset agreement at finalized height ────────────────── // Use the minimum post-recovery height minus a small finality buffer as the // comparison point. All converged nodes should agree on this tipset. - checkHeight := postMin - abi.ChainEpoch(finalityBuffer) + checkHeight := postMin - abi.ChainEpoch(quietFinalityBuffer) if checkHeight <= 0 { log.Println("[quiet-recovery] chain too short for finalized tipset check") return From 433bc6673ccddd9daa2093b564a33534a5610029 Mon Sep 17 00:00:00 2001 From: Asghar Musani Date: Thu, 23 Apr 2026 14:09:52 -0400 Subject: [PATCH 5/8] Add rate limiting to DoQuietRecovery: random 0-5 max executions with randomized cooldowns Signed-off-by: Asghar Musani --- .../stress-engine/quiet_recovery_vectors.go | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/workload/cmd/stress-engine/quiet_recovery_vectors.go b/workload/cmd/stress-engine/quiet_recovery_vectors.go index a8e4f5b7..1c8ca0f6 100644 --- a/workload/cmd/stress-engine/quiet_recovery_vectors.go +++ b/workload/cmd/stress-engine/quiet_recovery_vectors.go @@ -26,14 +26,38 @@ import ( // =========================================================================== const ( - quietDuration = "45" // seconds to pause faults (string for exec arg) - quietStabilizeSecs = 15 // seconds to wait for gossip/reconnection after faults pause - quietDriftThreshold = 3 // max block drift to consider nodes "converged" + quietDuration = "45" // seconds to pause faults (string for exec arg) + quietStabilizeSecs = 15 // seconds to wait for gossip/reconnection after faults pause + quietDriftThreshold = 3 // max block drift to consider nodes "converged" quietFinalityBuffer = 10 // epochs below min post-recovery height to check tipset agreement ) +var ( + quietRecoveryRemaining = -1 // -1 = not yet initialized; 0–5 once set + quietRecoveryEarliestRun time.Time // earliest wall-clock time the next execution is allowed +) + // DoQuietRecovery requests a fault-free window and verifies chain self-healing. func DoQuietRecovery() { + // One-time init: pick 0–5 total executions, delay first by 1–5 min + if quietRecoveryRemaining == -1 { + quietRecoveryRemaining = rngIntn(6) + quietRecoveryEarliestRun = time.Now().Add(time.Duration(60+rngIntn(240)) * time.Second) + log.Printf("[quiet-recovery] will fire at most %d times, first eligible at %v", quietRecoveryRemaining, quietRecoveryEarliestRun) + } + + if quietRecoveryRemaining <= 0 { + return + } + + if time.Now().Before(quietRecoveryEarliestRun) { + return + } + + quietRecoveryRemaining-- + quietRecoveryEarliestRun = time.Now().Add(time.Duration(120+rngIntn(300)) * time.Second) + log.Printf("[quiet-recovery] firing (remaining=%d, next eligible at %v)", quietRecoveryRemaining, quietRecoveryEarliestRun) + if os.Getenv("QUIET_RECOVERY_ENABLED") != "1" { return } From 2b8bb6bbe624b8825c916e3e471de48a4ce98774 Mon Sep 17 00:00:00 2001 From: Asghar Musani Date: Mon, 27 Apr 2026 13:17:59 -0400 Subject: [PATCH 6/8] first on first + move QUIET_RECOVERY_ENABLED as first check Signed-off-by: Asghar Musani --- .../cmd/stress-engine/quiet_recovery_vectors.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/workload/cmd/stress-engine/quiet_recovery_vectors.go b/workload/cmd/stress-engine/quiet_recovery_vectors.go index 1c8ca0f6..6a2241a6 100644 --- a/workload/cmd/stress-engine/quiet_recovery_vectors.go +++ b/workload/cmd/stress-engine/quiet_recovery_vectors.go @@ -39,11 +39,14 @@ var ( // DoQuietRecovery requests a fault-free window and verifies chain self-healing. func DoQuietRecovery() { - // One-time init: pick 0–5 total executions, delay first by 1–5 min + if os.Getenv("QUIET_RECOVERY_ENABLED") != "1" { + return + } + + // One-time init: pick 0–5 total executions, first fires immediately if quietRecoveryRemaining == -1 { quietRecoveryRemaining = rngIntn(6) - quietRecoveryEarliestRun = time.Now().Add(time.Duration(60+rngIntn(240)) * time.Second) - log.Printf("[quiet-recovery] will fire at most %d times, first eligible at %v", quietRecoveryRemaining, quietRecoveryEarliestRun) + log.Printf("[quiet-recovery] will fire at most %d times", quietRecoveryRemaining) } if quietRecoveryRemaining <= 0 { @@ -58,10 +61,6 @@ func DoQuietRecovery() { quietRecoveryEarliestRun = time.Now().Add(time.Duration(120+rngIntn(300)) * time.Second) log.Printf("[quiet-recovery] firing (remaining=%d, next eligible at %v)", quietRecoveryRemaining, quietRecoveryEarliestRun) - if os.Getenv("QUIET_RECOVERY_ENABLED") != "1" { - return - } - stopBin := os.Getenv("ANTITHESIS_STOP_FAULTS") if stopBin == "" { debugLog("[quiet-recovery] ANTITHESIS_STOP_FAULTS not set, skipping") From 14b96357ae01dba9d7a0fbc32a4e32e116729a01 Mon Sep 17 00:00:00 2001 From: Asghar Musani Date: Mon, 27 Apr 2026 14:59:12 -0400 Subject: [PATCH 7/8] skip if nsplit or reorg chaos is active Signed-off-by: Asghar Musani --- workload/cmd/stress-engine/quiet_recovery_vectors.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/workload/cmd/stress-engine/quiet_recovery_vectors.go b/workload/cmd/stress-engine/quiet_recovery_vectors.go index 6a2241a6..aefdf950 100644 --- a/workload/cmd/stress-engine/quiet_recovery_vectors.go +++ b/workload/cmd/stress-engine/quiet_recovery_vectors.go @@ -43,6 +43,15 @@ func DoQuietRecovery() { return } + // Skip if a workload-driven partition is active (n-split lifecycle or + // reorg-chaos). ANTITHESIS_STOP_FAULTS only pauses the fault injector, + // not RPC-applied NetBlockAdd/NetDisconnect from other vectors, so the + // drift assertion would falsely fail on the still-isolated nodes. + if partitionActive.Load() { + debugLog("[quiet-recovery] skipping — partition already active") + return + } + // One-time init: pick 0–5 total executions, first fires immediately if quietRecoveryRemaining == -1 { quietRecoveryRemaining = rngIntn(6) From d98423bba759c78915790ca8aaa94287a6bef993 Mon Sep 17 00:00:00 2001 From: Asghar Musani Date: Tue, 28 Apr 2026 16:39:06 -0400 Subject: [PATCH 8/8] bring back first init delay Signed-off-by: Asghar Musani --- workload/cmd/stress-engine/quiet_recovery_vectors.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workload/cmd/stress-engine/quiet_recovery_vectors.go b/workload/cmd/stress-engine/quiet_recovery_vectors.go index aefdf950..332c9a5a 100644 --- a/workload/cmd/stress-engine/quiet_recovery_vectors.go +++ b/workload/cmd/stress-engine/quiet_recovery_vectors.go @@ -52,10 +52,11 @@ func DoQuietRecovery() { return } - // One-time init: pick 0–5 total executions, first fires immediately + // One-time init: pick 0–5 total executions, delay first by 0–3 min if quietRecoveryRemaining == -1 { quietRecoveryRemaining = rngIntn(6) - log.Printf("[quiet-recovery] will fire at most %d times", quietRecoveryRemaining) + quietRecoveryEarliestRun = time.Now().Add(time.Duration(rngIntn(180)) * time.Second) + log.Printf("[quiet-recovery] will fire at most %d times, first eligible at %v", quietRecoveryRemaining, quietRecoveryEarliestRun) } if quietRecoveryRemaining <= 0 {