From f5ee3441fa1ef6b970fd654ec7d3d1e3f0913ab3 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Mon, 24 Aug 2026 19:01:10 +0300 Subject: [PATCH 01/18] Extend persist flag with a way to specify number of slots to be valid and a force stop if needed --- crates/core/src/scenarios/README.md | 13 +- crates/core/src/surfnet/svm.rs | 194 ++++++++++++++++++++++++++-- crates/types/src/rpc_endpoints.json | 2 +- crates/types/src/scenarios.rs | 103 ++++++++++++++- 4 files changed, 295 insertions(+), 17 deletions(-) diff --git a/crates/core/src/scenarios/README.md b/crates/core/src/scenarios/README.md index 5d8176be..f392741a 100644 --- a/crates/core/src/scenarios/README.md +++ b/crates/core/src/scenarios/README.md @@ -34,9 +34,16 @@ whole struct or array) also works, but it must be **complete** - every field of padding included - because the account is re-encoded with Borsh. An out-of-range index or a non-numeric segment on an array is a hard error, never a silent write elsewhere. -By default an override applies to exactly one slot. Set `"persist": true` and it is re-applied on -every following slot, which is needed when something else writes the account in between - a -transaction, or another override fetching it fresh. Persist inputs nothing in the scenario writes +By default an override applies to exactly one slot. `"persist"` controls how long it keeps +re-applying: `true` re-applies indefinitely and `{"slots": 10}` applies in ten slots in total, counting the +first. Prefer a +bounded window - an indefinite one outlives the scenario that created it. + +Registering a scenario replaces any override it has already queued, matched on id, account and +template together. That is how an override is updated, and how a persisted one is stopped: register +it again with `"persist": false` and the queued copy becomes a one-shot, so it stops re-arming. +Re-applying is what you want when something else writes the account in between - a transaction, or +another override fetching it fresh. Persist inputs nothing in the scenario writes (an oracle price, a disabled switch, a risk parameter), never state the transactions under test mutate: re-applying reverts their writes at the start of the next slot, so a pool would refill itself after every swap. Only one entry is queued per override, so it is never applied twice to diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index 7fa06023..09598ddd 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -2805,7 +2805,7 @@ impl SurfnetSvm { } // Queued before the write so a failed apply is retried next slot, still fetching. - if override_instance.persist { + if override_instance.persist.is_enabled() { self.reschedule_override_for_next_slot(&override_instance, target_slot); } @@ -2879,7 +2879,7 @@ impl SurfnetSvm { account_pubkey ); settled_this_slot.insert(account_pubkey); - if override_instance.persist && override_instance.fetch_before_use { + if override_instance.persist.is_enabled() && override_instance.fetch_before_use { let mut requeued = override_instance.clone(); requeued.fetch_before_use = false; self.reschedule_override_for_next_slot(&requeued, target_slot); @@ -2986,7 +2986,7 @@ impl SurfnetSvm { // The account is forked now. Re-fetching it every slot would cost one RPC // per slot and overwrite whatever local transactions wrote to the fields // this override leaves alone, so later slots re-pin without fetching. - if override_instance.persist && override_instance.fetch_before_use { + if override_instance.persist.is_enabled() && override_instance.fetch_before_use { let mut requeued = override_instance.clone(); requeued.fetch_before_use = false; self.reschedule_override_for_next_slot(&requeued, target_slot); @@ -3006,6 +3006,18 @@ impl SurfnetSvm { target_slot: Slot, ) { let next_slot = target_slot + 1; + + let Some(next_persist) = instance.persist.next_arming() else { + debug!( + "Override {} has reached the end of its persist window at slot {}", + instance.id, target_slot + ); + return; + }; + let mut instance = instance.clone(); + instance.persist = next_persist; + let instance = &instance; + let mut next = self .scheduled_overrides .get(&next_slot) @@ -4291,7 +4303,20 @@ impl SurfnetSvm { .ok() .flatten() .unwrap_or_default(); - slot_overrides.push(override_instance); + + if let Some(existing) = slot_overrides.iter_mut().find(|queued| { + queued.id == override_instance.id + && queued.account == override_instance.account + && queued.template_id == override_instance.template_id + }) { + debug!( + "Replacing already-scheduled override {} at slot {}", + override_instance.id, absolute_slot + ); + *existing = override_instance; + } else { + slot_overrides.push(override_instance); + } self.scheduled_overrides .store(absolute_slot, slot_overrides)?; } @@ -7114,7 +7139,7 @@ mod tests { "unhealthy_borrow_value_sf".to_string(), serde_json::json!(1_234u64), )])); - instance.persist = persist; + instance.persist = surfpool_types::Persist::Always(persist); (surfnet_svm, account_pubkey, instance) } @@ -7156,7 +7181,10 @@ mod tests { "exactly one override queued for the next slot" ); assert_eq!(next[0].id, instance_id); - assert!(next[0].persist, "persist flag must survive rescheduling"); + assert!( + next[0].persist.is_enabled(), + "persist flag must survive rescheduling" + ); assert!( svm.scheduled_overrides @@ -7187,7 +7215,10 @@ mod tests { .expect("storage read") .expect("next slot should have queued overrides"); assert_eq!(next.len(), 1, "one entry per override id"); - assert!(next[0].persist, "persist must survive rescheduling"); + assert!( + next[0].persist.is_enabled(), + "persist must survive rescheduling" + ); assert!( !next[0].fetch_before_use, "the account is forked, so later slots must not re-fetch it and discard local writes" @@ -7258,7 +7289,7 @@ mod tests { ); // The collision this guards against: a hand-written scenario reusing a plain id. first.id = "ov-1".to_string(); - first.persist = true; + first.persist = surfpool_types::Persist::Always(true); let mut second = first.clone(); second.account = surfpool_types::AccountAddress::Pubkey(second_account.to_string()); @@ -7291,6 +7322,153 @@ mod tests { ); } + /// A relative window re-arms for exactly the requested number of slots and then stops. + #[tokio::test] + async fn test_persist_for_slots_stops_after_the_window() { + const SLOT: u64 = 500; + const WINDOW: u64 = 3; + + let (mut svm, _account_pubkey, mut instance) = scheduled_persist_fixture(true); + instance.persist = surfpool_types::Persist::Slots { slots: WINDOW }; + svm.scheduled_overrides + .store(SLOT, vec![instance]) + .expect("schedule override"); + + // Walk the slots and record the last one the override was still queued for. + let mut last_queued = SLOT; + for slot in SLOT..=SLOT + WINDOW + 2 { + let queued = svm + .scheduled_overrides + .get(&slot) + .expect("read") + .unwrap_or_default(); + if queued.is_empty() { + break; + } + last_queued = slot; + svm.materialize_overrides_for_slot(&None, slot) + .await + .expect("materialize"); + } + + assert_eq!( + last_queued, + SLOT + WINDOW - 1, + "a window of {WINDOW} slots means {WINDOW} applications in total, so slots {SLOT} \ + through {} and then stop", + SLOT + WINDOW - 1 + ); + } + + /// Re-arming twice within one slot must burn exactly one slot of the window. + #[tokio::test] + async fn test_window_burns_one_slot_even_when_rearmed_twice() { + const SLOT: u64 = 500; + + let (mut svm, _account_pubkey, mut instance) = scheduled_persist_fixture(true); + instance.persist = surfpool_types::Persist::Slots { slots: 3 }; + + // Two reschedules for the same slot, exactly as the real flow does. + svm.reschedule_override_for_next_slot(&instance, SLOT); + svm.reschedule_override_for_next_slot(&instance, SLOT); + + let queued = svm + .scheduled_overrides + .get(&(SLOT + 1)) + .expect("read") + .expect("queued"); + assert_eq!( + queued.len(), + 1, + "re-arming twice must not duplicate the override" + ); + assert_eq!( + queued[0].persist, + surfpool_types::Persist::Slots { slots: 2 }, + "a window of 3 must have exactly 2 slots left after one slot, not 1" + ); + + // The last slot of the window does not re-arm. + let mut last = instance.clone(); + last.persist = surfpool_types::Persist::Slots { slots: 1 }; + svm.reschedule_override_for_next_slot(&last, SLOT + 5); + assert!( + svm.scheduled_overrides + .get(&(SLOT + 6)) + .expect("read") + .unwrap_or_default() + .is_empty(), + "a spent window must not queue anything further" + ); + } + + /// Re-registering a scenario updates its overrides instead of queueing a second copy. + #[tokio::test] + async fn test_re_registering_a_scenario_replaces_rather_than_duplicates() { + const SLOT: u64 = 500; + + let (mut svm, _account_pubkey, instance) = scheduled_persist_fixture(true); + let scenario = |instance: surfpool_types::OverrideInstance| surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![instance], + }; + + svm.register_scenario(scenario(instance.clone()), Some(SLOT)) + .expect("register once"); + svm.register_scenario(scenario(instance.clone()), Some(SLOT)) + .expect("register again"); + + let queued = svm + .scheduled_overrides + .get(&SLOT) + .expect("read") + .expect("queued"); + assert_eq!( + queued.len(), + 1, + "registering the same override twice must update it, not queue it twice" + ); + } + + /// A persisted override is stopped by re-registering it as a one-shot. + #[tokio::test] + async fn test_re_registering_with_persist_false_cancels_a_persisted_override() { + const SLOT: u64 = 500; + + let (mut svm, _account_pubkey, mut instance) = scheduled_persist_fixture(true); + let scenario = |instance: surfpool_types::OverrideInstance| surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![instance], + }; + + svm.register_scenario(scenario(instance.clone()), Some(SLOT)) + .expect("register persisted"); + + // Cancel it: same id, same account, same template, but a one-shot. + instance.persist = surfpool_types::Persist::Always(false); + svm.register_scenario(scenario(instance), Some(SLOT)) + .expect("register the cancelling one-shot"); + + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect("materialize"); + + assert!( + svm.scheduled_overrides + .get(&(SLOT + 1)) + .expect("read") + .unwrap_or_default() + .is_empty(), + "the override was cancelled, so it must not re-arm for the next slot" + ); + } + #[tokio::test] async fn test_non_persisted_override_is_not_rescheduled() { const SLOT: u64 = 500; diff --git a/crates/types/src/rpc_endpoints.json b/crates/types/src/rpc_endpoints.json index 740e1e8a..f45cc118 100644 --- a/crates/types/src/rpc_endpoints.json +++ b/crates/types/src/rpc_endpoints.json @@ -861,7 +861,7 @@ "label": "Option (An optional label for this override instance)", "enabled": "bool (Indicates whether this override instance is enabled)", "fetchBeforeUse": "bool (Indicates whether to fetch the latest on-chain account data before applying overrides)", - "persist": "bool (Optional, defaults to false. If true, re-applies this override on every following slot instead of only one, which is needed when something else writes the account in between. Use it only for values no transaction writes - an oracle price, a disabled switch, a risk parameter - never for state the transactions under test mutate: re-applying reverts their writes at the start of the next slot, so a pool would refill itself after every swap)", + "persist": "Optional, defaults to false. Controls how long the override keeps re-applying itself. Accepts a boolean or a bounded window: false applies once; true re-applies every following slot indefinitely; {\"slots\": 10} applies in 10 slots in total, counting the first. To stop a persisted override, register the same override again (same id, account and template) with persist false - it replaces the queued copy with a one-shot. Re-applying is needed when something else writes the account in between, but use it only for values no transaction writes - an oracle price, a disabled switch, a risk parameter - never for state the transactions under test mutate: re-applying reverts their writes at the start of the next slot, so a pool would refill itself after every swap. Prefer a bounded window over true, so a persisted override cannot outlive the scenario that created it.", "account": "AccountAddress (The account this override targets, as {\"pubkey\": \"\"} or {\"pda\": {\"programId\": \"\", \"seeds\": [ ... ]}})" } } diff --git a/crates/types/src/scenarios.rs b/crates/types/src/scenarios.rs index 40cbcb0e..1c807f53 100644 --- a/crates/types/src/scenarios.rs +++ b/crates/types/src/scenarios.rs @@ -493,6 +493,44 @@ impl OverrideTemplate { } } +/// How long an override keeps re-applying itself. +/// +/// persist: false // apply once (default) +/// persist: true // re-apply every following slot, indefinitely +/// persist: { slots: 10 } // apply in 10 slots in total, counting the first +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(untagged)] +#[cfg_attr(feature = "ts-bindings", derive(ts_rs::TS))] +pub enum Persist { + Always(bool), + Slots { + slots: Slot, + }, +} + +impl Default for Persist { + fn default() -> Self { + Persist::Always(false) + } +} + +impl Persist { + pub fn is_enabled(&self) -> bool { + !matches!(self, Persist::Always(false)) + } + + pub fn next_arming(&self) -> Option { + match self { + Persist::Always(false) => None, + Persist::Always(true) => Some(Persist::Always(true)), + Persist::Slots { slots } => match slots.checked_sub(1) { + None | Some(0) => None, + Some(remaining) => Some(Persist::Slots { slots: remaining }), + }, + } + } +} + /// A concrete instance of an override template with specific values #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] #[serde(rename_all = "camelCase")] @@ -537,8 +575,8 @@ pub struct OverrideInstance { description = "If true, re-applies this override every following slot. Use only for values no transaction writes: it reverts transaction writes to the same fields." )] #[serde(default)] - #[cfg_attr(feature = "ts-bindings", ts(as = "Option", optional))] - pub persist: bool, + #[cfg_attr(feature = "ts-bindings", ts(type = "boolean | { slots: number }", optional))] + pub persist: Persist, /// Account address to override - use pubkey for known addresses or pda for derived addresses #[schemars( description = "Account address: either {\"pubkey\": \"base58_address\"} or {\"pda\": {\"programId\": \"...\", \"seeds\": [...]}}" @@ -556,7 +594,7 @@ impl OverrideInstance { label: None, enabled: true, fetch_before_use: false, - persist: false, + persist: Persist::default(), account, } } @@ -572,10 +610,14 @@ impl OverrideInstance { } pub fn with_persist(mut self, persist: bool) -> Self { - self.persist = persist; + self.persist = Persist::Always(persist); self } -} + + pub fn with_persist_for_slots(mut self, slots: Slot) -> Self { + self.persist = Persist::Slots { slots }; + self + }} /// A scenario containing a timeline of overrides #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] @@ -1542,6 +1584,57 @@ mod tests { assert_eq!(u64::from_le_bytes(bytes.try_into().unwrap()), 0); } + #[test] + fn persist_accepts_the_legacy_boolean_and_the_bounded_forms() { + use super::Persist; + + let cases = [ + ("false", Persist::Always(false)), + ("true", Persist::Always(true)), + (r#"{"slots":10}"#, Persist::Slots { slots: 10 }), + (r#"{"slots":1}"#, Persist::Slots { slots: 1 }), + ]; + + for (json, expected) in cases { + let parsed: Persist = serde_json::from_str(json) + .unwrap_or_else(|e| panic!("{json} should deserialize: {e}")); + assert_eq!(parsed, expected, "{json} deserialized wrongly"); + } + + assert_eq!( + serde_json::to_string(&Persist::Always(true)).expect("serialize"), + "true" + ); + } + + /// The window is a bound, not a countdown, and resolving is idempotent. + #[test] + fn persist_window_resolves_to_an_absolute_end_and_stops_there() { + use super::Persist; + + // A window counts down one slot per re-arm and then stops. + assert_eq!( + Persist::Slots { slots: 3 }.next_arming(), + Some(Persist::Slots { slots: 2 }) + ); + assert_eq!( + Persist::Slots { slots: 1 }.next_arming(), + None, + "the last slot of the window must not re-arm" + ); + assert_eq!(Persist::Slots { slots: 0 }.next_arming(), None); + + assert_eq!(Persist::Always(false).next_arming(), None); + assert_eq!( + Persist::Always(true).next_arming(), + Some(Persist::Always(true)), + "an indefinite persist never runs out" + ); + + assert!(!Persist::Always(false).is_enabled()); + assert!(Persist::Always(true).is_enabled()); + } + #[test] fn raw_layout_rejects_writes_past_the_end_of_the_account() { use super::{Property, RawEncoding, RawLayout}; From ac543770bc7886ca0c0aa4cdd645ab0d31eb02f8 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Mon, 24 Aug 2026 19:01:29 +0300 Subject: [PATCH 02/18] Extend persist flag with a way to specify number of slots to be valid and a force stop if needed --- .github/{ => workflows}/openai-review.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/openai-review.yml (100%) diff --git a/.github/openai-review.yml b/.github/workflows/openai-review.yml similarity index 100% rename from .github/openai-review.yml rename to .github/workflows/openai-review.yml From f3a42fe63136ca8fedfb0b32498bbf41c06f9eea Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Mon, 24 Aug 2026 19:10:07 +0300 Subject: [PATCH 03/18] Fix templates to include perists better --- .../src/scenarios/protocols/bisonfi/overrides.yaml | 5 ++++- .../protocols/kamino/scope/v1/overrides.yaml | 8 ++++++-- .../src/scenarios/protocols/kamino/v1/overrides.yaml | 12 ++++++++---- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/core/src/scenarios/protocols/bisonfi/overrides.yaml b/crates/core/src/scenarios/protocols/bisonfi/overrides.yaml index 29d8fd34..e8739a29 100644 --- a/crates/core/src/scenarios/protocols/bisonfi/overrides.yaml +++ b/crates/core/src/scenarios/protocols/bisonfi/overrides.yaml @@ -255,7 +255,10 @@ templates: HOW TO USE THIS TEMPLATE: 1. Set last_update_slot to the current slot. The venue resumes quoting the price it already held - a fresh timestamp is enough, no new price is needed - 2. Set persist: true, or the next slot's state overwrites your value + 2. Set persist to a window covering your scenario - persist: { slots: N } for a scenario + spanning N slots - or the next slot's state overwrites your value. Prefer that to + persist: true, which never expires: the venue would stay artificially alive for every later + scenario in the same surfnet run A scenario that executes within a slot of forking does not need this. One that spends longer on setup does. diff --git a/crates/core/src/scenarios/protocols/kamino/scope/v1/overrides.yaml b/crates/core/src/scenarios/protocols/kamino/scope/v1/overrides.yaml index 9cb81179..d400d396 100644 --- a/crates/core/src/scenarios/protocols/kamino/scope/v1/overrides.yaml +++ b/crates/core/src/scenarios/protocols/kamino/scope/v1/overrides.yaml @@ -45,8 +45,12 @@ templates: [210, 3] means price = prices[210] * prices[3] 4. Set price.value = usd_price * 10^exp, keeping exp as you found it 5. Set last_updated_slot and unix_timestamp to now, or Kamino rejects the price as stale - 6. Set persist: true if the scenario runs past one slot, so a transaction that writes - this account cannot restore the real price. Safe here: nothing in a fork cranks Scope + 6. Set persist to a window covering your scenario if it runs past one slot, so a transaction + that writes this account cannot restore the real price - persist: { slots: N } where N is + how many slots the scenario spans. Prefer that to persist: true, which never expires: a + pinned price would then leak into every later scenario in the same surfnet run, and the + only symptom is numbers that are inexplicably wrong. Safe here: nothing in a fork cranks + Scope SCOPE INDICES (verified 2026-08-06, do not guess these): - 3t4JZcueEzTbVP6kLxXrL3VpWx45jDer4eqysweBchNH (Main Market): diff --git a/crates/core/src/scenarios/protocols/kamino/v1/overrides.yaml b/crates/core/src/scenarios/protocols/kamino/v1/overrides.yaml index 7ab89e5f..f6f02d64 100644 --- a/crates/core/src/scenarios/protocols/kamino/v1/overrides.yaml +++ b/crates/core/src/scenarios/protocols/kamino/v1/overrides.yaml @@ -388,8 +388,10 @@ templates: EXAMPLE - "liquidate SOL collateral above 50% LTV": config.liquidation_threshold_pct: 50 - persist: true is safe for the config.* fields only. liquidity.* and last_update.* are - rewritten by refresh_reserve, so pinning them fights every transaction that touches the reserve. + Persisting is safe for the config.* fields only. liquidity.* and last_update.* are rewritten by + refresh_reserve, so pinning them fights every transaction that touches the reserve. Bound the + window to the scenario - persist: { slots: N } - rather than persist: true, which never expires + and leaves the parameter pinned for every later scenario in the same surfnet run. - id: kamino-reserve-main-usdc name: Override USDC Reserve (Main Market) description: Override the USDC reserve of Kamino's Main Market @@ -422,8 +424,10 @@ templates: EXAMPLE - "USDC depegs to $0.90": use kamino-scope-price with prices.13.price.value: 90000000 and prices.13.price.exp: 8 - persist: true is safe for the config.* fields only. liquidity.* and last_update.* are - rewritten by refresh_reserve, so pinning them fights every transaction that touches the reserve. + Persisting is safe for the config.* fields only. liquidity.* and last_update.* are rewritten by + refresh_reserve, so pinning them fights every transaction that touches the reserve. Bound the + window to the scenario - persist: { slots: N } - rather than persist: true, which never expires + and leaves the parameter pinned for every later scenario in the same surfnet run. # ========================================== # Obligation # ========================================== From c7dbff67be0da0a3bf2713932574b1746e062086 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Tue, 25 Aug 2026 09:02:50 +0300 Subject: [PATCH 04/18] Fixed broken storage clear --- crates/core/src/surfnet/svm.rs | 109 ++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 8 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index 09598ddd..aaa93c32 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -3018,12 +3018,18 @@ impl SurfnetSvm { instance.persist = next_persist; let instance = &instance; - let mut next = self - .scheduled_overrides - .get(&next_slot) - .ok() - .flatten() - .unwrap_or_default(); + let mut next = match self.scheduled_overrides.get(&next_slot) { + Ok(Some(queued)) => queued, + Ok(None) => Vec::new(), + Err(e) => { + error!( + "Could not read the overrides queued for slot {next_slot} ({e}); not \ + rescheduling override {} rather than risk overwriting them", + instance.id + ); + return; + } + }; if let Some(existing) = next.iter_mut().find(|queued| { queued.id == instance.id @@ -3035,8 +3041,11 @@ impl SurfnetSvm { next.push(instance.clone()); } if let Err(e) = self.scheduled_overrides.store(next_slot, next) { - warn!( - "Failed to reschedule override {} for slot {}: {}", + // Nothing upstream can recover from this: the caller is the slot tick, not a request, so + // there is no response to fail. The override simply stops persisting from here on, which + // is worth an error rather than a warning. + error!( + "Failed to reschedule override {} for slot {}: {}. It will stop persisting.", instance.id, next_slot, e ); } @@ -7469,6 +7478,90 @@ mod tests { ); } + #[tokio::test] + async fn test_failed_read_does_not_overwrite_the_next_slots_overrides() { + use crate::storage::{Storage, StorageError, StorageResult}; + + #[derive(Clone)] + struct FailingReads { + inner: HashMap>, + writes: std::sync::Arc, + } + + impl Storage> for FailingReads { + fn store( + &mut self, + key: u64, + value: Vec, + ) -> StorageResult<()> { + self.writes + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.inner.insert(key, value); + Ok(()) + } + fn clear(&mut self) -> StorageResult<()> { + self.inner.clear(); + Ok(()) + } + fn get(&self, _key: &u64) -> StorageResult>> { + Err(StorageError::SqliteNotEnabled) + } + fn take( + &mut self, + key: &u64, + ) -> StorageResult>> { + Ok(self.inner.remove(key)) + } + fn keys(&self) -> StorageResult> { + Ok(self.inner.keys().copied().collect()) + } + fn into_iter( + &self, + ) -> StorageResult< + Box)> + '_>, + > { + Ok(Box::new( + self.inner.iter().map(|(k, v)| (*k, v.clone())), + )) + } + fn count(&self) -> StorageResult { + Ok(self.inner.len() as u64) + } + fn clone_box(&self) -> Box>> { + Box::new(self.clone()) + } + } + + const SLOT: u64 = 500; + let (mut svm, _account_pubkey, instance) = scheduled_persist_fixture(true); + + // Something else is already queued for the next slot. + let mut bystander = instance.clone(); + bystander.id = "someone-elses-override".to_string(); + + let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + svm.scheduled_overrides = Box::new(FailingReads { + inner: HashMap::from([(SLOT + 1, vec![bystander.clone()])]), + writes: writes.clone(), + }); + + svm.reschedule_override_for_next_slot(&instance, SLOT); + + assert_eq!( + writes.load(std::sync::atomic::Ordering::SeqCst), + 0, + "a failed read must not be followed by a write; the queue would be replaced by a single \ + entry and every other override for that slot lost" + ); + let survivors = svm + .scheduled_overrides + .take(&(SLOT + 1)) + .expect("take") + .expect("the bystander must still be queued"); + assert_eq!(survivors.len(), 1); + assert_eq!(survivors[0].id, "someone-elses-override"); + } + #[tokio::test] async fn test_non_persisted_override_is_not_rescheduled() { const SLOT: u64 = 500; From 717aed70680ee909cb94381b02dcaf9034be9cfb Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Wed, 26 Aug 2026 14:07:08 +0300 Subject: [PATCH 05/18] fix: keep bounded persist alive across clock jumps and cancellation --- crates/core/src/scenarios/README.md | 15 +- crates/core/src/surfnet/svm.rs | 242 +++++++++++++++++- .../kit/generated/OverrideInstance.ts | 5 +- crates/types/src/rpc_endpoints.json | 2 +- crates/types/src/scenarios.rs | 64 +++-- 5 files changed, 288 insertions(+), 40 deletions(-) diff --git a/crates/core/src/scenarios/README.md b/crates/core/src/scenarios/README.md index f392741a..d74b4e7a 100644 --- a/crates/core/src/scenarios/README.md +++ b/crates/core/src/scenarios/README.md @@ -35,13 +35,20 @@ padding included - because the account is re-encoded with Borsh. An out-of-range non-numeric segment on an array is a hard error, never a silent write elsewhere. By default an override applies to exactly one slot. `"persist"` controls how long it keeps -re-applying: `true` re-applies indefinitely and `{"slots": 10}` applies in ten slots in total, counting the -first. Prefer a -bounded window - an indefinite one outlives the scenario that created it. +re-applying: `true` re-applies indefinitely and `{"slots": 10}` applies in ten slots in total, +counting the first. Prefer a bounded window - an indefinite one outlives the scenario that created +it. `{"slots": 0}` is refused rather than granted as one application: an override always applies on +the slot it is scheduled for, so zero applications is not something it can honour. Registering a scenario replaces any override it has already queued, matched on id, account and template together. That is how an override is updated, and how a persisted one is stopped: register -it again with `"persist": false` and the queued copy becomes a one-shot, so it stops re-arming. +it again with `"persist": false`. A persisted override has usually already armed a copy one slot +ahead by then, so the cancelling registration also drops matching copies queued for later slots - +otherwise the armed one would keep re-arming out of a bucket the registration never touched. + +Persistence survives a clock jump. Each slot claims every override queued at or before it, not only +an exact match, so a `timeTravel` forward past an armed copy carries it to the slot actually reached +instead of stranding it in a key nothing materializes again. Re-applying is what you want when something else writes the account in between - a transaction, or another override fetching it fresh. Persist inputs nothing in the scenario writes (an oracle price, a disabled switch, a risk parameter), never state the transactions under test diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index aaa93c32..8e6e0254 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -2702,11 +2702,32 @@ impl SurfnetSvm { remote_ctx: &Option<(SurfnetRemoteClient, CommitmentConfig)>, target_slot: Slot, ) -> SurfpoolResult<()> { - // Remove and get overrides for this slot - let Some(overrides) = self.scheduled_overrides.take(&target_slot)? else { - // No overrides for this slot + let mut due: Vec = self + .scheduled_overrides + .keys()? + .into_iter() + .filter(|slot| *slot <= target_slot) + .collect(); + due.sort_unstable(); + + let mut overrides = Vec::new(); + for slot in due { + if let Some(queued) = self.scheduled_overrides.take(&slot)? { + if slot != target_slot && !queued.is_empty() { + debug!( + "Materializing {} override(s) left behind at slot {} at slot {}", + queued.len(), + slot, + target_slot + ); + } + overrides.extend(queued); + } + } + + if overrides.is_empty() { return Ok(()); - }; + } debug!( "Materializing {} override(s) for slot {}", @@ -2879,7 +2900,9 @@ impl SurfnetSvm { account_pubkey ); settled_this_slot.insert(account_pubkey); - if override_instance.persist.is_enabled() && override_instance.fetch_before_use { + if override_instance.persist.is_enabled() + && override_instance.fetch_before_use + { let mut requeued = override_instance.clone(); requeued.fetch_before_use = false; self.reschedule_override_for_next_slot(&requeued, target_slot); @@ -2986,7 +3009,8 @@ impl SurfnetSvm { // The account is forked now. Re-fetching it every slot would cost one RPC // per slot and overwrite whatever local transactions wrote to the fields // this override leaves alone, so later slots re-pin without fetching. - if override_instance.persist.is_enabled() && override_instance.fetch_before_use { + if override_instance.persist.is_enabled() && override_instance.fetch_before_use + { let mut requeued = override_instance.clone(); requeued.fetch_before_use = false; self.reschedule_override_for_next_slot(&requeued, target_slot); @@ -4298,9 +4322,28 @@ impl SurfnetSvm { // Schedule overrides by adding base slot to their scenario-relative slots for override_instance in scenario.overrides { + if matches!( + override_instance.persist, + surfpool_types::Persist::Slots { slots: 0 } + ) { + return Err(SurfpoolError::internal(format!( + "Override {} sets persist.slots to 0, which asks for zero applications. An \ + override always applies on the slot it is scheduled for, so use \ + persist: false for a single application, or slots >= 1 for a window.", + override_instance.id + ))); + } + let scenario_relative_slot = override_instance.scenario_relative_slot; let absolute_slot = base_slot + scenario_relative_slot; + // Re-registering as a one-shot is how a persisted override is cancelled, but by then it + // has already armed a copy one slot ahead - in a bucket this registration never + // touches. Without this it keeps re-arming forever. + if !override_instance.persist.is_enabled() { + self.cancel_queued_copies_after(&override_instance, absolute_slot)?; + } + debug!( "Scheduling override at absolute slot {} (base {} + relative {})", absolute_slot, base_slot, scenario_relative_slot @@ -4332,6 +4375,46 @@ impl SurfnetSvm { Ok(()) } + + /// Drops copies of `instance` queued for any slot after `from_slot`. + /// + /// Matching is the same (id, account, template_id) triple the scheduler dedupes on, so this + /// removes the override's own re-armed copies and nothing else. + fn cancel_queued_copies_after( + &mut self, + instance: &OverrideInstance, + from_slot: Slot, + ) -> SurfpoolResult<()> { + let later: Vec = self + .scheduled_overrides + .keys()? + .into_iter() + .filter(|slot| *slot > from_slot) + .collect(); + + for slot in later { + let Some(mut queued) = self.scheduled_overrides.get(&slot)? else { + continue; + }; + let before = queued.len(); + queued.retain(|other| { + !(other.id == instance.id + && other.account == instance.account + && other.template_id == instance.template_id) + }); + if queued.len() != before { + debug!( + "Cancelled {} queued copy(ies) of override {} at slot {}", + before - queued.len(), + instance.id, + slot + ); + self.scheduled_overrides.store(slot, queued)?; + } + } + + Ok(()) + } } #[cfg(test)] @@ -7279,7 +7362,11 @@ mod tests { 1_234, "the first override must survive the second override's fetch" ); - assert_eq!(read(ALLOWED_OFFSET), 5_678, "the second override must apply"); + assert_eq!( + read(ALLOWED_OFFSET), + 5_678, + "the second override must apply" + ); } /// Two persistent overrides that share a caller-supplied id but target different accounts must both survive re-arming. @@ -7442,6 +7529,138 @@ mod tests { ); } + /// A forward clock jump must not strand an armed copy. The override arms `slot + 1`; if the + /// clock then jumps past it, nothing would ever materialize that key again and the override + /// would silently stop persisting. + #[tokio::test] + async fn test_persist_survives_a_forward_time_jump() { + const SLOT: u64 = 500; + const JUMP_TO: u64 = 900; + + let (mut svm, _pk, instance) = scheduled_persist_fixture(true); + svm.scheduled_overrides + .store(SLOT, vec![instance]) + .expect("schedule"); + + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect("materialize at the original slot"); + assert_eq!( + svm.scheduled_overrides + .get(&(SLOT + 1)) + .expect("read") + .unwrap_or_default() + .len(), + 1, + "the override must arm the following slot" + ); + + // Time travel forward, then produce a block at the new slot. + svm.materialize_overrides_for_slot(&None, JUMP_TO) + .await + .expect("materialize after the jump"); + + assert!( + svm.scheduled_overrides + .get(&(SLOT + 1)) + .expect("read") + .unwrap_or_default() + .is_empty(), + "the stranded copy must be claimed, not left to fire if the clock travels back" + ); + assert_eq!( + svm.scheduled_overrides + .get(&(JUMP_TO + 1)) + .expect("read") + .unwrap_or_default() + .len(), + 1, + "persistence must continue from the slot actually reached" + ); + } + + #[tokio::test] + async fn test_cancelling_after_re_arming_stops_the_override() { + const SLOT: u64 = 500; + + let (mut svm, _pk, mut instance) = scheduled_persist_fixture(true); + let scenario = |instance: surfpool_types::OverrideInstance| surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![instance], + }; + + svm.register_scenario(scenario(instance.clone()), Some(SLOT)) + .expect("register persisted"); + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect("materialize"); + assert_eq!( + svm.scheduled_overrides + .get(&(SLOT + 1)) + .expect("read") + .unwrap_or_default() + .len(), + 1, + "it must have armed the next slot before we cancel" + ); + + // Cancel it now, from the slot the caller is on. + instance.persist = surfpool_types::Persist::Always(false); + svm.register_scenario(scenario(instance), Some(SLOT)) + .expect("register the cancelling one-shot"); + + assert!( + svm.scheduled_overrides + .get(&(SLOT + 1)) + .expect("read") + .unwrap_or_default() + .is_empty(), + "cancelling must remove the copy already armed for a later slot" + ); + + svm.materialize_overrides_for_slot(&None, SLOT + 1) + .await + .expect("materialize the next slot"); + assert!( + svm.scheduled_overrides + .get(&(SLOT + 2)) + .expect("read") + .unwrap_or_default() + .is_empty(), + "and it must not re-arm from there" + ); + } + + /// `slots: 0` asks for zero applications, which no override can honour - it always applies on + /// the slot it is scheduled for. + #[tokio::test] + async fn test_persist_slots_zero_is_rejected() { + const SLOT: u64 = 500; + + let (mut svm, _pk, mut instance) = scheduled_persist_fixture(false); + instance.persist = surfpool_types::Persist::Slots { slots: 0 }; + + let err = svm + .register_scenario( + surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![instance], + }, + Some(SLOT), + ) + .expect_err("slots: 0 must be refused"); + assert!( + err.to_string().contains("zero applications"), + "the error must say why: {err}" + ); + } + /// A persisted override is stopped by re-registering it as a one-shot. #[tokio::test] async fn test_re_registering_with_persist_false_cancels_a_persisted_override() { @@ -7503,7 +7722,10 @@ mod tests { self.inner.clear(); Ok(()) } - fn get(&self, _key: &u64) -> StorageResult>> { + fn get( + &self, + _key: &u64, + ) -> StorageResult>> { Err(StorageError::SqliteNotEnabled) } fn take( @@ -7520,9 +7742,7 @@ mod tests { ) -> StorageResult< Box)> + '_>, > { - Ok(Box::new( - self.inner.iter().map(|(k, v)| (*k, v.clone())), - )) + Ok(Box::new(self.inner.iter().map(|(k, v)| (*k, v.clone())))) } fn count(&self) -> StorageResult { Ok(self.inner.len() as u64) diff --git a/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts b/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts index 80a87f24..c2f5f62c 100644 --- a/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts +++ b/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts @@ -36,9 +36,10 @@ enabled: boolean, */ fetchBeforeUse?: boolean, /** - * Whether to re-apply this override on every subsequent slot, rather than only once + * How long to keep re-applying this override: `false` applies it once, `true` re-applies it + * on every following slot, and `{ slots: N }` applies it N times in total, counting the first */ -persist?: boolean, +persist?: boolean | { slots: number }, /** * Account address to override - use pubkey for known addresses or pda for derived addresses */ diff --git a/crates/types/src/rpc_endpoints.json b/crates/types/src/rpc_endpoints.json index f45cc118..aa917870 100644 --- a/crates/types/src/rpc_endpoints.json +++ b/crates/types/src/rpc_endpoints.json @@ -861,7 +861,7 @@ "label": "Option (An optional label for this override instance)", "enabled": "bool (Indicates whether this override instance is enabled)", "fetchBeforeUse": "bool (Indicates whether to fetch the latest on-chain account data before applying overrides)", - "persist": "Optional, defaults to false. Controls how long the override keeps re-applying itself. Accepts a boolean or a bounded window: false applies once; true re-applies every following slot indefinitely; {\"slots\": 10} applies in 10 slots in total, counting the first. To stop a persisted override, register the same override again (same id, account and template) with persist false - it replaces the queued copy with a one-shot. Re-applying is needed when something else writes the account in between, but use it only for values no transaction writes - an oracle price, a disabled switch, a risk parameter - never for state the transactions under test mutate: re-applying reverts their writes at the start of the next slot, so a pool would refill itself after every swap. Prefer a bounded window over true, so a persisted override cannot outlive the scenario that created it.", + "persist": "Optional, defaults to false. Controls how long the override keeps re-applying itself. Accepts a boolean or a bounded window: false applies once; true re-applies every following slot indefinitely; {\"slots\": 10} applies in 10 slots in total, counting the first, and {\"slots\": 0} is refused because an override always applies on the slot it is scheduled for. To stop a persisted override, register the same override again (same id, account and template) with persist false - that drops both the queued copy and any copy already armed for a later slot. Re-applying is needed when something else writes the account in between, but use it only for values no transaction writes - an oracle price, a disabled switch, a risk parameter - never for state the transactions under test mutate: re-applying reverts their writes at the start of the next slot, so a pool would refill itself after every swap. Prefer a bounded window over true, so a persisted override cannot outlive the scenario that created it.", "account": "AccountAddress (The account this override targets, as {\"pubkey\": \"\"} or {\"pda\": {\"programId\": \"\", \"seeds\": [ ... ]}})" } } diff --git a/crates/types/src/scenarios.rs b/crates/types/src/scenarios.rs index 1c807f53..af4ed4e8 100644 --- a/crates/types/src/scenarios.rs +++ b/crates/types/src/scenarios.rs @@ -503,9 +503,7 @@ impl OverrideTemplate { #[cfg_attr(feature = "ts-bindings", derive(ts_rs::TS))] pub enum Persist { Always(bool), - Slots { - slots: Slot, - }, + Slots { slots: Slot }, } impl Default for Persist { @@ -570,12 +568,16 @@ pub struct OverrideInstance { #[serde(default)] #[cfg_attr(feature = "ts-bindings", ts(as = "Option", optional))] pub fetch_before_use: bool, - /// Whether to re-apply this override on every subsequent slot, rather than only once + /// How long to keep re-applying this override: `false` applies it once, `true` re-applies it + /// on every following slot, and `{ slots: N }` applies it N times in total, counting the first #[schemars( - description = "If true, re-applies this override every following slot. Use only for values no transaction writes: it reverts transaction writes to the same fields." + description = "false applies once; true re-applies every following slot; {\"slots\": N} re-applies until N applications have happened, counting the first. Use only for values no transaction writes: re-applying reverts transaction writes to the same fields." )] #[serde(default)] - #[cfg_attr(feature = "ts-bindings", ts(type = "boolean | { slots: number }", optional))] + #[cfg_attr( + feature = "ts-bindings", + ts(type = "boolean | { slots: number }", optional) + )] pub persist: Persist, /// Account address to override - use pubkey for known addresses or pda for derived addresses #[schemars( @@ -617,7 +619,8 @@ impl OverrideInstance { pub fn with_persist_for_slots(mut self, slots: Slot) -> Self { self.persist = Persist::Slots { slots }; self - }} + } +} /// A scenario containing a timeline of overrides #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] @@ -1065,7 +1068,9 @@ pub enum RawEncoding { /// A base58 pubkey, written as 32 bytes. Bytes32, /// The slot the override materializes at, plus `lead` (may be negative). - Slot { lead: i64 }, + Slot { + lead: i64, + }, } impl RawEncoding { @@ -1093,11 +1098,7 @@ impl RawEncoding { } /// The little-endian bytes for `value`. `target_slot` is only read by [`RawEncoding::Slot`]. - pub fn encode( - &self, - value: &serde_json::Value, - target_slot: Slot, - ) -> Result, String> { + pub fn encode(&self, value: &serde_json::Value, target_slot: Slot) -> Result, String> { // Read the digits as text so nothing passes through f64, which cannot hold a u128 // exactly. A decimal string is the only way to express values above u64::MAX in JSON. let digits = |what: &str| -> Result { @@ -1110,7 +1111,9 @@ impl RawEncoding { } serde_json::Value::Number(n) => Ok(n.to_string()), serde_json::Value::String(s) => Ok(s.trim().to_string()), - other => Err(format!("expected a number or decimal string for {what}, found {other}")), + other => Err(format!( + "expected a number or decimal string for {what}, found {other}" + )), } }; macro_rules! int { @@ -1226,9 +1229,10 @@ impl RawLayout { let (count, stride) = encoding.placements(); for i in 0..count { let at = offset - .checked_add(i.checked_mul(stride).ok_or_else(|| { - format!("stride overflow for '{name}'") - })?) + .checked_add( + i.checked_mul(stride) + .ok_or_else(|| format!("stride overflow for '{name}'"))?, + ) .ok_or_else(|| format!("offset overflow for '{name}'"))?; let end = at .checked_add(bytes.len()) @@ -1396,7 +1400,11 @@ impl YamlOverrideTemplateCollection { protocol: self.protocol.clone(), idl: idl.clone(), address: entry.address.into(), - properties: describe_properties_from_idl(entry.properties, idl.as_ref(), &account_type), + properties: describe_properties_from_idl( + entry.properties, + idl.as_ref(), + &account_type, + ), account_type, constants: constants.clone(), tags: self.tags.clone(), @@ -1576,11 +1584,15 @@ mod tests { let bytes = RawEncoding::I64.encode(&json!(-25599i64 << 32), 0).unwrap(); assert_eq!(i64::from_le_bytes(bytes.try_into().unwrap()) >> 32, -25599); - let bytes = RawEncoding::Slot { lead: -1 }.encode(&json!(0), 500).unwrap(); + let bytes = RawEncoding::Slot { lead: -1 } + .encode(&json!(0), 500) + .unwrap(); assert_eq!(u64::from_le_bytes(bytes.try_into().unwrap()), 499); // A lead that would go below zero clamps rather than wrapping. - let bytes = RawEncoding::Slot { lead: -10 }.encode(&json!(0), 3).unwrap(); + let bytes = RawEncoding::Slot { lead: -10 } + .encode(&json!(0), 3) + .unwrap(); assert_eq!(u64::from_le_bytes(bytes.try_into().unwrap()), 0); } @@ -1648,7 +1660,12 @@ mod tests { property.encoding = Some(RawEncoding::U64); let err = layout - .materialize(&[0u8; 16], &[property], &HashMap::from([("tail".to_string(), json!(1))]), 0) + .materialize( + &[0u8; 16], + &[property], + &HashMap::from([("tail".to_string(), json!(1))]), + 0, + ) .expect_err("a field crossing the end must be refused"); assert!(err.contains("exceeds"), "unexpected error: {err}"); } @@ -1688,7 +1705,10 @@ mod tests { let written: Vec = (0..3).flat_map(|i| (4 + i * 16)..(8 + i * 16)).collect(); for (i, b) in out.iter().enumerate() { if !written.contains(&i) { - assert_eq!(*b, 0, "byte {i} lies between strided slots and must not change"); + assert_eq!( + *b, 0, + "byte {i} lies between strided slots and must not change" + ); } } } From 870d5acbc5cce4cc4f63d2a1cef56e83aa0e2d0b Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Wed, 26 Aug 2026 15:22:14 +0300 Subject: [PATCH 06/18] fix: harden override scheduling against duplicate application --- crates/core/src/surfnet/svm.rs | 203 ++++++++++++++++++++++++++++++--- 1 file changed, 188 insertions(+), 15 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index 32aece75..82e3c9ce 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -2822,6 +2822,24 @@ impl SurfnetSvm { return Ok(()); } + let mut deduped: Vec = Vec::with_capacity(overrides.len()); + for instance in overrides.into_iter().rev() { + if deduped.iter().any(|kept| { + kept.id == instance.id + && kept.account == instance.account + && kept.template_id == instance.template_id + }) { + debug!( + "Dropping a superseded copy of override {} claimed from an earlier slot", + instance.id + ); + continue; + } + deduped.push(instance); + } + deduped.reverse(); + let overrides = deduped; + debug!( "Materializing {} override(s) for slot {}", overrides.len(), @@ -4521,12 +4539,7 @@ impl SurfnetSvm { )) })?; - // Re-registering as a one-shot is how a persisted override is cancelled, but by then it - // has already armed a copy one slot ahead - in a bucket this registration never - // touches. Without this it keeps re-arming forever. - if !override_instance.persist.is_enabled() { - self.cancel_queued_copies_after(&override_instance, absolute_slot)?; - } + self.remove_queued_copies_elsewhere(&override_instance, absolute_slot)?; debug!( "Scheduling override at absolute slot {} (base {} + relative {})", @@ -4558,23 +4571,25 @@ impl SurfnetSvm { Ok(()) } - /// Drops copies of `instance` queued for any slot after `from_slot`. + /// Drops copies of `instance` queued for any slot other than `keep_slot`. /// /// Matching is the same (id, account, template_id) triple the scheduler dedupes on, so this - /// removes the override's own re-armed copies and nothing else. - fn cancel_queued_copies_after( + /// only removes the override's own copies - in practice the one it armed for the next slot. + /// Sweeping every other bucket rather than only later ones matters because the armed copy sits + /// *before* a cancellation scheduled at a positive relative slot. + fn remove_queued_copies_elsewhere( &mut self, instance: &OverrideInstance, - from_slot: Slot, + keep_slot: Slot, ) -> SurfpoolResult<()> { - let later: Vec = self + let others: Vec = self .scheduled_overrides .keys()? .into_iter() - .filter(|slot| *slot > from_slot) + .filter(|slot| *slot != keep_slot) .collect(); - for slot in later { + for slot in others { let Some(mut queued) = self.scheduled_overrides.get(&slot)? else { continue; }; @@ -4586,12 +4601,18 @@ impl SurfnetSvm { }); if queued.len() != before { debug!( - "Cancelled {} queued copy(ies) of override {} at slot {}", + "Removed {} superseded copy(ies) of override {} at slot {}", before - queued.len(), instance.id, slot ); - self.scheduled_overrides.store(slot, queued)?; + // Drop the key rather than leaving an empty vec behind: a slot that holds nothing + // still shows up in `keys()`, so every sweep would walk more dead entries. + if queued.is_empty() { + self.scheduled_overrides.take(&slot)?; + } else { + self.scheduled_overrides.store(slot, queued)?; + } } } @@ -8303,6 +8324,158 @@ mod tests { ); } + /// Cancelling at a FUTURE relative slot. The armed copy sits before the cancellation bucket, so + /// sweeping only later slots leaves it alive - and when it re-arms it lands on the cancellation + /// bucket and replaces the one-shot with itself, so the override never stops. + #[tokio::test] + async fn test_cancelling_at_a_future_slot_stops_the_override() { + const SLOT: u64 = 500; + const CANCEL_AT: u64 = 5; + + let (mut svm, _pk, mut instance) = scheduled_persist_fixture(true); + let scenario = |instance: surfpool_types::OverrideInstance| surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![instance], + }; + + svm.register_scenario(scenario(instance.clone()), Some(SLOT)) + .expect("register persisted"); + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect("materialize"); + + // Cancel, but scheduled a few slots out rather than for the current slot. + instance.persist = surfpool_types::Persist::Always(false); + instance.scenario_relative_slot = CANCEL_AT; + svm.register_scenario(scenario(instance), Some(SLOT)) + .expect("register the cancelling one-shot"); + + // Run past the cancellation slot. + for slot in (SLOT + 1)..=(SLOT + CANCEL_AT + 2) { + svm.materialize_overrides_for_slot(&None, slot) + .await + .expect("materialize"); + } + + let leftover: Vec = svm + .scheduled_overrides + .keys() + .expect("keys") + .into_iter() + .filter(|slot| { + !svm.scheduled_overrides + .get(slot) + .expect("read") + .unwrap_or_default() + .is_empty() + }) + .collect(); + assert!( + leftover.is_empty(), + "the override was cancelled, so nothing may still be queued; found {leftover:?}" + ); + } + + /// Claiming several overdue buckets at once must still apply an override only once. The final + /// state hides a duplicate - both copies write the same bytes and the second re-arm replaces + /// the first - so this counts the work instead: one application means one reschedule write. + #[tokio::test] + async fn test_overdue_buckets_apply_an_override_once() { + use crate::storage::{Storage, StorageResult}; + + #[derive(Clone)] + struct CountingStore { + inner: HashMap>, + writes: std::sync::Arc, + } + + impl Storage> for CountingStore { + fn store( + &mut self, + key: u64, + value: Vec, + ) -> StorageResult<()> { + self.writes + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.inner.insert(key, value); + Ok(()) + } + fn clear(&mut self) -> StorageResult<()> { + self.inner.clear(); + Ok(()) + } + fn get( + &self, + key: &u64, + ) -> StorageResult>> { + Ok(self.inner.get(key).cloned()) + } + fn take( + &mut self, + key: &u64, + ) -> StorageResult>> { + Ok(self.inner.remove(key)) + } + fn keys(&self) -> StorageResult> { + Ok(self.inner.keys().copied().collect()) + } + fn into_iter( + &self, + ) -> StorageResult< + Box)> + '_>, + > { + Ok(Box::new(self.inner.iter().map(|(k, v)| (*k, v.clone())))) + } + fn count(&self) -> StorageResult { + Ok(self.inner.len() as u64) + } + fn clone_box(&self) -> Box>> { + Box::new(self.clone()) + } + } + + const SLOT: u64 = 500; + + let (mut svm, _pk, base) = scheduled_persist_fixture(true); + + let mut earlier = base.clone(); + earlier.persist = surfpool_types::Persist::Slots { slots: 5 }; + let mut later = base.clone(); + later.persist = surfpool_types::Persist::Slots { slots: 2 }; + + let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + svm.scheduled_overrides = Box::new(CountingStore { + inner: HashMap::from([(SLOT, vec![earlier]), (SLOT + 1, vec![later])]), + writes: writes.clone(), + }); + + svm.materialize_overrides_for_slot(&None, SLOT + 1) + .await + .expect("materialize both overdue buckets"); + + assert_eq!( + writes.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the override must be handled once, so it re-arms once; a duplicate would reschedule \ + twice and could fetch or write on behalf of a copy the operator has superseded" + ); + + let armed = svm + .scheduled_overrides + .get(&(SLOT + 2)) + .expect("read") + .unwrap_or_default(); + assert_eq!(armed.len(), 1, "one entry per override id"); + assert_eq!( + armed[0].persist, + surfpool_types::Persist::Slots { slots: 1 }, + "the copy scheduled latest must win, so its window is the one that advances" + ); + } + /// `slots: 0` asks for zero applications, which no override can honour - it always applies on /// the slot it is scheduled for. #[tokio::test] From c1fe344d8e1d64efb217355a75121791b4de14dd Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Wed, 26 Aug 2026 15:35:06 +0300 Subject: [PATCH 07/18] fix: make scenario registration atomic and timeline-safe --- crates/core/src/surfnet/svm.rs | 109 +++++++++++++++++- .../kit/generated/OverrideInstance.ts | 2 +- crates/types/src/scenarios.rs | 2 +- 3 files changed, 105 insertions(+), 8 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index 82e3c9ce..97ffe315 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -4516,7 +4516,11 @@ impl SurfnetSvm { base_slot ); - // Schedule overrides by adding base slot to their scenario-relative slots + // Validate and resolve every override before touching storage, so a scenario that is + // rejected leaves nothing behind: the RPC returns an error and the caller can assume none + // of it was scheduled. + let mut planned: Vec<(Slot, surfpool_types::OverrideInstance)> = + Vec::with_capacity(scenario.overrides.len()); for override_instance in scenario.overrides { if matches!( override_instance.persist, @@ -4539,7 +4543,19 @@ impl SurfnetSvm { )) })?; - self.remove_queued_copies_elsewhere(&override_instance, absolute_slot)?; + planned.push((absolute_slot, override_instance)); + } + + // Slots this scenario is about to write. The sweep below must leave them alone: a scenario + // may deliberately schedule one override at several slots to walk a value over a timeline, + // and those entries are not the re-armed copies the sweep is there to clear. + let scenario_slots: Vec = planned.iter().map(|(slot, _)| *slot).collect(); + + // Schedule overrides by adding base slot to their scenario-relative slots + for (absolute_slot, override_instance) in planned { + let scenario_relative_slot = override_instance.scenario_relative_slot; + + self.remove_queued_copies_elsewhere(&override_instance, &scenario_slots)?; debug!( "Scheduling override at absolute slot {} (base {} + relative {})", @@ -4571,22 +4587,24 @@ impl SurfnetSvm { Ok(()) } - /// Drops copies of `instance` queued for any slot other than `keep_slot`. + /// Drops copies of `instance` queued for any slot the registering scenario does not itself + /// write. /// /// Matching is the same (id, account, template_id) triple the scheduler dedupes on, so this /// only removes the override's own copies - in practice the one it armed for the next slot. /// Sweeping every other bucket rather than only later ones matters because the armed copy sits - /// *before* a cancellation scheduled at a positive relative slot. + /// *before* a cancellation scheduled at a positive relative slot. `keep_slots` holds every slot + /// the scenario schedules, so a deliberate timeline that reuses one id across slots survives. fn remove_queued_copies_elsewhere( &mut self, instance: &OverrideInstance, - keep_slot: Slot, + keep_slots: &[Slot], ) -> SurfpoolResult<()> { let others: Vec = self .scheduled_overrides .keys()? .into_iter() - .filter(|slot| *slot != keep_slot) + .filter(|slot| !keep_slots.contains(slot)) .collect(); for slot in others { @@ -8476,6 +8494,85 @@ mod tests { ); } + /// A scenario may walk one override across a timeline, reusing the same id at several slots. + /// The sweep that clears re-armed copies must not eat those deliberate entries. + #[tokio::test] + async fn test_a_timeline_sharing_one_id_survives_registration() { + const SLOT: u64 = 500; + const LATER: u64 = 3; + + let (mut svm, _pk, first) = scheduled_persist_fixture(false); + let mut second = first.clone(); + second.scenario_relative_slot = LATER; + + svm.register_scenario( + surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![first, second], + }, + Some(SLOT), + ) + .expect("register the timeline"); + + for slot in [SLOT, SLOT + LATER] { + assert_eq!( + svm.scheduled_overrides + .get(&slot) + .expect("read") + .unwrap_or_default() + .len(), + 1, + "the step at slot {slot} must still be scheduled" + ); + } + } + + /// A rejected scenario must schedule nothing. Validating while writing left the overrides ahead + /// of the bad one queued, so the caller saw an error and got half a scenario. + #[tokio::test] + async fn test_a_rejected_scenario_schedules_nothing() { + const SLOT: u64 = 500; + + let (mut svm, _pk, good) = scheduled_persist_fixture(false); + let mut bad = good.clone(); + bad.id = "the-invalid-one".to_string(); + bad.scenario_relative_slot = 1; + bad.persist = surfpool_types::Persist::Slots { slots: 0 }; + + svm.register_scenario( + surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![good, bad], + }, + Some(SLOT), + ) + .expect_err("slots: 0 must reject the whole scenario"); + + let scheduled: Vec = svm + .scheduled_overrides + .keys() + .expect("keys") + .into_iter() + .filter(|slot| { + !svm.scheduled_overrides + .get(slot) + .expect("read") + .unwrap_or_default() + .is_empty() + }) + .collect(); + assert!( + scheduled.is_empty(), + "the valid override must not have been scheduled either; found {scheduled:?}" + ); + } + /// `slots: 0` asks for zero applications, which no override can honour - it always applies on /// the slot it is scheduled for. #[tokio::test] diff --git a/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts b/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts index c2f5f62c..8f47f1aa 100644 --- a/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts +++ b/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts @@ -39,7 +39,7 @@ fetchBeforeUse?: boolean, * How long to keep re-applying this override: `false` applies it once, `true` re-applies it * on every following slot, and `{ slots: N }` applies it N times in total, counting the first */ -persist?: boolean | { slots: number }, +persist?: boolean | { slots: number | bigint }, /** * Account address to override - use pubkey for known addresses or pda for derived addresses */ diff --git a/crates/types/src/scenarios.rs b/crates/types/src/scenarios.rs index 1d32d944..4d2dd4f2 100644 --- a/crates/types/src/scenarios.rs +++ b/crates/types/src/scenarios.rs @@ -576,7 +576,7 @@ pub struct OverrideInstance { #[serde(default)] #[cfg_attr( feature = "ts-bindings", - ts(type = "boolean | { slots: number }", optional) + ts(type = "boolean | { slots: number | bigint }", optional) )] pub persist: Persist, /// Account address to override - use pubkey for known addresses or pda for derived addresses From 351b736f3bfe57571b32258915ebfde7d887481b Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Wed, 26 Aug 2026 16:06:04 +0300 Subject: [PATCH 08/18] fix: tell re-armed copies apart from scheduled timeline steps --- crates/core/src/surfnet/svm.rs | 183 +++++++++++++++++++++++++-------- crates/types/src/scenarios.rs | 7 ++ 2 files changed, 149 insertions(+), 41 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index 97ffe315..92f831c2 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -2824,13 +2824,16 @@ impl SurfnetSvm { let mut deduped: Vec = Vec::with_capacity(overrides.len()); for instance in overrides.into_iter().rev() { - if deduped.iter().any(|kept| { - kept.id == instance.id - && kept.account == instance.account - && kept.template_id == instance.template_id - }) { + if instance.re_armed + && deduped.iter().any(|kept| { + kept.re_armed + && kept.id == instance.id + && kept.account == instance.account + && kept.template_id == instance.template_id + }) + { debug!( - "Dropping a superseded copy of override {} claimed from an earlier slot", + "Dropping a superseded continuation of override {} claimed from an earlier slot", instance.id ); continue; @@ -3211,6 +3214,7 @@ impl SurfnetSvm { }; let mut instance = instance.clone(); instance.persist = next_persist; + instance.re_armed = true; let instance = &instance; let mut next = self @@ -4546,16 +4550,11 @@ impl SurfnetSvm { planned.push((absolute_slot, override_instance)); } - // Slots this scenario is about to write. The sweep below must leave them alone: a scenario - // may deliberately schedule one override at several slots to walk a value over a timeline, - // and those entries are not the re-armed copies the sweep is there to clear. - let scenario_slots: Vec = planned.iter().map(|(slot, _)| *slot).collect(); - // Schedule overrides by adding base slot to their scenario-relative slots for (absolute_slot, override_instance) in planned { let scenario_relative_slot = override_instance.scenario_relative_slot; - self.remove_queued_copies_elsewhere(&override_instance, &scenario_slots)?; + self.remove_queued_copies_elsewhere(&override_instance)?; debug!( "Scheduling override at absolute slot {} (base {} + relative {})", @@ -4587,33 +4586,27 @@ impl SurfnetSvm { Ok(()) } - /// Drops copies of `instance` queued for any slot the registering scenario does not itself - /// write. + /// Drops the copies `instance` queued for itself to keep persisting, wherever they sit. /// - /// Matching is the same (id, account, template_id) triple the scheduler dedupes on, so this - /// only removes the override's own copies - in practice the one it armed for the next slot. - /// Sweeping every other bucket rather than only later ones matters because the armed copy sits - /// *before* a cancellation scheduled at a positive relative slot. `keep_slots` holds every slot - /// the scenario schedules, so a deliberate timeline that reuses one id across slots survives. + /// Only re-armed copies are removed. Entries an operator scheduled are left alone at every + /// slot, because a scenario may place one id at several slots to walk a value over a timeline - + /// possibly across separate registrations - and those are not this sweep's business. Sweeping + /// every slot rather than only later ones matters because the armed copy sits *before* a + /// cancellation scheduled at a positive relative slot. fn remove_queued_copies_elsewhere( &mut self, instance: &OverrideInstance, - keep_slots: &[Slot], ) -> SurfpoolResult<()> { - let others: Vec = self - .scheduled_overrides - .keys()? - .into_iter() - .filter(|slot| !keep_slots.contains(slot)) - .collect(); + let slots: Vec = self.scheduled_overrides.keys()?; - for slot in others { + for slot in slots { let Some(mut queued) = self.scheduled_overrides.get(&slot)? else { continue; }; let before = queued.len(); queued.retain(|other| { - !(other.id == instance.id + !(other.re_armed + && other.id == instance.id && other.account == instance.account && other.template_id == instance.template_id) }); @@ -8397,6 +8390,58 @@ mod tests { ); } + /// Pins the semantics of a future-dated one-shot, which are deliberate rather than incidental: + /// registering is declarative about the timeline, so it says "this override applies at slot N, + /// once" - it does not mean "keep persisting until N". The intervening slots are therefore + /// skipped. Use `persist: { slots: N }` to keep re-applying for a bounded window instead. + #[tokio::test] + async fn test_a_future_one_shot_skips_the_intervening_slots() { + const SLOT: u64 = 500; + const CANCEL_AT: u64 = 3; + + let (mut svm, _pk, mut instance) = scheduled_persist_fixture(true); + let scenario = |o: surfpool_types::OverrideInstance| surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![o], + }; + + svm.register_scenario(scenario(instance.clone()), Some(SLOT)) + .expect("register persisted"); + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect("materialize"); + + instance.persist = surfpool_types::Persist::Always(false); + instance.scenario_relative_slot = CANCEL_AT; + svm.register_scenario(scenario(instance), Some(SLOT)) + .expect("register the future one-shot"); + + // Nothing between now and the one-shot: the armed continuation is gone. + for slot in (SLOT + 1)..(SLOT + CANCEL_AT) { + assert!( + svm.scheduled_overrides + .get(&slot) + .expect("read") + .unwrap_or_default() + .is_empty(), + "slot {slot} must be empty; a future one-shot replaces the timeline rather than \ + scheduling a stop" + ); + } + assert_eq!( + svm.scheduled_overrides + .get(&(SLOT + CANCEL_AT)) + .expect("read") + .unwrap_or_default() + .len(), + 1, + "and the one-shot itself is queued for its own slot" + ); + } + /// Claiming several overdue buckets at once must still apply an override only once. The final /// state hides a duplicate - both copies write the same bytes and the second re-arm replaces /// the first - so this counts the work instead: one application means one reschedule write. @@ -8459,10 +8504,13 @@ mod tests { let (mut svm, _pk, base) = scheduled_persist_fixture(true); + // Two continuations of the same override, as a stranded copy plus a newer one would be. let mut earlier = base.clone(); earlier.persist = surfpool_types::Persist::Slots { slots: 5 }; + earlier.re_armed = true; let mut later = base.clone(); later.persist = surfpool_types::Persist::Slots { slots: 2 }; + later.re_armed = true; let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); svm.scheduled_overrides = Box::new(CountingStore { @@ -8477,8 +8525,8 @@ mod tests { assert_eq!( writes.load(std::sync::atomic::Ordering::SeqCst), 1, - "the override must be handled once, so it re-arms once; a duplicate would reschedule \ - twice and could fetch or write on behalf of a copy the operator has superseded" + "a superseded continuation must not be handled again; it would reschedule twice and \ + could fetch or write on behalf of state the newer copy already carries forward" ); let armed = svm @@ -8504,18 +8552,19 @@ mod tests { let (mut svm, _pk, first) = scheduled_persist_fixture(false); let mut second = first.clone(); second.scenario_relative_slot = LATER; + let scenario = |o: surfpool_types::OverrideInstance| surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![o], + }; - svm.register_scenario( - surfpool_types::Scenario { - id: "s-1".to_string(), - name: "s".to_string(), - description: String::new(), - tags: vec![], - overrides: vec![first, second], - }, - Some(SLOT), - ) - .expect("register the timeline"); + // Two separate calls, so nothing but the re-armed marker distinguishes intent. + svm.register_scenario(scenario(first), Some(SLOT)) + .expect("register the first step"); + svm.register_scenario(scenario(second), Some(SLOT)) + .expect("register the later step"); for slot in [SLOT, SLOT + LATER] { assert_eq!( @@ -8573,6 +8622,58 @@ mod tests { ); } + /// A timeline whose steps land in several overdue buckets must apply every step, in slot order. + /// Collapsing them to the latest loses the fields the earlier steps wrote. + #[tokio::test] + async fn test_overdue_timeline_steps_all_apply_in_order() { + const SLOT: u64 = 500; + const LATER: u64 = 3; + const JUMP_TO: u64 = 510; + const ALLOWED_OFFSET: usize = UNHEALTHY_OFFSET - 16; + + let (mut svm, account_pubkey, first) = scheduled_persist_fixture(false); + + // Step one writes one field, step two writes a different one, same identity. + let mut second = first.clone(); + second.scenario_relative_slot = LATER; + second.values = HashMap::from([( + "allowed_borrow_value_sf".to_string(), + serde_json::json!(5_678u64), + )]); + + let scenario = |o: surfpool_types::OverrideInstance| surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![o], + }; + svm.register_scenario(scenario(first), Some(SLOT)) + .expect("register step one"); + svm.register_scenario(scenario(second), Some(SLOT)) + .expect("register step two"); + + // Jump past both, so a single materialization claims each bucket. + svm.materialize_overrides_for_slot(&None, JUMP_TO) + .await + .expect("materialize"); + + let account = svm + .inner + .get_account(&account_pubkey) + .expect("get_account") + .expect("account present"); + let read = |off: usize| { + u128::from_le_bytes(account.data[off..off + 16].try_into().expect("16 bytes")) + }; + assert_eq!( + read(UNHEALTHY_OFFSET), + 1_234, + "the earlier step must still have been applied" + ); + assert_eq!(read(ALLOWED_OFFSET), 5_678, "and so must the later one"); + } + /// `slots: 0` asks for zero applications, which no override can honour - it always applies on /// the slot it is scheduled for. #[tokio::test] diff --git a/crates/types/src/scenarios.rs b/crates/types/src/scenarios.rs index 4d2dd4f2..967339fc 100644 --- a/crates/types/src/scenarios.rs +++ b/crates/types/src/scenarios.rs @@ -584,6 +584,12 @@ pub struct OverrideInstance { description = "Account address: either {\"pubkey\": \"base58_address\"} or {\"pda\": {\"programId\": \"...\", \"seeds\": [...]}}" )] pub account: AccountAddress, + /// Set by the scheduler, not by callers: marks a copy this override queued for itself to + /// continue persisting. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + #[schemars(skip)] + #[cfg_attr(feature = "ts-bindings", ts(skip))] + pub re_armed: bool, } impl OverrideInstance { @@ -598,6 +604,7 @@ impl OverrideInstance { fetch_before_use: false, persist: Persist::default(), account, + re_armed: false, } } From d93e409e5f42247e794a92eb121d99d75bb13293 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Wed, 26 Aug 2026 16:17:24 +0300 Subject: [PATCH 09/18] fix: keep re-arming and caller input off scheduled entries --- crates/core/src/surfnet/svm.rs | 116 +++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 5 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index 92f831c2..488111d5 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -3222,14 +3222,30 @@ impl SurfnetSvm { .get(&next_slot)? .unwrap_or_default(); - if let Some(existing) = next.iter_mut().find(|queued| { + let same_override = |queued: &OverrideInstance| { queued.id == instance.id && queued.account == instance.account && queued.template_id == instance.template_id - }) { - *existing = instance.clone(); - } else { - next.push(instance.clone()); + }; + + match next + .iter() + .position(|queued| queued.re_armed && same_override(queued)) + { + // Replace our own previous continuation, so one slot never holds two. + Some(index) => next[index] = instance.clone(), + None if next.iter().any(same_override) => { + // The operator has scheduled this override for that slot themselves. Theirs is the + // newer instruction and carries its own `persist`, so a continuation would only + // overwrite it with the values being carried forward - which is the transition they + // asked for, undone. Appending would do the same, just later in the slot. + debug!( + "Override {} is already scheduled for slot {}, so no continuation is queued", + instance.id, next_slot + ); + return Ok(()); + } + None => next.push(instance.clone()), } self.scheduled_overrides.store(next_slot, next)?; Ok(()) @@ -4547,6 +4563,12 @@ impl SurfnetSvm { )) })?; + // Scheduler bookkeeping, never a caller input: it is hidden from the schema and the + // bindings but still deserializes, and a caller who set it would have their own entry + // swept as though the scheduler had queued it. + let mut override_instance = override_instance; + override_instance.re_armed = false; + planned.push((absolute_slot, override_instance)); } @@ -8674,6 +8696,90 @@ mod tests { assert_eq!(read(ALLOWED_OFFSET), 5_678, "and so must the later one"); } + /// A persisted step immediately before a deliberate transition on the same id: re-arming lands + /// in the transition's bucket, and must not replace it with the values it is carrying forward. + #[tokio::test] + async fn test_re_arming_does_not_overwrite_a_scheduled_transition() { + const SLOT: u64 = 500; + const ALLOWED_OFFSET: usize = UNHEALTHY_OFFSET - 16; + + let (mut svm, account_pubkey, persisted) = scheduled_persist_fixture(true); + + // The deliberate transition, one slot later, same identity, different field. + let mut transition = persisted.clone(); + transition.scenario_relative_slot = 1; + transition.persist = surfpool_types::Persist::Always(false); + transition.values = HashMap::from([( + "allowed_borrow_value_sf".to_string(), + serde_json::json!(5_678u64), + )]); + + let scenario = |o: surfpool_types::OverrideInstance| surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![o], + }; + svm.register_scenario(scenario(persisted), Some(SLOT)) + .expect("register the persisted step"); + svm.register_scenario(scenario(transition), Some(SLOT)) + .expect("register the transition"); + + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect("materialize the persisted step"); + svm.materialize_overrides_for_slot(&None, SLOT + 1) + .await + .expect("materialize the transition"); + + let account = svm + .inner + .get_account(&account_pubkey) + .expect("get_account") + .expect("account present"); + let read = |off: usize| { + u128::from_le_bytes(account.data[off..off + 16].try_into().expect("16 bytes")) + }; + assert_eq!( + read(ALLOWED_OFFSET), + 5_678, + "the scheduled transition must have applied; re-arming replaced it instead" + ); + } + + /// `re_armed` is the scheduler's bookkeeping. A caller who sets it on a deliberate entry would + /// have that entry swept as though the scheduler had queued it, so registration clears it. + #[tokio::test] + async fn test_caller_supplied_re_armed_is_ignored() { + const SLOT: u64 = 500; + + let (mut svm, _pk, mut instance) = scheduled_persist_fixture(false); + instance.re_armed = true; + + svm.register_scenario( + surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![instance], + }, + Some(SLOT), + ) + .expect("register"); + + let queued = svm + .scheduled_overrides + .get(&SLOT) + .expect("read") + .expect("queued"); + assert!( + !queued[0].re_armed, + "registration must clear the flag, or a caller can have their own entry swept" + ); + } + /// `slots: 0` asks for zero applications, which no override can honour - it always applies on /// the slot it is scheduled for. #[tokio::test] From 47a8967e4c89aabee5310ecc407eb316c34636c7 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Wed, 26 Aug 2026 16:38:32 +0300 Subject: [PATCH 10/18] fix: suppress a continuation the same batch already supersedes --- crates/core/src/surfnet/svm.rs | 436 +++++++++------------------------ 1 file changed, 117 insertions(+), 319 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index 488111d5..ee5cb40d 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -3004,7 +3004,18 @@ impl SurfnetSvm { FetchOutcome::NoRemote | FetchOutcome::NotOnRemote => existing_account.is_some(), }; - if override_instance.persist.is_enabled() { + // A later entry in this batch that the operator scheduled supersedes this one. The + // scheduler's own collision check cannot see it: claiming several overdue buckets at + // once already took it out of its slot, so re-arming here would outlive it and restore + // the superseded value on the following slot. + let superseded_later = overrides[index + 1..].iter().any(|later| { + !later.re_armed + && later.id == override_instance.id + && later.account == override_instance.account + && later.template_id == override_instance.template_id + }); + + if override_instance.persist.is_enabled() && !superseded_later { let mut requeued = override_instance.clone(); if requeued.fetch_before_use && fetch_retired { requeued.fetch_before_use = false; @@ -4576,7 +4587,7 @@ impl SurfnetSvm { for (absolute_slot, override_instance) in planned { let scenario_relative_slot = override_instance.scenario_relative_slot; - self.remove_queued_copies_elsewhere(&override_instance)?; + self.remove_queued_copies_elsewhere(&override_instance, absolute_slot)?; debug!( "Scheduling override at absolute slot {} (base {} + relative {})", @@ -4618,8 +4629,14 @@ impl SurfnetSvm { fn remove_queued_copies_elsewhere( &mut self, instance: &OverrideInstance, + from_slot: Slot, ) -> SurfpoolResult<()> { - let slots: Vec = self.scheduled_overrides.keys()?; + let slots: Vec = self + .scheduled_overrides + .keys()? + .into_iter() + .filter(|slot| *slot >= from_slot) + .collect(); for slot in slots { let Some(mut queued) = self.scheduled_overrides.get(&slot)? else { @@ -8412,304 +8429,98 @@ mod tests { ); } - /// Pins the semantics of a future-dated one-shot, which are deliberate rather than incidental: - /// registering is declarative about the timeline, so it says "this override applies at slot N, - /// once" - it does not mean "keep persisting until N". The intervening slots are therefore - /// skipped. Use `persist: { slots: N }` to keep re-applying for a bounded window instead. + /// A future-dated one-shot schedules when the override stops, not an immediate stop: it keeps + /// re-applying until that slot, applies there, and does not re-arm. The armed continuation is + /// what carries it through the intervening slots, so this must hold whether the one-shot was + /// registered before or after the override armed itself - otherwise the same request behaves + /// differently depending on when it was made. #[tokio::test] - async fn test_a_future_one_shot_skips_the_intervening_slots() { + async fn test_a_future_one_shot_persists_until_its_slot() { const SLOT: u64 = 500; const CANCEL_AT: u64 = 3; - let (mut svm, _pk, mut instance) = scheduled_persist_fixture(true); - let scenario = |o: surfpool_types::OverrideInstance| surfpool_types::Scenario { - id: "s-1".to_string(), - name: "s".to_string(), - description: String::new(), - tags: vec![], - overrides: vec![o], - }; - - svm.register_scenario(scenario(instance.clone()), Some(SLOT)) - .expect("register persisted"); - svm.materialize_overrides_for_slot(&None, SLOT) - .await - .expect("materialize"); - - instance.persist = surfpool_types::Persist::Always(false); - instance.scenario_relative_slot = CANCEL_AT; - svm.register_scenario(scenario(instance), Some(SLOT)) - .expect("register the future one-shot"); - - // Nothing between now and the one-shot: the armed continuation is gone. - for slot in (SLOT + 1)..(SLOT + CANCEL_AT) { - assert!( - svm.scheduled_overrides - .get(&slot) - .expect("read") - .unwrap_or_default() - .is_empty(), - "slot {slot} must be empty; a future one-shot replaces the timeline rather than \ - scheduling a stop" - ); - } - assert_eq!( - svm.scheduled_overrides - .get(&(SLOT + CANCEL_AT)) - .expect("read") - .unwrap_or_default() - .len(), - 1, - "and the one-shot itself is queued for its own slot" - ); - } - - /// Claiming several overdue buckets at once must still apply an override only once. The final - /// state hides a duplicate - both copies write the same bytes and the second re-arm replaces - /// the first - so this counts the work instead: one application means one reschedule write. - #[tokio::test] - async fn test_overdue_buckets_apply_an_override_once() { - use crate::storage::{Storage, StorageResult}; - - #[derive(Clone)] - struct CountingStore { - inner: HashMap>, - writes: std::sync::Arc, - } - - impl Storage> for CountingStore { - fn store( - &mut self, - key: u64, - value: Vec, - ) -> StorageResult<()> { - self.writes - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - self.inner.insert(key, value); - Ok(()) - } - fn clear(&mut self) -> StorageResult<()> { - self.inner.clear(); - Ok(()) - } - fn get( - &self, - key: &u64, - ) -> StorageResult>> { - Ok(self.inner.get(key).cloned()) - } - fn take( - &mut self, - key: &u64, - ) -> StorageResult>> { - Ok(self.inner.remove(key)) - } - fn keys(&self) -> StorageResult> { - Ok(self.inner.keys().copied().collect()) - } - fn into_iter( - &self, - ) -> StorageResult< - Box)> + '_>, - > { - Ok(Box::new(self.inner.iter().map(|(k, v)| (*k, v.clone())))) - } - fn count(&self) -> StorageResult { - Ok(self.inner.len() as u64) - } - fn clone_box(&self) -> Box>> { - Box::new(self.clone()) - } - } - - const SLOT: u64 = 500; - - let (mut svm, _pk, base) = scheduled_persist_fixture(true); - - // Two continuations of the same override, as a stranded copy plus a newer one would be. - let mut earlier = base.clone(); - earlier.persist = surfpool_types::Persist::Slots { slots: 5 }; - earlier.re_armed = true; - let mut later = base.clone(); - later.persist = surfpool_types::Persist::Slots { slots: 2 }; - later.re_armed = true; - - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - svm.scheduled_overrides = Box::new(CountingStore { - inner: HashMap::from([(SLOT, vec![earlier]), (SLOT + 1, vec![later])]), - writes: writes.clone(), - }); - - svm.materialize_overrides_for_slot(&None, SLOT + 1) - .await - .expect("materialize both overdue buckets"); - - assert_eq!( - writes.load(std::sync::atomic::Ordering::SeqCst), - 1, - "a superseded continuation must not be handled again; it would reschedule twice and \ - could fetch or write on behalf of state the newer copy already carries forward" - ); - - let armed = svm - .scheduled_overrides - .get(&(SLOT + 2)) - .expect("read") - .unwrap_or_default(); - assert_eq!(armed.len(), 1, "one entry per override id"); - assert_eq!( - armed[0].persist, - surfpool_types::Persist::Slots { slots: 1 }, - "the copy scheduled latest must win, so its window is the one that advances" - ); - } - - /// A scenario may walk one override across a timeline, reusing the same id at several slots. - /// The sweep that clears re-armed copies must not eat those deliberate entries. - #[tokio::test] - async fn test_a_timeline_sharing_one_id_survives_registration() { - const SLOT: u64 = 500; - const LATER: u64 = 3; - - let (mut svm, _pk, first) = scheduled_persist_fixture(false); - let mut second = first.clone(); - second.scenario_relative_slot = LATER; - let scenario = |o: surfpool_types::OverrideInstance| surfpool_types::Scenario { - id: "s-1".to_string(), - name: "s".to_string(), - description: String::new(), - tags: vec![], - overrides: vec![o], - }; - - // Two separate calls, so nothing but the re-armed marker distinguishes intent. - svm.register_scenario(scenario(first), Some(SLOT)) - .expect("register the first step"); - svm.register_scenario(scenario(second), Some(SLOT)) - .expect("register the later step"); - - for slot in [SLOT, SLOT + LATER] { - assert_eq!( - svm.scheduled_overrides - .get(&slot) - .expect("read") - .unwrap_or_default() - .len(), - 1, - "the step at slot {slot} must still be scheduled" - ); - } - } - - /// A rejected scenario must schedule nothing. Validating while writing left the overrides ahead - /// of the bad one queued, so the caller saw an error and got half a scenario. - #[tokio::test] - async fn test_a_rejected_scenario_schedules_nothing() { - const SLOT: u64 = 500; - - let (mut svm, _pk, good) = scheduled_persist_fixture(false); - let mut bad = good.clone(); - bad.id = "the-invalid-one".to_string(); - bad.scenario_relative_slot = 1; - bad.persist = surfpool_types::Persist::Slots { slots: 0 }; + for materialize_first in [true, false] { + let (mut svm, _pk, persisted) = scheduled_persist_fixture(true); + let mut one_shot = persisted.clone(); + one_shot.persist = surfpool_types::Persist::Always(false); + one_shot.scenario_relative_slot = CANCEL_AT; - svm.register_scenario( - surfpool_types::Scenario { + let scenario = |o: surfpool_types::OverrideInstance| surfpool_types::Scenario { id: "s-1".to_string(), name: "s".to_string(), description: String::new(), tags: vec![], - overrides: vec![good, bad], - }, - Some(SLOT), - ) - .expect_err("slots: 0 must reject the whole scenario"); + overrides: vec![o], + }; + svm.register_scenario(scenario(persisted), Some(SLOT)) + .expect("register persisted"); + if materialize_first { + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect("materialize before cancelling"); + } + svm.register_scenario(scenario(one_shot), Some(SLOT)) + .expect("register the future one-shot"); + + // Run up to the slot before the one-shot: it must still be re-applying. + let start = if materialize_first { SLOT + 1 } else { SLOT }; + for slot in start..(SLOT + CANCEL_AT) { + svm.materialize_overrides_for_slot(&None, slot) + .await + .expect("materialize an intervening slot"); + assert_eq!( + svm.scheduled_overrides + .get(&(slot + 1)) + .expect("read") + .unwrap_or_default() + .len(), + 1, + "materialize_first={materialize_first}: the override must still be armed after \ + slot {slot}, because the one-shot is not due until {}", + SLOT + CANCEL_AT + ); + } - let scheduled: Vec = svm - .scheduled_overrides - .keys() - .expect("keys") - .into_iter() - .filter(|slot| { - !svm.scheduled_overrides - .get(slot) - .expect("read") - .unwrap_or_default() - .is_empty() - }) - .collect(); - assert!( - scheduled.is_empty(), - "the valid override must not have been scheduled either; found {scheduled:?}" - ); + // The one-shot's own slot: it applies, and nothing survives it. + svm.materialize_overrides_for_slot(&None, SLOT + CANCEL_AT) + .await + .expect("materialize the one-shot"); + let leftover: Vec = svm + .scheduled_overrides + .keys() + .expect("keys") + .into_iter() + .filter(|slot| { + !svm.scheduled_overrides + .get(slot) + .expect("read") + .unwrap_or_default() + .is_empty() + }) + .collect(); + assert!( + leftover.is_empty(), + "materialize_first={materialize_first}: the one-shot must end it; found {leftover:?}" + ); + } } - /// A timeline whose steps land in several overdue buckets must apply every step, in slot order. - /// Collapsing them to the latest loses the fields the earlier steps wrote. + /// A clock jump that claims both a continuation and the one-shot meant to end it. Both buckets + /// are taken before either is processed, so the collision check in the scheduler cannot see the + /// one-shot any more - the continuation must be suppressed from the batch instead, or it survives + /// the one-shot and restores the old value on the next slot. #[tokio::test] - async fn test_overdue_timeline_steps_all_apply_in_order() { + async fn test_a_jump_over_a_one_shot_does_not_leave_a_continuation() { const SLOT: u64 = 500; - const LATER: u64 = 3; + const ONE_SHOT_AT: u64 = 3; const JUMP_TO: u64 = 510; const ALLOWED_OFFSET: usize = UNHEALTHY_OFFSET - 16; - let (mut svm, account_pubkey, first) = scheduled_persist_fixture(false); - - // Step one writes one field, step two writes a different one, same identity. - let mut second = first.clone(); - second.scenario_relative_slot = LATER; - second.values = HashMap::from([( - "allowed_borrow_value_sf".to_string(), - serde_json::json!(5_678u64), - )]); - - let scenario = |o: surfpool_types::OverrideInstance| surfpool_types::Scenario { - id: "s-1".to_string(), - name: "s".to_string(), - description: String::new(), - tags: vec![], - overrides: vec![o], - }; - svm.register_scenario(scenario(first), Some(SLOT)) - .expect("register step one"); - svm.register_scenario(scenario(second), Some(SLOT)) - .expect("register step two"); - - // Jump past both, so a single materialization claims each bucket. - svm.materialize_overrides_for_slot(&None, JUMP_TO) - .await - .expect("materialize"); - - let account = svm - .inner - .get_account(&account_pubkey) - .expect("get_account") - .expect("account present"); - let read = |off: usize| { - u128::from_le_bytes(account.data[off..off + 16].try_into().expect("16 bytes")) - }; - assert_eq!( - read(UNHEALTHY_OFFSET), - 1_234, - "the earlier step must still have been applied" - ); - assert_eq!(read(ALLOWED_OFFSET), 5_678, "and so must the later one"); - } - - /// A persisted step immediately before a deliberate transition on the same id: re-arming lands - /// in the transition's bucket, and must not replace it with the values it is carrying forward. - #[tokio::test] - async fn test_re_arming_does_not_overwrite_a_scheduled_transition() { - const SLOT: u64 = 500; - const ALLOWED_OFFSET: usize = UNHEALTHY_OFFSET - 16; - let (mut svm, account_pubkey, persisted) = scheduled_persist_fixture(true); - - // The deliberate transition, one slot later, same identity, different field. - let mut transition = persisted.clone(); - transition.scenario_relative_slot = 1; - transition.persist = surfpool_types::Persist::Always(false); - transition.values = HashMap::from([( + let mut one_shot = persisted.clone(); + one_shot.persist = surfpool_types::Persist::Always(false); + one_shot.scenario_relative_slot = ONE_SHOT_AT; + one_shot.values = HashMap::from([( "allowed_borrow_value_sf".to_string(), serde_json::json!(5_678u64), )]); @@ -8722,16 +8533,17 @@ mod tests { overrides: vec![o], }; svm.register_scenario(scenario(persisted), Some(SLOT)) - .expect("register the persisted step"); - svm.register_scenario(scenario(transition), Some(SLOT)) - .expect("register the transition"); - + .expect("register persisted"); svm.materialize_overrides_for_slot(&None, SLOT) .await - .expect("materialize the persisted step"); - svm.materialize_overrides_for_slot(&None, SLOT + 1) + .expect("materialize, arming the next slot"); + svm.register_scenario(scenario(one_shot), Some(SLOT)) + .expect("register the one-shot"); + + // Jump clean over both the continuation and the one-shot. + svm.materialize_overrides_for_slot(&None, JUMP_TO) .await - .expect("materialize the transition"); + .expect("materialize after the jump"); let account = svm .inner @@ -8744,39 +8556,25 @@ mod tests { assert_eq!( read(ALLOWED_OFFSET), 5_678, - "the scheduled transition must have applied; re-arming replaced it instead" + "the one-shot must have applied" ); - } - /// `re_armed` is the scheduler's bookkeeping. A caller who sets it on a deliberate entry would - /// have that entry swept as though the scheduler had queued it, so registration clears it. - #[tokio::test] - async fn test_caller_supplied_re_armed_is_ignored() { - const SLOT: u64 = 500; - - let (mut svm, _pk, mut instance) = scheduled_persist_fixture(false); - instance.re_armed = true; - - svm.register_scenario( - surfpool_types::Scenario { - id: "s-1".to_string(), - name: "s".to_string(), - description: String::new(), - tags: vec![], - overrides: vec![instance], - }, - Some(SLOT), - ) - .expect("register"); - - let queued = svm + let leftover: Vec = svm .scheduled_overrides - .get(&SLOT) - .expect("read") - .expect("queued"); + .keys() + .expect("keys") + .into_iter() + .filter(|slot| { + !svm.scheduled_overrides + .get(slot) + .expect("read") + .unwrap_or_default() + .is_empty() + }) + .collect(); assert!( - !queued[0].re_armed, - "registration must clear the flag, or a caller can have their own entry swept" + leftover.is_empty(), + "the one-shot ended the override, so no continuation may outlive it; found {leftover:?}" ); } From f4100ac9ea43b27f110510f611634765940fa368 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Thu, 3 Sep 2026 16:04:33 +0300 Subject: [PATCH 11/18] feat(scenarios): add cancellation-only RPC for persisted overrides --- crates/core/src/rpc/surfnet_cheatcodes.rs | 40 ++++++- crates/core/src/surfnet/locker.rs | 12 ++ crates/core/src/surfnet/svm.rs | 108 ++++++++++++++++++ .../surfpool-sdk/kit/generated/methods.ts | 1 + crates/types/src/types.rs | 3 +- 5 files changed, 159 insertions(+), 5 deletions(-) diff --git a/crates/core/src/rpc/surfnet_cheatcodes.rs b/crates/core/src/rpc/surfnet_cheatcodes.rs index 15e05e34..1b478d55 100644 --- a/crates/core/src/rpc/surfnet_cheatcodes.rs +++ b/crates/core/src/rpc/surfnet_cheatcodes.rs @@ -16,10 +16,10 @@ use solana_system_interface::program as system_program; use solana_transaction::versioned::VersionedTransaction; use spl_associated_token_account_interface::address::get_associated_token_address_with_program_id; use surfpool_types::{ - AccountSnapshot, CheatcodeControlConfig, CheatcodeFilter, ClockCommand, ExportSnapshotConfig, - GetStreamedAccountsResponse, GetSurfnetInfoResponse, Idl, OfflineAccountConfig, - ResetAccountConfig, RpcProfileResultConfig, Scenario, SimnetCommand, StreamAccountConfig, - StreamAccountsEntry, UiKeyedProfileResult, + AccountAddress, AccountSnapshot, CheatcodeControlConfig, CheatcodeFilter, ClockCommand, + ExportSnapshotConfig, GetStreamedAccountsResponse, GetSurfnetInfoResponse, Idl, + OfflineAccountConfig, ResetAccountConfig, RpcProfileResultConfig, Scenario, SimnetCommand, + StreamAccountConfig, StreamAccountsEntry, UiKeyedProfileResult, types::{AccountUpdate, SetSomeAccount, SupplyUpdate, TokenAccountUpdate, UuidOrSignature}, }; @@ -1375,6 +1375,17 @@ pub trait SurfnetCheatcodes { scenario: Scenario, slot: Option, ) -> BoxFuture>>; + + /// Stops persisted copies of one override. This does not materialize the override again and + /// leaves independently scheduled one-shot scenario entries intact. + #[rpc(meta, name = "surfnet_stopPersistingOverride")] + fn stop_persisting_override( + &self, + meta: Self::Metadata, + id: String, + account: AccountAddress, + template_id: String, + ) -> Result>; } #[derive(Clone)] @@ -2357,6 +2368,27 @@ impl SurfnetCheatcodes for SurfnetCheatcodesRpc { }) }) } + + fn stop_persisting_override( + &self, + meta: Self::Metadata, + id: String, + account: AccountAddress, + template_id: String, + ) -> Result> { + let svm_locker = meta.get_svm_locker()?; + let removed = svm_locker + .stop_persisting_override(id, account, template_id) + .map_err(|e| jsonrpc_core::Error { + code: jsonrpc_core::ErrorCode::InternalError, + message: format!("Failed to stop persisted override: {}", e), + data: None, + })?; + Ok(RpcResponse { + context: RpcResponseContext::new(svm_locker.get_latest_absolute_slot()), + value: removed, + }) + } } #[cfg(test)] diff --git a/crates/core/src/surfnet/locker.rs b/crates/core/src/surfnet/locker.rs index 17faa1a1..23586172 100644 --- a/crates/core/src/surfnet/locker.rs +++ b/crates/core/src/surfnet/locker.rs @@ -2764,6 +2764,18 @@ impl SurfnetSvmLocker { self.with_svm_writer(move |svm_writer| svm_writer.register_scenario(scenario, slot)) } + /// Stops persisted copies of an override without materializing another account write. + pub fn stop_persisting_override( + &self, + id: String, + account: surfpool_types::AccountAddress, + template_id: String, + ) -> SurfpoolResult { + self.with_svm_writer(move |svm_writer| { + svm_writer.stop_persisting_override(&id, &account, &template_id) + }) + } + /// Materializes overrides for a specific slot (not necessarily the current slot) pub async fn materialize_overrides_for_slot( &self, diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index ee5cb40d..ce449dd2 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -4668,6 +4668,41 @@ impl SurfnetSvm { Ok(()) } + + /// Stops persisted entries of an override without applying the override again. + pub fn stop_persisting_override( + &mut self, + id: &str, + account: &surfpool_types::AccountAddress, + template_id: &str, + ) -> SurfpoolResult { + let slots = self.scheduled_overrides.keys()?; + let mut removed = 0; + + for slot in slots { + let Some(mut queued) = self.scheduled_overrides.get(&slot)? else { + continue; + }; + let before = queued.len(); + queued.retain(|other| { + !((other.re_armed || other.persist.is_enabled()) + && other.id == id + && &other.account == account + && other.template_id == template_id) + }); + removed += before - queued.len(); + + if queued.len() != before { + if queued.is_empty() { + self.scheduled_overrides.take(&slot)?; + } else { + self.scheduled_overrides.store(slot, queued)?; + } + } + } + + Ok(removed) + } } #[cfg(test)] @@ -8374,6 +8409,79 @@ mod tests { ); } + #[tokio::test] + async fn test_stop_persisting_removes_persistent_entries_without_reapplying() { + const SLOT: u64 = 500; + + let (mut svm, account_pubkey, instance) = scheduled_persist_fixture(true); + svm.scheduled_overrides + .store(SLOT, vec![instance.clone()]) + .expect("schedule persisted override"); + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect("materialize first application"); + + // Simulate a transaction updating the field after persistence was armed. Stopping must not + // replay the old override value over this newer state. + let mut account = svm + .inner + .get_account(&account_pubkey) + .expect("get account") + .expect("account present"); + account.data[UNHEALTHY_OFFSET..UNHEALTHY_OFFSET + 16] + .copy_from_slice(&9_999u128.to_le_bytes()); + svm.inner + .set_account(account_pubkey, account) + .expect("update account after first application"); + + // A future persistent start must be cancelled even though it has not re-armed yet. An + // ordinary one-shot with the same identity is not persistence bookkeeping and survives. + let future_persistent = instance.clone(); + let mut operator_scheduled = instance.clone(); + operator_scheduled.persist = surfpool_types::Persist::Always(false); + operator_scheduled.re_armed = false; + svm.scheduled_overrides + .store(SLOT + 10, vec![future_persistent, operator_scheduled]) + .expect("schedule explicit future overrides"); + + let removed = svm + .stop_persisting_override(&instance.id, &instance.account, &instance.template_id) + .expect("stop persistence"); + assert_eq!( + removed, 2, + "the continuation and future persistent start are removed" + ); + assert!( + svm.scheduled_overrides + .get(&(SLOT + 1)) + .expect("read continuation slot") + .unwrap_or_default() + .is_empty(), + "the active continuation must be gone" + ); + assert_eq!( + svm.scheduled_overrides + .get(&(SLOT + 10)) + .expect("read operator slot") + .unwrap_or_default() + .len(), + 1, + "an independently scheduled override must remain" + ); + + let account = svm + .inner + .get_account(&account_pubkey) + .expect("get account") + .expect("account present"); + let unhealthy = u128::from_le_bytes( + account.data[UNHEALTHY_OFFSET..UNHEALTHY_OFFSET + 16] + .try_into() + .expect("16 bytes"), + ); + assert_eq!(unhealthy, 9_999, "cancellation must not write account data"); + } + /// Cancelling at a FUTURE relative slot. The armed copy sits before the cancellation bucket, so /// sweeping only later slots leaves it alive - and when it re-arms it lands on the cancellation /// bucket and replaces the one-shot with itself, so the override never stops. diff --git a/crates/sdk-node/surfpool-sdk/kit/generated/methods.ts b/crates/sdk-node/surfpool-sdk/kit/generated/methods.ts index 7f4613f1..630f2dd0 100644 --- a/crates/sdk-node/surfpool-sdk/kit/generated/methods.ts +++ b/crates/sdk-node/surfpool-sdk/kit/generated/methods.ts @@ -25,6 +25,7 @@ export const SURFNET_CHEATCODE_METHODS = [ "surfnet_setTokenAccount", "surfnet_streamAccount", "surfnet_streamAccounts", + "surfnet_stopPersistingOverride", "surfnet_timeTravel", "surfnet_writeProgram", ] as const; diff --git a/crates/types/src/types.rs b/crates/types/src/types.rs index ae86afd2..f242aef4 100644 --- a/crates/types/src/types.rs +++ b/crates/types/src/types.rs @@ -1715,7 +1715,7 @@ pub enum CheatcodeFilter { /// `surfpool-core/src/rpc/surfnet_cheatcodes.rs` asserts it matches the /// methods actually registered by the `SurfnetCheatcodes` trait, so adding, /// removing, or renaming a cheatcode without updating this list fails CI. -pub const SURFNET_CHEATCODE_METHODS: [&str; 26] = [ +pub const SURFNET_CHEATCODE_METHODS: [&str; 27] = [ "surfnet_cloneProgramAccount", "surfnet_disableCheatcode", "surfnet_enableCheatcode", @@ -1740,6 +1740,7 @@ pub const SURFNET_CHEATCODE_METHODS: [&str; 26] = [ "surfnet_setTokenAccount", "surfnet_streamAccount", "surfnet_streamAccounts", + "surfnet_stopPersistingOverride", "surfnet_timeTravel", "surfnet_writeProgram", ]; From 1766e872bb034de1bf8a25943e60c42ad55f1a8b Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Thu, 3 Sep 2026 17:24:20 +0300 Subject: [PATCH 12/18] fix(scenarios): distinguish PDA override targets by resolved account --- crates/core/src/surfnet/svm.rs | 165 +++++++++++++++++++++++++++------ 1 file changed, 137 insertions(+), 28 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index c771b161..45963722 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -545,6 +545,23 @@ pub struct SurfnetSvm { storage_backend: StorageBackend, } +/// Whether two scheduled entries describe the same logical override on the same concrete account. +fn same_override_target(left: &OverrideInstance, right: &OverrideInstance) -> bool { + if left.id != right.id || left.template_id != right.template_id { + return false; + } + + match ( + left.account.resolve(Some(&left.values)), + right.account.resolve(Some(&right.values)), + ) { + (Some(left), Some(right)) => left == right, + // An entry whose target cannot be resolved will be skipped during materialization. Do not + // let it suppress or replace another entry merely because their recipes look alike. + _ => false, + } +} + /// Add `pubkey_str` to the pubkey-list at `key`, creating the entry when absent /// and deduplicating on insert. The shared-pubkey indexes (`accounts_by_owner`, /// `token_accounts_by_owner`, `token_accounts_by_mint`, @@ -2878,12 +2895,9 @@ impl SurfnetSvm { let mut deduped: Vec = Vec::with_capacity(overrides.len()); for instance in overrides.into_iter().rev() { if instance.re_armed - && deduped.iter().any(|kept| { - kept.re_armed - && kept.id == instance.id - && kept.account == instance.account - && kept.template_id == instance.template_id - }) + && deduped + .iter() + .any(|kept| kept.re_armed && same_override_target(kept, &instance)) { debug!( "Dropping a superseded continuation of override {} claimed from an earlier slot", @@ -3075,12 +3089,9 @@ impl SurfnetSvm { // scheduler's own collision check cannot see it: claiming several overdue buckets at // once already took it out of its slot, so re-arming here would outlive it and restore // the superseded value on the following slot. - let superseded_later = overrides[index + 1..].iter().any(|later| { - !later.re_armed - && later.id == override_instance.id - && later.account == override_instance.account - && later.template_id == override_instance.template_id - }); + let superseded_later = overrides[index + 1..] + .iter() + .any(|later| !later.re_armed && same_override_target(later, override_instance)); if override_instance.persist.is_enabled() && !superseded_later { let mut requeued = override_instance.clone(); @@ -3317,11 +3328,7 @@ impl SurfnetSvm { .get(&next_slot)? .unwrap_or_default(); - let same_override = |queued: &OverrideInstance| { - queued.id == instance.id - && queued.account == instance.account - && queued.template_id == instance.template_id - }; + let same_override = |queued: &OverrideInstance| same_override_target(queued, instance); match next .iter() @@ -4683,11 +4690,10 @@ impl SurfnetSvm { .get(&absolute_slot)? .unwrap_or_default(); - if let Some(existing) = slot_overrides.iter_mut().find(|queued| { - queued.id == override_instance.id - && queued.account == override_instance.account - && queued.template_id == override_instance.template_id - }) { + if let Some(existing) = slot_overrides + .iter_mut() + .find(|queued| same_override_target(queued, &override_instance)) + { debug!( "Replacing already-scheduled override {} at slot {}", override_instance.id, absolute_slot @@ -4727,12 +4733,7 @@ impl SurfnetSvm { continue; }; let before = queued.len(); - queued.retain(|other| { - !(other.re_armed - && other.id == instance.id - && other.account == instance.account - && other.template_id == instance.template_id) - }); + queued.retain(|other| !(other.re_armed && same_override_target(other, instance))); if queued.len() != before { debug!( "Removed {} superseded copy(ies) of override {} at slot {}", @@ -8363,6 +8364,114 @@ mod tests { ); } + /// Structurally identical PDA recipes can still identify different accounts when their seeds + /// come from override values. + #[tokio::test] + async fn test_overdue_pda_overrides_compare_resolved_accounts() { + const SLOT: u64 = 500; + const LATER_SLOT: u64 = SLOT + 5; + + let (mut svm, fixture_pubkey, mut first) = scheduled_persist_fixture(true); + let fixture_account = svm + .inner + .get_account(&fixture_pubkey) + .expect("get fixture account") + .expect("fixture account present"); + + let program_id = Pubkey::new_unique(); + let recipe = surfpool_types::AccountAddress::Pda { + program_id: program_id.to_string(), + seeds: vec![surfpool_types::PdaSeed::PropertyRef("market".to_string())], + }; + + first.id = "shared-pda-id".to_string(); + first.account = recipe.clone(); + first.values.insert( + "market".to_string(), + serde_json::Value::String("first-market".to_string()), + ); + let first_pubkey = first + .account + .resolve(Some(&first.values)) + .expect("derive first PDA"); + + let mut later = first.clone(); + later.scenario_relative_slot = LATER_SLOT - SLOT; + later.values.insert( + "market".to_string(), + serde_json::Value::String("second-market".to_string()), + ); + later.values.insert( + "unhealthy_borrow_value_sf".to_string(), + serde_json::json!(5_678u64), + ); + let later_pubkey = later + .account + .resolve(Some(&later.values)) + .expect("derive later PDA"); + assert_ne!( + first_pubkey, later_pubkey, + "different seeds must derive different PDAs" + ); + + svm.inner + .set_account(first_pubkey, fixture_account.clone()) + .expect("set first PDA account"); + svm.inner + .set_account(later_pubkey, fixture_account) + .expect("set later PDA account"); + svm.scheduled_overrides + .store(SLOT, vec![first]) + .expect("schedule first override"); + svm.scheduled_overrides + .store(LATER_SLOT, vec![later]) + .expect("schedule later override"); + + // Claim both buckets together, reproducing the overdue-slot batch from the report. + svm.materialize_overrides_for_slot(&None, LATER_SLOT) + .await + .expect("materialize overdue batch"); + + let queued = svm + .scheduled_overrides + .get(&(LATER_SLOT + 1)) + .expect("read continuations") + .expect("continuations queued"); + assert_eq!( + queued.len(), + 2, + "the later PDA must not suppress the first PDA's continuation" + ); + let queued_accounts: HashSet = queued + .iter() + .map(|entry| { + entry + .account + .resolve(Some(&entry.values)) + .expect("resolve queued PDA") + }) + .collect(); + assert_eq!( + queued_accounts, + HashSet::from([first_pubkey, later_pubkey]), + "each concrete PDA must retain its own continuation" + ); + + // The overdue deduplication path must distinguish the same two resolved accounts too. + svm.materialize_overrides_for_slot(&None, LATER_SLOT + 1) + .await + .expect("materialize continuations"); + assert_eq!( + svm.scheduled_overrides + .get(&(LATER_SLOT + 2)) + .expect("read next continuations") + .unwrap_or_default() + .len(), + 2, + "re-armed PDA overrides must not be deduplicated by recipe alone" + ); + } + /// A relative window re-arms for exactly the requested number of slots and then stops. #[tokio::test] async fn test_persist_for_slots_stops_after_the_window() { From f2598cd72e2c1b382e2bc3644ed9a80965c583b7 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Thu, 3 Sep 2026 17:42:42 +0300 Subject: [PATCH 13/18] fix(scenarios): resolve PDA targets when stopping persistent overrides --- crates/core/src/rpc/surfnet_cheatcodes.rs | 72 ++++++++++++- crates/core/src/surfnet/locker.rs | 3 +- crates/core/src/surfnet/svm.rs | 102 +++++++++++++++++- .../kit/__typetests__/typetests.ts | 17 +++ crates/sdk-node/surfpool-sdk/kit/types/api.ts | 10 ++ crates/types/src/rpc_endpoints.json | 27 +++++ 6 files changed, 224 insertions(+), 7 deletions(-) diff --git a/crates/core/src/rpc/surfnet_cheatcodes.rs b/crates/core/src/rpc/surfnet_cheatcodes.rs index 6751d592..43617035 100644 --- a/crates/core/src/rpc/surfnet_cheatcodes.rs +++ b/crates/core/src/rpc/surfnet_cheatcodes.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashMap}, sync::{Arc, RwLock}, }; @@ -1518,7 +1518,10 @@ pub trait SurfnetCheatcodes { ) -> BoxFuture>>; /// Stops scheduler-generated persisted copies of one override. This does not materialize the - /// override again and leaves all independently authored timeline entries intact. + /// override again and leaves all independently authored timeline entries intact. `values` + /// supplies property-reference PDA seeds; callers may instead pass the resolved pubkey as + /// `account`. Omitting both forms of concrete identity is rejected rather than cancelling + /// every continuation that happens to share a PDA recipe. #[rpc(meta, name = "surfnet_stopPersistingOverride")] fn stop_persisting_override( &self, @@ -1526,6 +1529,7 @@ pub trait SurfnetCheatcodes { id: String, account: AccountAddress, template_id: String, + values: Option>, ) -> Result>; } @@ -2598,10 +2602,11 @@ impl SurfnetCheatcodes for SurfnetCheatcodesRpc { id: String, account: AccountAddress, template_id: String, + values: Option>, ) -> Result> { let svm_locker = meta.get_svm_locker()?; let removed = svm_locker - .stop_persisting_override(id, account, template_id) + .stop_persisting_override(id, account, template_id, values) .map_err(|e| jsonrpc_core::Error { code: jsonrpc_core::ErrorCode::InternalError, message: format!("Failed to stop persisted override: {}", e), @@ -2661,6 +2666,67 @@ mod tests { assert_eq!(registered, manifest); } + #[tokio::test] + async fn stop_persisting_wire_accepts_optional_pda_values() { + let mut io: jsonrpc_core::MetaIoHandler> = + jsonrpc_core::MetaIoHandler::default(); + io.extend_with(SurfnetCheatcodesRpc::empty().to_delegate()); + let account = serde_json::json!({ "pubkey": Pubkey::new_unique().to_string() }); + + for params in [ + serde_json::json!(["override", account.clone(), "template"]), + serde_json::json!(["override", account.clone(), "template", { "market": "SOL" }]), + ] { + let request = serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "surfnet_stopPersistingOverride", + "params": params, + }) + .to_string(); + let response = io + .handle_request(&request, None) + .await + .expect("request produces a response"); + let response: serde_json::Value = + serde_json::from_str(&response).expect("valid JSON-RPC response"); + assert_ne!( + response + .pointer("/error/code") + .and_then(|code| code.as_i64()), + Some(-32602), + "both the legacy three-argument call and the PDA-aware four-argument call must deserialize" + ); + } + } + + #[test] + fn stop_persisting_endpoint_metadata_matches_the_wire_contract() { + let metadata: serde_json::Value = + serde_json::from_str(include_str!("../../../types/src/rpc_endpoints.json")) + .expect("RPC endpoint metadata is valid JSON"); + let endpoint = metadata["categories"] + .as_array() + .expect("categories array") + .iter() + .flat_map(|category| category["endpoints"].as_array().into_iter().flatten()) + .find(|endpoint| endpoint["method"] == "surfnet_stopPersistingOverride") + .expect("stop-persistence endpoint is discoverable"); + let parameter_names: Vec<&str> = endpoint["params"] + .as_array() + .expect("params array") + .iter() + .map(|parameter| parameter["name"].as_str().expect("parameter name")) + .collect(); + assert_eq!(parameter_names, ["id", "account", "template_id", "values"]); + assert_eq!( + endpoint["returns"].as_str(), + Some( + "A `RpcResponse` containing the number of scheduler-generated continuations removed." + ) + ); + } + /// Pins the wire shape of `TimeTravelConfig`, which is hand-mirrored in /// the TypeScript bindings /// (`crates/sdk-node/surfpool-sdk/kit/types/api.ts`). Update that mirror diff --git a/crates/core/src/surfnet/locker.rs b/crates/core/src/surfnet/locker.rs index 6e54542c..6a34ed48 100644 --- a/crates/core/src/surfnet/locker.rs +++ b/crates/core/src/surfnet/locker.rs @@ -2810,9 +2810,10 @@ impl SurfnetSvmLocker { id: String, account: surfpool_types::AccountAddress, template_id: String, + values: Option>, ) -> SurfpoolResult { self.with_svm_writer(move |svm_writer| { - svm_writer.stop_persisting_override(&id, &account, &template_id) + svm_writer.stop_persisting_override(&id, &account, &template_id, values.as_ref()) }) } diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index 45963722..e67ca119 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -4763,7 +4763,14 @@ impl SurfnetSvm { id: &str, account: &surfpool_types::AccountAddress, template_id: &str, + values: Option<&HashMap>, ) -> SurfpoolResult { + let target_account = account.resolve(values).ok_or_else(|| { + SurfpoolError::internal(format!( + "Cannot stop persisted override {id}: its account address cannot be resolved. \ + Supply the property-reference values used by the PDA, or pass the resolved pubkey." + )) + })?; let slots = self.scheduled_overrides.keys()?; let mut removed = 0; @@ -4775,8 +4782,8 @@ impl SurfnetSvm { queued.retain(|other| { !(other.re_armed && other.id == id - && &other.account == account - && other.template_id == template_id) + && other.template_id == template_id + && other.account.resolve(Some(&other.values)) == Some(target_account)) }); removed += before - queued.len(); @@ -8738,7 +8745,12 @@ mod tests { .expect("schedule future one-shot transition"); let removed = svm - .stop_persisting_override(&instance.id, &instance.account, &instance.template_id) + .stop_persisting_override( + &instance.id, + &instance.account, + &instance.template_id, + Some(&instance.values), + ) .expect("stop persistence"); assert_eq!(removed, 1, "only the generated continuation is removed"); assert!( @@ -8825,6 +8837,90 @@ mod tests { ); } + #[test] + fn test_stop_persisting_distinguishes_property_backed_pdas() { + const SLOT: u64 = 500; + + let (mut svm, _account_pubkey, mut first) = scheduled_persist_fixture(true); + let recipe = surfpool_types::AccountAddress::Pda { + program_id: Pubkey::new_unique().to_string(), + seeds: vec![surfpool_types::PdaSeed::PropertyRef("market".to_string())], + }; + first.id = "shared-pda-id".to_string(); + first.account = recipe.clone(); + first.re_armed = true; + first.values.insert( + "market".to_string(), + serde_json::Value::String("first-market".to_string()), + ); + let first_pubkey = first + .account + .resolve(Some(&first.values)) + .expect("derive first PDA"); + + let mut second = first.clone(); + second.values.insert( + "market".to_string(), + serde_json::Value::String("second-market".to_string()), + ); + let second_pubkey = second + .account + .resolve(Some(&second.values)) + .expect("derive second PDA"); + assert_ne!(first_pubkey, second_pubkey); + + svm.scheduled_overrides + .store(SLOT, vec![first.clone(), second.clone()]) + .expect("schedule PDA continuations"); + + let unresolved = svm.stop_persisting_override(&first.id, &recipe, &first.template_id, None); + assert!( + unresolved.is_err(), + "an unresolved PDA recipe must fail safely rather than cancelling both accounts" + ); + assert_eq!( + svm.scheduled_overrides + .get(&SLOT) + .expect("read after rejected stop") + .expect("continuations remain") + .len(), + 2 + ); + + let removed = svm + .stop_persisting_override(&first.id, &recipe, &first.template_id, Some(&first.values)) + .expect("stop first PDA by recipe and values"); + assert_eq!(removed, 1); + let remaining = svm + .scheduled_overrides + .get(&SLOT) + .expect("read remaining continuation") + .expect("second continuation remains"); + assert_eq!(remaining.len(), 1); + assert_eq!( + remaining[0].account.resolve(Some(&remaining[0].values)), + Some(second_pubkey), + "stopping the first PDA must preserve the second PDA" + ); + + let removed = svm + .stop_persisting_override( + &second.id, + &surfpool_types::AccountAddress::Pubkey(second_pubkey.to_string()), + &second.template_id, + None, + ) + .expect("stop second PDA by resolved pubkey"); + assert_eq!(removed, 1); + assert!( + svm.scheduled_overrides + .get(&SLOT) + .expect("read final slot") + .unwrap_or_default() + .is_empty() + ); + } + /// Cancelling at a FUTURE relative slot. The armed copy sits before the cancellation bucket, so /// sweeping only later slots leaves it alive - and when it re-arms it lands on the cancellation /// bucket and replaces the one-shot with itself, so the override never stops. diff --git a/crates/sdk-node/surfpool-sdk/kit/__typetests__/typetests.ts b/crates/sdk-node/surfpool-sdk/kit/__typetests__/typetests.ts index d3e11662..1f28a97a 100644 --- a/crates/sdk-node/surfpool-sdk/kit/__typetests__/typetests.ts +++ b/crates/sdk-node/surfpool-sdk/kit/__typetests__/typetests.ts @@ -15,6 +15,23 @@ void (async () => { client.surfnet.stop(); void client.payer.address; void client.cheatcodes.pauseClock(); + void client.cheatcodes.stopPersistingOverride( + 'override-id', + { + pda: { + programId: '11111111111111111111111111111111', + seeds: [{ propertyRef: 'market' }], + }, + }, + 'template-id', + { market: 'SOL' }, + ); + // Concrete pubkeys do not require PDA seed values. + void client.cheatcodes.stopPersistingOverride( + 'override-id', + { pubkey: '11111111111111111111111111111111' }, + 'template-id', + ); void client.rpc.getSlot(); void client.sendTransactions; }); diff --git a/crates/sdk-node/surfpool-sdk/kit/types/api.ts b/crates/sdk-node/surfpool-sdk/kit/types/api.ts index 0c8de043..92381bb2 100644 --- a/crates/sdk-node/surfpool-sdk/kit/types/api.ts +++ b/crates/sdk-node/surfpool-sdk/kit/types/api.ts @@ -2,6 +2,7 @@ import type { Address, GetEpochInfoApi } from '@solana/kit'; import type { AccountSnapshot, + AccountAddress, AccountUpdate, CheatcodeControlConfig, ConfidentialBalanceKeys, @@ -184,6 +185,14 @@ export type SurfnetExportSnapshotApi = { export type SurfnetRegisterScenarioApi = { registerScenario(scenario: Scenario, slot?: number | bigint): null; }; +export type SurfnetStopPersistingOverrideApi = { + stopPersistingOverride( + id: string, + account: AccountAddress, + templateId: string, + values?: Readonly> + ): number; +}; // Local export type SurfnetGetLocalSignaturesApi = { @@ -222,6 +231,7 @@ export type SurfnetCheatcodesApi = SurfnetCloneProgramAccountApi & SurfnetSetTokenAccountApi & SurfnetStreamAccountApi & SurfnetStreamAccountsApi & + SurfnetStopPersistingOverrideApi & SurfnetTimeTravelApi & SurfnetWriteProgramApi; diff --git a/crates/types/src/rpc_endpoints.json b/crates/types/src/rpc_endpoints.json index aa917870..77ddac78 100644 --- a/crates/types/src/rpc_endpoints.json +++ b/crates/types/src/rpc_endpoints.json @@ -875,6 +875,33 @@ } ], "returns": "A `RpcResponse<()>` indicating whether the write was successful." + }, + { + "method": "surfnet_stopPersistingOverride", + "description": "Stops scheduler-generated continuations of one persisted scenario override without applying the override again. Independently authored future timeline entries are preserved.", + "params": [ + { + "name": "id", + "type": "string", + "description": "The override instance identifier." + }, + { + "name": "account", + "type": "AccountAddress", + "description": "The account identity as either a concrete pubkey or the original PDA recipe." + }, + { + "name": "template_id", + "type": "string", + "description": "The override template identifier." + }, + { + "name": "values", + "type": "Option>", + "description": "Values used to resolve property-backed PDA seeds. Optional when account is already a concrete pubkey or contains no property-reference seeds." + } + ], + "returns": "A `RpcResponse` containing the number of scheduler-generated continuations removed." } ] }, From a6bd2f9f34b281caa1acbfc984e5514504003169 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Fri, 4 Sep 2026 09:35:19 +0300 Subject: [PATCH 14/18] fix(scenarios): prevent disabled overrides from stopping persistence --- crates/core/src/surfnet/svm.rs | 146 ++++++++++++++++-- .../kit/__typetests__/typetests.ts | 35 +++-- crates/sdk-node/surfpool-sdk/kit/types/api.ts | 2 +- 3 files changed, 158 insertions(+), 25 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index e67ca119..750a56a1 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -3089,9 +3089,9 @@ impl SurfnetSvm { // scheduler's own collision check cannot see it: claiming several overdue buckets at // once already took it out of its slot, so re-arming here would outlive it and restore // the superseded value on the following slot. - let superseded_later = overrides[index + 1..] - .iter() - .any(|later| !later.re_armed && same_override_target(later, override_instance)); + let superseded_later = overrides[index + 1..].iter().any(|later| { + later.enabled && !later.re_armed && same_override_target(later, override_instance) + }); if override_instance.persist.is_enabled() && !superseded_later { let mut requeued = override_instance.clone(); @@ -3336,7 +3336,10 @@ impl SurfnetSvm { { // Replace our own previous continuation, so one slot never holds two. Some(index) => next[index] = instance.clone(), - None if next.iter().any(same_override) => { + None if next + .iter() + .any(|queued| queued.enabled && same_override(queued)) => + { // The operator has scheduled this override for that slot themselves. Theirs is the // newer instruction and carries its own `persist`, so a continuation would only // overwrite it with the values being carried forward - which is the transition they @@ -4678,7 +4681,9 @@ impl SurfnetSvm { for (absolute_slot, override_instance) in planned { let scenario_relative_slot = override_instance.scenario_relative_slot; - self.remove_queued_copies_elsewhere(&override_instance, absolute_slot)?; + if override_instance.enabled { + self.remove_queued_copies_elsewhere(&override_instance, absolute_slot)?; + } debug!( "Scheduling override at absolute slot {} (base {} + relative {})", @@ -4690,10 +4695,10 @@ impl SurfnetSvm { .get(&absolute_slot)? .unwrap_or_default(); - if let Some(existing) = slot_overrides - .iter_mut() - .find(|queued| same_override_target(queued, &override_instance)) - { + if let Some(existing) = slot_overrides.iter_mut().find(|queued| { + same_override_target(queued, &override_instance) + && (override_instance.enabled || !queued.re_armed) + }) { debug!( "Replacing already-scheduled override {} at slot {}", override_instance.id, absolute_slot @@ -8698,6 +8703,129 @@ mod tests { ); } + #[tokio::test] + async fn test_disabled_entries_do_not_stop_persistence() { + const SLOT: u64 = 500; + + let scenario = |instance: surfpool_types::OverrideInstance| surfpool_types::Scenario { + id: "s-1".to_string(), + name: "s".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![instance], + }; + + // A disabled entry that is already waiting in the next slot must not block the active + // override from arming that slot. + let (mut svm, _account_pubkey, persisted) = scheduled_persist_fixture(true); + let mut disabled = persisted.clone(); + disabled.enabled = false; + disabled.scenario_relative_slot = 1; + disabled.values.insert( + "unhealthy_borrow_value_sf".to_string(), + serde_json::json!(9_999u64), + ); + svm.register_scenario(scenario(persisted.clone()), Some(SLOT)) + .expect("register persistent override"); + svm.register_scenario(scenario(disabled.clone()), Some(SLOT)) + .expect("register disabled entry"); + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect("materialize persistent override"); + let next = svm + .scheduled_overrides + .get(&(SLOT + 1)) + .expect("read next slot") + .expect("entries queued"); + assert_eq!( + next.len(), + 2, + "the disabled entry and continuation must coexist" + ); + assert_eq!(next.iter().filter(|entry| entry.re_armed).count(), 1); + + // Registering a disabled entry after the continuation exists must neither sweep it nor + // replace it at the same slot. + let (mut svm, account_pubkey, persisted) = scheduled_persist_fixture(true); + svm.register_scenario(scenario(persisted.clone()), Some(SLOT)) + .expect("register persistent override"); + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect("materialize persistent override"); + let mut disabled = persisted.clone(); + disabled.enabled = false; + disabled.scenario_relative_slot = 1; + disabled.values.insert( + "unhealthy_borrow_value_sf".to_string(), + serde_json::json!(9_999u64), + ); + svm.register_scenario(scenario(disabled), Some(SLOT)) + .expect("register disabled entry after arming"); + let next = svm + .scheduled_overrides + .get(&(SLOT + 1)) + .expect("read next slot") + .expect("entries queued"); + assert_eq!(next.len(), 2, "registration must preserve the continuation"); + assert_eq!(next.iter().filter(|entry| entry.re_armed).count(), 1); + svm.materialize_overrides_for_slot(&None, SLOT + 1) + .await + .expect("materialize continuation beside disabled entry"); + let account = svm + .inner + .get_account(&account_pubkey) + .expect("get account") + .expect("account present"); + assert_eq!( + u128::from_le_bytes( + account.data[UNHEALTHY_OFFSET..UNHEALTHY_OFFSET + 16] + .try_into() + .expect("16 bytes") + ), + 1_234, + "the disabled entry must not write its distinct value" + ); + assert_eq!( + svm.scheduled_overrides + .get(&(SLOT + 2)) + .expect("read re-armed slot") + .unwrap_or_default() + .iter() + .filter(|entry| entry.re_armed) + .count(), + 1, + "the enabled continuation must keep persisting" + ); + + // A clock jump claims the continuation and a later disabled entry in one batch. The later + // inert entry must not suppress re-arming from the overdue continuation. + let (mut svm, _account_pubkey, persisted) = scheduled_persist_fixture(true); + svm.register_scenario(scenario(persisted.clone()), Some(SLOT)) + .expect("register persistent override"); + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect("materialize persistent override"); + let mut disabled = persisted; + disabled.enabled = false; + disabled.scenario_relative_slot = 3; + svm.register_scenario(scenario(disabled), Some(SLOT)) + .expect("register later disabled entry"); + svm.materialize_overrides_for_slot(&None, SLOT + 10) + .await + .expect("materialize overdue batch"); + assert_eq!( + svm.scheduled_overrides + .get(&(SLOT + 11)) + .expect("read post-jump slot") + .unwrap_or_default() + .iter() + .filter(|entry| entry.re_armed) + .count(), + 1, + "a disabled later entry must not suppress overdue re-arming" + ); + } + #[tokio::test] async fn test_stop_persisting_removes_only_continuations_without_reapplying() { const SLOT: u64 = 500; diff --git a/crates/sdk-node/surfpool-sdk/kit/__typetests__/typetests.ts b/crates/sdk-node/surfpool-sdk/kit/__typetests__/typetests.ts index 1f28a97a..87934c6e 100644 --- a/crates/sdk-node/surfpool-sdk/kit/__typetests__/typetests.ts +++ b/crates/sdk-node/surfpool-sdk/kit/__typetests__/typetests.ts @@ -15,23 +15,28 @@ void (async () => { client.surfnet.stop(); void client.payer.address; void client.cheatcodes.pauseClock(); - void client.cheatcodes.stopPersistingOverride( - 'override-id', - { - pda: { - programId: '11111111111111111111111111111111', - seeds: [{ propertyRef: 'market' }], + const removedByPda: bigint = await client.cheatcodes + .stopPersistingOverride( + 'override-id', + { + pda: { + programId: '11111111111111111111111111111111', + seeds: [{ propertyRef: 'market' }], + }, }, - }, - 'template-id', - { market: 'SOL' }, - ); + 'template-id', + { market: 'SOL' }, + ) + .send(); // Concrete pubkeys do not require PDA seed values. - void client.cheatcodes.stopPersistingOverride( - 'override-id', - { pubkey: '11111111111111111111111111111111' }, - 'template-id', - ); + const removedByPubkey: bigint = await client.cheatcodes + .stopPersistingOverride( + 'override-id', + { pubkey: '11111111111111111111111111111111' }, + 'template-id', + ) + .send(); + void (removedByPda + removedByPubkey); void client.rpc.getSlot(); void client.sendTransactions; }); diff --git a/crates/sdk-node/surfpool-sdk/kit/types/api.ts b/crates/sdk-node/surfpool-sdk/kit/types/api.ts index 92381bb2..544c2732 100644 --- a/crates/sdk-node/surfpool-sdk/kit/types/api.ts +++ b/crates/sdk-node/surfpool-sdk/kit/types/api.ts @@ -191,7 +191,7 @@ export type SurfnetStopPersistingOverrideApi = { account: AccountAddress, templateId: string, values?: Readonly> - ): number; + ): bigint; }; // Local From 3660c1600f73b61d430eadd7b1634f217cb71844 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Fri, 4 Sep 2026 09:58:07 +0300 Subject: [PATCH 15/18] fix(scenarios): make persistence queue updates atomic --- crates/core/src/surfnet/svm.rs | 238 +++++++++++++++++++++++++-------- 1 file changed, 182 insertions(+), 56 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index 750a56a1..3d3490f3 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -4677,12 +4677,24 @@ impl SurfnetSvm { planned.push((absolute_slot, override_instance)); } - // Schedule overrides by adding base slot to their scenario-relative slots + let original_queue: BTreeMap> = + self.scheduled_overrides.into_iter()?.collect(); + let mut staged_queue = original_queue.clone(); + + // Schedule overrides by adding base slot to their scenario-relative slots. for (absolute_slot, override_instance) in planned { let scenario_relative_slot = override_instance.scenario_relative_slot; if override_instance.enabled { - self.remove_queued_copies_elsewhere(&override_instance, absolute_slot)?; + for queued in staged_queue + .range_mut(absolute_slot..) + .map(|(_, queued)| queued) + { + queued.retain(|other| { + !(other.re_armed && same_override_target(other, &override_instance)) + }); + } + staged_queue.retain(|_, queued| !queued.is_empty()); } debug!( @@ -4690,10 +4702,7 @@ impl SurfnetSvm { absolute_slot, base_slot, scenario_relative_slot ); - let mut slot_overrides = self - .scheduled_overrides - .get(&absolute_slot)? - .unwrap_or_default(); + let slot_overrides = staged_queue.entry(absolute_slot).or_default(); if let Some(existing) = slot_overrides.iter_mut().find(|queued| { same_override_target(queued, &override_instance) @@ -4707,53 +4716,56 @@ impl SurfnetSvm { } else { slot_overrides.push(override_instance); } - self.scheduled_overrides - .store(absolute_slot, slot_overrides)?; } - Ok(()) + self.commit_scheduled_override_plan(&original_queue, &staged_queue) } - /// Drops the copies `instance` queued for itself to keep persisting, wherever they sit. - /// - /// Only re-armed copies are removed. Entries an operator scheduled are left alone at every - /// slot, because a scenario may place one id at several slots to walk a value over a timeline - - /// possibly across separate registrations - and those are not this sweep's business. Sweeping - /// every slot rather than only later ones matters because the armed copy sits *before* a - /// cancellation scheduled at a positive relative slot. - fn remove_queued_copies_elsewhere( + /// Commits a staged scheduled-override queue and restores every attempted bucket if any + /// storage operation fails. The failed bucket is included in rollback because a storage + /// implementation is not required to guarantee that an error happened before mutation. + fn commit_scheduled_override_plan( &mut self, - instance: &OverrideInstance, - from_slot: Slot, + original: &BTreeMap>, + staged: &BTreeMap>, ) -> SurfpoolResult<()> { - let slots: Vec = self - .scheduled_overrides - .keys()? - .into_iter() - .filter(|slot| *slot >= from_slot) - .collect(); - - for slot in slots { - let Some(mut queued) = self.scheduled_overrides.get(&slot)? else { + let mut changed_slots: Vec = original.keys().chain(staged.keys()).copied().collect(); + changed_slots.sort_unstable(); + changed_slots.dedup(); + changed_slots.retain(|slot| original.get(slot) != staged.get(slot)); + + let mut attempted = Vec::with_capacity(changed_slots.len()); + for slot in changed_slots { + attempted.push(slot); + let commit = match staged.get(&slot) { + Some(queued) => self.scheduled_overrides.store(slot, queued.clone()), + None => self.scheduled_overrides.take(&slot).map(|_| ()), + }; + let Err(commit_error) = commit else { continue; }; - let before = queued.len(); - queued.retain(|other| !(other.re_armed && same_override_target(other, instance))); - if queued.len() != before { - debug!( - "Removed {} superseded copy(ies) of override {} at slot {}", - before - queued.len(), - instance.id, - slot - ); - // Drop the key rather than leaving an empty vec behind: a slot that holds nothing - // still shows up in `keys()`, so every sweep would walk more dead entries. - if queued.is_empty() { - self.scheduled_overrides.take(&slot)?; - } else { - self.scheduled_overrides.store(slot, queued)?; + + let mut rollback_errors = Vec::new(); + for rollback_slot in attempted.into_iter().rev() { + let rollback = match original.get(&rollback_slot) { + Some(queued) => self + .scheduled_overrides + .store(rollback_slot, queued.clone()), + None => self.scheduled_overrides.take(&rollback_slot).map(|_| ()), + }; + if let Err(error) = rollback { + rollback_errors.push(format!("slot {rollback_slot}: {error}")); } } + + if rollback_errors.is_empty() { + return Err(commit_error.into()); + } + + return Err(SurfpoolError::internal(format!( + "Failed to commit scheduled overrides ({commit_error}); rollback also failed for {}", + rollback_errors.join(", ") + ))); } Ok(()) @@ -4776,13 +4788,12 @@ impl SurfnetSvm { Supply the property-reference values used by the PDA, or pass the resolved pubkey." )) })?; - let slots = self.scheduled_overrides.keys()?; + let original_queue: BTreeMap> = + self.scheduled_overrides.into_iter()?.collect(); + let mut staged_queue = original_queue.clone(); let mut removed = 0; - for slot in slots { - let Some(mut queued) = self.scheduled_overrides.get(&slot)? else { - continue; - }; + for queued in staged_queue.values_mut() { let before = queued.len(); queued.retain(|other| { !(other.re_armed @@ -4791,16 +4802,10 @@ impl SurfnetSvm { && other.account.resolve(Some(&other.values)) == Some(target_account)) }); removed += before - queued.len(); - - if queued.len() != before { - if queued.is_empty() { - self.scheduled_overrides.take(&slot)?; - } else { - self.scheduled_overrides.store(slot, queued)?; - } - } } + staged_queue.retain(|_, queued| !queued.is_empty()); + self.commit_scheduled_override_plan(&original_queue, &staged_queue)?; Ok(removed) } } @@ -9316,6 +9321,127 @@ mod tests { ); } + #[test] + fn test_scheduled_override_queue_rolls_back_after_a_storage_failure() { + use crate::storage::{Storage, StorageError, StorageResult}; + + #[derive(Clone)] + struct FailOnceOnMutation { + inner: BTreeMap>, + mutations: usize, + fail_on: usize, + } + + impl Storage> for FailOnceOnMutation { + fn store(&mut self, key: Slot, value: Vec) -> StorageResult<()> { + self.mutations += 1; + self.inner.insert(key, value); + if self.mutations == self.fail_on { + Err(StorageError::SqliteNotEnabled) + } else { + Ok(()) + } + } + + fn clear(&mut self) -> StorageResult<()> { + self.inner.clear(); + Ok(()) + } + + fn get(&self, key: &Slot) -> StorageResult>> { + Ok(self.inner.get(key).cloned()) + } + + fn take(&mut self, key: &Slot) -> StorageResult>> { + self.mutations += 1; + let removed = self.inner.remove(key); + if self.mutations == self.fail_on { + Err(StorageError::SqliteNotEnabled) + } else { + Ok(removed) + } + } + + fn keys(&self) -> StorageResult> { + Ok(self.inner.keys().copied().collect()) + } + + fn into_iter( + &self, + ) -> StorageResult)> + '_>> + { + Ok(Box::new(self.inner.clone().into_iter())) + } + + fn count(&self) -> StorageResult { + Ok(self.inner.len() as u64) + } + + fn clone_box(&self) -> Box>> { + Box::new(self.clone()) + } + } + + const SLOT: Slot = 500; + let (mut svm, _account_pubkey, instance) = scheduled_persist_fixture(true); + let mut continuation = instance.clone(); + continuation.re_armed = true; + let original = BTreeMap::from([ + (SLOT + 10, vec![continuation.clone()]), + (SLOT + 11, vec![continuation]), + ]); + svm.scheduled_overrides = Box::new(FailOnceOnMutation { + inner: original.clone(), + mutations: 0, + // The authored bucket is stored, the first continuation is deleted, then this failure + // occurs after deleting the second continuation. Rollback must restore all three. + fail_on: 3, + }); + + let scenario = surfpool_types::Scenario { + id: "atomic-registration".to_string(), + name: "atomic registration".to_string(), + description: String::new(), + tags: vec![], + overrides: vec![instance.clone()], + }; + svm.register_scenario(scenario, Some(SLOT)) + .expect_err("the injected third mutation must fail registration"); + + let after: BTreeMap> = svm + .scheduled_overrides + .into_iter() + .expect("read queue after rollback") + .collect(); + assert_eq!( + after, original, + "a failed registration must restore every added and removed bucket" + ); + + svm.scheduled_overrides = Box::new(FailOnceOnMutation { + inner: original.clone(), + mutations: 0, + // Both continuations are deleted before the second deletion reports failure. + fail_on: 2, + }); + svm.stop_persisting_override( + &instance.id, + &instance.account, + &instance.template_id, + Some(&instance.values), + ) + .expect_err("the injected second mutation must fail cancellation"); + let after: BTreeMap> = svm + .scheduled_overrides + .into_iter() + .expect("read queue after cancellation rollback") + .collect(); + assert_eq!( + after, original, + "a failed cancellation must restore every removed continuation" + ); + } + #[tokio::test] async fn test_failed_read_does_not_overwrite_the_next_slots_overrides() { use crate::storage::{Storage, StorageError, StorageResult}; From 4084d4b94a2ce7798c034f4e7d65dbd0bc81b1cd Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Fri, 4 Sep 2026 10:08:49 +0300 Subject: [PATCH 16/18] fix(scenarios): atomically claim overdue override buckets --- crates/core/src/surfnet/svm.rs | 192 +++++++++++++++++++++------------ 1 file changed, 124 insertions(+), 68 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index 3d3490f3..fd58a773 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -2865,17 +2865,23 @@ impl SurfnetSvm { remote_ctx: &Option<(SurfnetRemoteClient, CommitmentConfig)>, target_slot: Slot, ) -> SurfpoolResult<()> { - let mut due: Vec = self - .scheduled_overrides - .keys()? - .into_iter() + let original_queue: BTreeMap> = + self.scheduled_overrides.into_iter()?.collect(); + let mut due: Vec = original_queue + .keys() + .copied() .filter(|slot| *slot <= target_slot) .collect(); due.sort_unstable(); + if due.is_empty() { + return Ok(()); + } + + let mut staged_queue = original_queue.clone(); let mut overrides = Vec::new(); for slot in due { - if let Some(queued) = self.scheduled_overrides.take(&slot)? { + if let Some(queued) = staged_queue.remove(&slot) { if slot != target_slot && !queued.is_empty() { debug!( "Materializing {} override(s) left behind at slot {} at slot {}", @@ -2887,6 +2893,7 @@ impl SurfnetSvm { overrides.extend(queued); } } + self.commit_scheduled_override_plan(&original_queue, &staged_queue)?; if overrides.is_empty() { return Ok(()); @@ -2918,8 +2925,8 @@ impl SurfnetSvm { let mut settled_this_slot: HashSet = HashSet::new(); - // `take` already emptied the slot, so bailing out mid-loop would drop every override that - // has not been reached yet. Put the unprocessed tail back before returning the error. + // The atomic claim already emptied every due slot, so bailing out mid-loop would drop every + // override that has not been reached yet. Put the unprocessed tail back before returning. let restore_unprocessed = |svm: &mut Self, from: usize| { if let Err(e) = svm .scheduled_overrides @@ -7930,6 +7937,71 @@ mod tests { (surfnet_svm, account_pubkey, instance) } + #[derive(Clone)] + struct FailOnceOnScheduledMutation { + inner: BTreeMap>, + mutations: usize, + fail_on: usize, + } + + impl crate::storage::Storage> for FailOnceOnScheduledMutation { + fn store( + &mut self, + key: Slot, + value: Vec, + ) -> crate::storage::StorageResult<()> { + self.mutations += 1; + self.inner.insert(key, value); + if self.mutations == self.fail_on { + Err(crate::storage::StorageError::SqliteNotEnabled) + } else { + Ok(()) + } + } + + fn clear(&mut self) -> crate::storage::StorageResult<()> { + self.inner.clear(); + Ok(()) + } + + fn get(&self, key: &Slot) -> crate::storage::StorageResult>> { + Ok(self.inner.get(key).cloned()) + } + + fn take( + &mut self, + key: &Slot, + ) -> crate::storage::StorageResult>> { + self.mutations += 1; + let removed = self.inner.remove(key); + if self.mutations == self.fail_on { + Err(crate::storage::StorageError::SqliteNotEnabled) + } else { + Ok(removed) + } + } + + fn keys(&self) -> crate::storage::StorageResult> { + Ok(self.inner.keys().copied().collect()) + } + + fn into_iter( + &self, + ) -> crate::storage::StorageResult< + Box)> + '_>, + > { + Ok(Box::new(self.inner.clone().into_iter())) + } + + fn count(&self) -> crate::storage::StorageResult { + Ok(self.inner.len() as u64) + } + + fn clone_box(&self) -> Box>> { + Box::new(self.clone()) + } + } + #[tokio::test] async fn test_persisted_override_is_rescheduled_for_the_next_slot() { const SLOT: u64 = 500; @@ -9323,65 +9395,6 @@ mod tests { #[test] fn test_scheduled_override_queue_rolls_back_after_a_storage_failure() { - use crate::storage::{Storage, StorageError, StorageResult}; - - #[derive(Clone)] - struct FailOnceOnMutation { - inner: BTreeMap>, - mutations: usize, - fail_on: usize, - } - - impl Storage> for FailOnceOnMutation { - fn store(&mut self, key: Slot, value: Vec) -> StorageResult<()> { - self.mutations += 1; - self.inner.insert(key, value); - if self.mutations == self.fail_on { - Err(StorageError::SqliteNotEnabled) - } else { - Ok(()) - } - } - - fn clear(&mut self) -> StorageResult<()> { - self.inner.clear(); - Ok(()) - } - - fn get(&self, key: &Slot) -> StorageResult>> { - Ok(self.inner.get(key).cloned()) - } - - fn take(&mut self, key: &Slot) -> StorageResult>> { - self.mutations += 1; - let removed = self.inner.remove(key); - if self.mutations == self.fail_on { - Err(StorageError::SqliteNotEnabled) - } else { - Ok(removed) - } - } - - fn keys(&self) -> StorageResult> { - Ok(self.inner.keys().copied().collect()) - } - - fn into_iter( - &self, - ) -> StorageResult)> + '_>> - { - Ok(Box::new(self.inner.clone().into_iter())) - } - - fn count(&self) -> StorageResult { - Ok(self.inner.len() as u64) - } - - fn clone_box(&self) -> Box>> { - Box::new(self.clone()) - } - } - const SLOT: Slot = 500; let (mut svm, _account_pubkey, instance) = scheduled_persist_fixture(true); let mut continuation = instance.clone(); @@ -9390,7 +9403,7 @@ mod tests { (SLOT + 10, vec![continuation.clone()]), (SLOT + 11, vec![continuation]), ]); - svm.scheduled_overrides = Box::new(FailOnceOnMutation { + svm.scheduled_overrides = Box::new(FailOnceOnScheduledMutation { inner: original.clone(), mutations: 0, // The authored bucket is stored, the first continuation is deleted, then this failure @@ -9418,7 +9431,7 @@ mod tests { "a failed registration must restore every added and removed bucket" ); - svm.scheduled_overrides = Box::new(FailOnceOnMutation { + svm.scheduled_overrides = Box::new(FailOnceOnScheduledMutation { inner: original.clone(), mutations: 0, // Both continuations are deleted before the second deletion reports failure. @@ -9442,6 +9455,49 @@ mod tests { ); } + #[tokio::test] + async fn test_overdue_bucket_claim_rolls_back_after_a_storage_failure() { + const FIRST_SLOT: Slot = 500; + const SECOND_SLOT: Slot = 501; + const TARGET_SLOT: Slot = 510; + + let (mut svm, account_pubkey, instance) = scheduled_persist_fixture(true); + let mut first = instance.clone(); + first.re_armed = true; + let mut second = first.clone(); + second.id = "second-persisted-override".to_string(); + let original = BTreeMap::from([(FIRST_SLOT, vec![first]), (SECOND_SLOT, vec![second])]); + svm.scheduled_overrides = Box::new(FailOnceOnScheduledMutation { + inner: original.clone(), + mutations: 0, + // Both due buckets are removed before the second removal reports failure. + fail_on: 2, + }); + + svm.materialize_overrides_for_slot(&None, TARGET_SLOT) + .await + .expect_err("the injected second removal must fail the atomic claim"); + + let after: BTreeMap> = svm + .scheduled_overrides + .into_iter() + .expect("read queue after claim rollback") + .collect(); + assert_eq!( + after, original, + "a failed overdue claim must restore every due bucket" + ); + let account = svm + .get_account(&account_pubkey) + .expect("read fixture account") + .expect("fixture account exists"); + assert_eq!( + &account.data[UNHEALTHY_OFFSET..UNHEALTHY_OFFSET + 16], + &[0; 16], + "no claimed override may be materialized after an atomic claim failure" + ); + } + #[tokio::test] async fn test_failed_read_does_not_overwrite_the_next_slots_overrides() { use crate::storage::{Storage, StorageError, StorageResult}; From 36d76604aeda2adc129d14aad981d254c062925e Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Fri, 4 Sep 2026 10:25:37 +0300 Subject: [PATCH 17/18] fix(scenarios): re-arm persistent overrides only after application --- crates/core/src/surfnet/svm.rs | 166 ++++++++++++++---- .../kit/generated/OverrideInstance.ts | 4 +- 2 files changed, 134 insertions(+), 36 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index fd58a773..63f39449 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -2928,10 +2928,17 @@ impl SurfnetSvm { // The atomic claim already emptied every due slot, so bailing out mid-loop would drop every // override that has not been reached yet. Put the unprocessed tail back before returning. let restore_unprocessed = |svm: &mut Self, from: usize| { - if let Err(e) = svm - .scheduled_overrides - .store(target_slot, overrides[from..].to_vec()) - { + let restore = (|| -> SurfpoolResult<()> { + let original_queue: BTreeMap> = + svm.scheduled_overrides.into_iter()?.collect(); + let mut staged_queue = original_queue.clone(); + staged_queue + .entry(target_slot) + .or_default() + .extend_from_slice(&overrides[from..]); + svm.commit_scheduled_override_plan(&original_queue, &staged_queue) + })(); + if let Err(e) = restore { error!( "Failed to restore {} unprocessed override(s) for slot {}: {}", overrides.len() - from, @@ -3100,19 +3107,12 @@ impl SurfnetSvm { later.enabled && !later.re_armed && same_override_target(later, override_instance) }); - if override_instance.persist.is_enabled() && !superseded_later { - let mut requeued = override_instance.clone(); - if requeued.fetch_before_use && fetch_retired { - requeued.fetch_before_use = false; - } - if let Err(e) = self.reschedule_override_for_next_slot(&requeued, target_slot) { - restore_unprocessed(self, index); - return Err(e); + // Apply the override values to the account data + 'apply_override: { + if override_instance.values.is_empty() { + break 'apply_override; } - } - // Apply the override values to the account data - if !override_instance.values.is_empty() { // Filter out values that are only used for PDA derivation (not account data) let pda_refs = override_instance.account.get_pda_seed_references(); let account_values: HashMap = override_instance @@ -3127,7 +3127,7 @@ impl SurfnetSvm { "Override {} has no account data modifications (all values are PDA seeds)", override_instance.id ); - continue; + break 'apply_override; } debug!( @@ -3144,7 +3144,7 @@ impl SurfnetSvm { "Account {} not found in SVM for override {}, skipping modifications", account_pubkey, override_instance.id ); - continue; + break 'apply_override; }; // Programs with no usable IDL carry a byte layout instead, and this MUST come @@ -3188,14 +3188,23 @@ impl SurfnetSvm { override_instance.id, account_pubkey, e ), } - continue; + break 'apply_override; } // Mints fail the token unpack and keep flowing through the IDL path. if is_supported_token_program(account.owner()) { if let Ok(token_account) = TokenAccount::unpack(account.data()) { - let new_account_data = - forge_token_account_data(&account, token_account, &account_values)?; + let new_account_data = match forge_token_account_data( + &account, + token_account, + &account_values, + ) { + Ok(data) => data, + Err(e) => { + restore_unprocessed(self, index); + return Err(e); + } + }; let modified_account = Account { lamports: account.lamports(), data: new_account_data, @@ -3203,8 +3212,11 @@ impl SurfnetSvm { executable: account.executable(), rent_epoch: account.rent_epoch(), }; - self.inner.set_account(account_pubkey, modified_account)?; - continue; + if let Err(e) = self.inner.set_account(account_pubkey, modified_account) { + restore_unprocessed(self, index); + return Err(e); + } + break 'apply_override; } } @@ -3219,14 +3231,14 @@ impl SurfnetSvm { "No IDL registered for program {} (owner of account {}), skipping override {}", owner_program_id, account_pubkey, override_instance.id ); - continue; + break 'apply_override; } Err(e) => { warn!( "Failed to get IDL for program {}: {}, skipping override {}", owner_program_id, e, override_instance.id ); - continue; + break 'apply_override; } }; @@ -3236,7 +3248,7 @@ impl SurfnetSvm { "IDL versions empty for program {}, skipping override {}", owner_program_id, override_instance.id ); - continue; + break 'apply_override; }; let idl = &versioned_idl.1; @@ -3253,7 +3265,7 @@ impl SurfnetSvm { account_data.len(), override_instance.id ); - continue; + break 'apply_override; } // Use get_forged_account_data to apply the overrides (with PDA refs filtered out) @@ -3270,7 +3282,7 @@ impl SurfnetSvm { If the account doesn't exist locally, enable fetchBeforeUse: true.", account_pubkey, override_instance.id, e ); - continue; + break 'apply_override; } }; @@ -3299,6 +3311,20 @@ impl SurfnetSvm { settled_this_slot.insert(account_pubkey); } } + + // Only queue the continuation after this application attempt has completed. A hard + // failure above restores the current and remaining entries without leaving a future + // continuation for work that never applied. + if override_instance.persist.is_enabled() && !superseded_later { + let mut requeued = override_instance.clone(); + if requeued.fetch_before_use && fetch_retired { + requeued.fetch_before_use = false; + } + if let Err(e) = self.reschedule_override_for_next_slot(&requeued, target_slot) { + restore_unprocessed(self, index); + return Err(e); + } + } } Ok(()) @@ -3330,10 +3356,10 @@ impl SurfnetSvm { instance.re_armed = true; let instance = &instance; - let mut next = self - .scheduled_overrides - .get(&next_slot)? - .unwrap_or_default(); + let original_queue: BTreeMap> = + self.scheduled_overrides.into_iter()?.collect(); + let mut staged_queue = original_queue.clone(); + let next = staged_queue.entry(next_slot).or_default(); let same_override = |queued: &OverrideInstance| same_override_target(queued, instance); @@ -3359,8 +3385,7 @@ impl SurfnetSvm { } None => next.push(instance.clone()), } - self.scheduled_overrides.store(next_slot, next)?; - Ok(()) + self.commit_scheduled_override_plan(&original_queue, &staged_queue) } /// Forges account data by applying overrides to existing account data @@ -9498,6 +9523,79 @@ mod tests { ); } + #[tokio::test] + async fn test_persisted_token_forge_failure_restores_current_without_continuation() { + const SLOT: Slot = 500; + + let (mut svm, _events_rx, _geyser_rx) = SurfnetSvm::default(); + let account_pubkey = Pubkey::new_unique(); + let (_token_account, account) = token_2022_vault_with_tail(Pubkey::new_unique()); + svm.inner + .set_account(account_pubkey, account) + .expect("set token account"); + + let mut instance = OverrideInstance::new( + "spl-token-account-balance".to_string(), + 0, + surfpool_types::AccountAddress::Pubkey(account_pubkey.to_string()), + ) + .with_values(HashMap::from([( + "amount".to_string(), + serde_json::json!("not-a-u64"), + )])); + instance.persist = surfpool_types::Persist::Always(true); + svm.scheduled_overrides + .store(SLOT, vec![instance.clone()]) + .expect("schedule persisted token override"); + + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect_err("invalid token amount must fail materialization"); + + assert_eq!( + svm.scheduled_overrides.get(&SLOT).expect("read current"), + Some(vec![instance]), + "the failed current entry must be restored" + ); + assert!( + svm.scheduled_overrides + .get(&(SLOT + 1)) + .expect("read continuation") + .is_none(), + "a failed application must not leave a continuation" + ); + } + + #[tokio::test] + async fn test_persist_rearm_failure_restores_current_without_continuation() { + const SLOT: Slot = 500; + + let (mut svm, _account_pubkey, instance) = scheduled_persist_fixture(true); + svm.scheduled_overrides = Box::new(FailOnceOnScheduledMutation { + inner: BTreeMap::from([(SLOT, vec![instance.clone()])]), + mutations: 0, + // Claiming the current slot succeeds. Storing its continuation mutates and then fails. + fail_on: 2, + }); + + svm.materialize_overrides_for_slot(&None, SLOT) + .await + .expect_err("the injected continuation write must fail"); + + assert_eq!( + svm.scheduled_overrides.get(&SLOT).expect("read current"), + Some(vec![instance]), + "the current entry must be restored after a failed re-arm" + ); + assert!( + svm.scheduled_overrides + .get(&(SLOT + 1)) + .expect("read continuation") + .is_none(), + "a failed re-arm must roll back a continuation even if storage mutated before erroring" + ); + } + #[tokio::test] async fn test_failed_read_does_not_overwrite_the_next_slots_overrides() { use crate::storage::{Storage, StorageError, StorageResult}; @@ -9543,7 +9641,7 @@ mod tests { ) -> StorageResult< Box)> + '_>, > { - Ok(Box::new(self.inner.iter().map(|(k, v)| (*k, v.clone())))) + Err(StorageError::SqliteNotEnabled) } fn count(&self) -> StorageResult { Ok(self.inner.len() as u64) diff --git a/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts b/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts index 8f47f1aa..49799c87 100644 --- a/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts +++ b/crates/sdk-node/surfpool-sdk/kit/generated/OverrideInstance.ts @@ -34,12 +34,12 @@ enabled: boolean, /** * Whether to fetch fresh account data just before transaction execution */ -fetchBeforeUse?: boolean, +fetchBeforeUse?: boolean, /** * How long to keep re-applying this override: `false` applies it once, `true` re-applies it * on every following slot, and `{ slots: N }` applies it N times in total, counting the first */ -persist?: boolean | { slots: number | bigint }, +persist?: boolean | { slots: number | bigint }, /** * Account address to override - use pubkey for known addresses or pda for derived addresses */ From 538055765fc3862a2d6b563e739d087da1e20c40 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Fri, 4 Sep 2026 10:40:17 +0300 Subject: [PATCH 18/18] fix(scenarios): preserve bounded persistence across failed writes --- crates/core/src/surfnet/svm.rs | 233 +++++++++++++++++++++++++++++---- 1 file changed, 204 insertions(+), 29 deletions(-) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index 63f39449..80d3fec3 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -2954,6 +2954,12 @@ impl SurfnetSvm { continue; } + // A later authored entry replaces this value regardless of whether the current account + // can be resolved or written. Do not let a retry outlive that explicit transition. + let superseded_later = overrides[index + 1..].iter().any(|later| { + later.enabled && !later.re_armed && same_override_target(later, override_instance) + }); + // Resolve account address using the centralized method let account_pubkey = match override_instance .account @@ -2976,6 +2982,16 @@ impl SurfnetSvm { "Failed to resolve account address for override {}", override_instance.id ); + if override_instance.persist.is_enabled() && !superseded_later { + if let Err(e) = self.reschedule_override_for_next_slot( + override_instance, + target_slot, + false, + ) { + restore_unprocessed(self, index); + return Err(e); + } + } continue; } }; @@ -3099,17 +3115,16 @@ impl SurfnetSvm { FetchOutcome::NoRemote | FetchOutcome::NotOnRemote => existing_account.is_some(), }; - // A later entry in this batch that the operator scheduled supersedes this one. The - // scheduler's own collision check cannot see it: claiming several overdue buckets at - // once already took it out of its slot, so re-arming here would outlive it and restore - // the superseded value on the following slot. - let superseded_later = overrides[index + 1..].iter().any(|later| { - later.enabled && !later.re_armed && same_override_target(later, override_instance) - }); + // A bounded window counts successful applications, not scheduler attempts. No-value + // and PDA-seed-only overrides are successful no-ops only when their target exists; + // every path that expected an account write but could not complete it marks the attempt + // for a non-consuming retry. + let mut application_succeeded = true; // Apply the override values to the account data 'apply_override: { if override_instance.values.is_empty() { + application_succeeded = existing_account.is_some(); break 'apply_override; } @@ -3127,6 +3142,7 @@ impl SurfnetSvm { "Override {} has no account data modifications (all values are PDA seeds)", override_instance.id ); + application_succeeded = existing_account.is_some(); break 'apply_override; } @@ -3144,6 +3160,7 @@ impl SurfnetSvm { "Account {} not found in SVM for override {}, skipping modifications", account_pubkey, override_instance.id ); + application_succeeded = false; break 'apply_override; }; @@ -3173,6 +3190,7 @@ impl SurfnetSvm { }; if let Err(e) = self.inner.set_account(account_pubkey, modified) { warn!("Failed to set raw-layout account {}: {}", account_pubkey, e); + application_succeeded = false; } else { debug!( "Raw-layout override {} applied {} field(s) to {}", @@ -3183,10 +3201,13 @@ impl SurfnetSvm { settled_this_slot.insert(account_pubkey); } } - Err(e) => warn!( - "Raw-layout override {} failed on {}: {}", - override_instance.id, account_pubkey, e - ), + Err(e) => { + warn!( + "Raw-layout override {} failed on {}: {}", + override_instance.id, account_pubkey, e + ); + application_succeeded = false; + } } break 'apply_override; } @@ -3231,6 +3252,7 @@ impl SurfnetSvm { "No IDL registered for program {} (owner of account {}), skipping override {}", owner_program_id, account_pubkey, override_instance.id ); + application_succeeded = false; break 'apply_override; } Err(e) => { @@ -3238,6 +3260,7 @@ impl SurfnetSvm { "Failed to get IDL for program {}: {}, skipping override {}", owner_program_id, e, override_instance.id ); + application_succeeded = false; break 'apply_override; } }; @@ -3248,6 +3271,7 @@ impl SurfnetSvm { "IDL versions empty for program {}, skipping override {}", owner_program_id, override_instance.id ); + application_succeeded = false; break 'apply_override; }; @@ -3265,6 +3289,7 @@ impl SurfnetSvm { account_data.len(), override_instance.id ); + application_succeeded = false; break 'apply_override; } @@ -3282,6 +3307,7 @@ impl SurfnetSvm { If the account doesn't exist locally, enable fetchBeforeUse: true.", account_pubkey, override_instance.id, e ); + application_succeeded = false; break 'apply_override; } }; @@ -3301,6 +3327,7 @@ impl SurfnetSvm { "Failed to set modified account {} in SVM: {}", account_pubkey, e ); + application_succeeded = false; } else { debug!( "Successfully applied {} override(s) to account {} (override {})", @@ -3320,7 +3347,11 @@ impl SurfnetSvm { if requeued.fetch_before_use && fetch_retired { requeued.fetch_before_use = false; } - if let Err(e) = self.reschedule_override_for_next_slot(&requeued, target_slot) { + if let Err(e) = self.reschedule_override_for_next_slot( + &requeued, + target_slot, + application_succeeded, + ) { restore_unprocessed(self, index); return Err(e); } @@ -3336,21 +3367,32 @@ impl SurfnetSvm { &mut self, instance: &OverrideInstance, target_slot: Slot, + application_succeeded: bool, ) -> SurfpoolResult<()> { - let next_slot = target_slot.checked_add(1).ok_or_else(|| { - SurfpoolError::internal(format!( - "Override {} cannot persist past slot {}: there is no next slot", - instance.id, target_slot - )) - })?; - - let Some(next_persist) = instance.persist.next_arming() else { + let next_persist = match (&instance.persist, application_succeeded) { + (surfpool_types::Persist::Always(false), _) => None, + (surfpool_types::Persist::Always(true), _) => { + Some(surfpool_types::Persist::Always(true)) + } + (surfpool_types::Persist::Slots { slots: 0 }, _) => None, + (surfpool_types::Persist::Slots { .. }, true) => instance.persist.next_arming(), + (surfpool_types::Persist::Slots { slots }, false) => { + Some(surfpool_types::Persist::Slots { slots: *slots }) + } + }; + let Some(next_persist) = next_persist else { debug!( "Override {} has reached the end of its persist window at slot {}", instance.id, target_slot ); return Ok(()); }; + let next_slot = target_slot.checked_add(1).ok_or_else(|| { + SurfpoolError::internal(format!( + "Override {} cannot persist past slot {}: there is no next slot", + instance.id, target_slot + )) + })?; let mut instance = instance.clone(); instance.persist = next_persist; instance.re_armed = true; @@ -8307,11 +8349,21 @@ mod tests { let (mut svm, account_pubkey, instance) = scheduled_persist_fixture(true); assert!( - svm.reschedule_override_for_next_slot(&instance, u64::MAX) + svm.reschedule_override_for_next_slot(&instance, u64::MAX, true) .is_err(), "there is no slot after u64::MAX" ); + let mut final_bounded = instance.clone(); + final_bounded.persist = surfpool_types::Persist::Slots { slots: 1 }; + svm.reschedule_override_for_next_slot(&final_bounded, u64::MAX, true) + .expect("a successful final application needs no slot after u64::MAX"); + assert!( + svm.reschedule_override_for_next_slot(&final_bounded, u64::MAX, false) + .is_err(), + "a failed final application still needs a retry slot and must report overflow" + ); + let mut far = surfpool_types::OverrideInstance::new( "kamino-obligation-health".to_string(), 10, @@ -8445,10 +8497,10 @@ mod tests { second.account = surfpool_types::AccountAddress::Pubkey(second_account.to_string()); surfnet_svm - .reschedule_override_for_next_slot(&first, SLOT) + .reschedule_override_for_next_slot(&first, SLOT, true) .expect("reschedule"); surfnet_svm - .reschedule_override_for_next_slot(&second, SLOT) + .reschedule_override_for_next_slot(&second, SLOT, true) .expect("reschedule"); let queued = surfnet_svm @@ -8464,7 +8516,7 @@ mod tests { ); surfnet_svm - .reschedule_override_for_next_slot(&first, SLOT) + .reschedule_override_for_next_slot(&first, SLOT, true) .expect("reschedule"); let queued = surfnet_svm .scheduled_overrides @@ -8633,9 +8685,9 @@ mod tests { instance.persist = surfpool_types::Persist::Slots { slots: 3 }; // Two reschedules for the same slot, exactly as the real flow does. - svm.reschedule_override_for_next_slot(&instance, SLOT) + svm.reschedule_override_for_next_slot(&instance, SLOT, true) .expect("reschedule"); - svm.reschedule_override_for_next_slot(&instance, SLOT) + svm.reschedule_override_for_next_slot(&instance, SLOT, true) .expect("reschedule"); let queued = svm @@ -8657,7 +8709,7 @@ mod tests { // The last slot of the window does not re-arm. let mut last = instance.clone(); last.persist = surfpool_types::Persist::Slots { slots: 1 }; - svm.reschedule_override_for_next_slot(&last, SLOT + 5) + svm.reschedule_override_for_next_slot(&last, SLOT + 5, true) .expect("reschedule"); assert!( svm.scheduled_overrides @@ -9543,7 +9595,7 @@ mod tests { "amount".to_string(), serde_json::json!("not-a-u64"), )])); - instance.persist = surfpool_types::Persist::Always(true); + instance.persist = surfpool_types::Persist::Slots { slots: 3 }; svm.scheduled_overrides .store(SLOT, vec![instance.clone()]) .expect("schedule persisted token override"); @@ -9566,6 +9618,129 @@ mod tests { ); } + #[tokio::test] + async fn test_bounded_persist_retries_skipped_writes_without_consuming_the_window() { + const SLOT: Slot = 500; + const WINDOW: Slot = 3; + + async fn assert_window_is_unchanged( + mut svm: SurfnetSvm, + mut instance: OverrideInstance, + slot: Slot, + case: &str, + ) { + instance.persist = surfpool_types::Persist::Slots { slots: WINDOW }; + svm.scheduled_overrides + .store(slot, vec![instance]) + .expect("schedule bounded override"); + svm.materialize_overrides_for_slot(&None, slot) + .await + .expect("a skipped write is a retryable materialization outcome"); + let next = svm + .scheduled_overrides + .get(&(slot + 1)) + .expect("read retry") + .expect("failed write must retry"); + assert_eq!(next.len(), 1, "{case}: exactly one retry must be queued"); + assert_eq!( + next[0].persist, + surfpool_types::Persist::Slots { slots: WINDOW }, + "{case}: a failed or skipped write must not consume the bounded window" + ); + } + + let (missing_svm, _account_pubkey, mut missing) = scheduled_persist_fixture(true); + missing.account = surfpool_types::AccountAddress::Pubkey(Pubkey::new_unique().to_string()); + assert_window_is_unchanged(missing_svm, missing, SLOT, "missing account").await; + + let (missing_noop_svm, _account_pubkey, mut missing_noop) = scheduled_persist_fixture(true); + missing_noop.account = + surfpool_types::AccountAddress::Pubkey(Pubkey::new_unique().to_string()); + missing_noop.values.clear(); + assert_window_is_unchanged( + missing_noop_svm, + missing_noop, + SLOT, + "missing account with no fields", + ) + .await; + + let (unresolved_svm, _account_pubkey, _instance) = scheduled_persist_fixture(true); + let unresolved = OverrideInstance::new( + "unresolved-pda".to_string(), + 0, + surfpool_types::AccountAddress::Pda { + program_id: Pubkey::new_unique().to_string(), + seeds: vec![surfpool_types::PdaSeed::PropertyRef( + "missing_seed".to_string(), + )], + }, + ); + assert_window_is_unchanged(unresolved_svm, unresolved, SLOT, "unresolved PDA").await; + + let (mut absent_idl_svm, _events_rx, _geyser_rx) = SurfnetSvm::default(); + let absent_idl_pubkey = Pubkey::new_unique(); + absent_idl_svm + .inner + .set_account( + absent_idl_pubkey, + Account { + lamports: 1_000_000, + data: vec![0; 16], + owner: Pubkey::new_unique(), + executable: false, + rent_epoch: 0, + }, + ) + .expect("set account with no registered IDL"); + let absent_idl = OverrideInstance::new( + "unknown-idl-template".to_string(), + 0, + surfpool_types::AccountAddress::Pubkey(absent_idl_pubkey.to_string()), + ) + .with_values(HashMap::from([("value".to_string(), serde_json::json!(1))])); + assert_window_is_unchanged(absent_idl_svm, absent_idl, SLOT, "absent IDL").await; + + let (forge_svm, _account_pubkey, mut forge_failure) = scheduled_persist_fixture(true); + forge_failure.values = HashMap::from([( + "field_that_does_not_exist".to_string(), + serde_json::json!(1), + )]); + assert_window_is_unchanged(forge_svm, forge_failure, SLOT, "IDL forge failure").await; + } + + #[test] + fn test_bounded_persist_transition_consumes_only_successful_applications() { + const SLOT: Slot = 500; + + let (mut svm, _account_pubkey, mut instance) = scheduled_persist_fixture(true); + instance.persist = surfpool_types::Persist::Slots { slots: 3 }; + svm.reschedule_override_for_next_slot(&instance, SLOT, false) + .expect("queue retry after failed write"); + assert_eq!( + svm.scheduled_overrides + .get(&(SLOT + 1)) + .expect("read retry") + .expect("retry queued")[0] + .persist, + surfpool_types::Persist::Slots { slots: 3 }, + "a failed write must retain all remaining successful applications" + ); + + svm.scheduled_overrides.clear().expect("clear retry"); + svm.reschedule_override_for_next_slot(&instance, SLOT, true) + .expect("queue continuation after successful write"); + assert_eq!( + svm.scheduled_overrides + .get(&(SLOT + 1)) + .expect("read continuation") + .expect("continuation queued")[0] + .persist, + surfpool_types::Persist::Slots { slots: 2 }, + "a successful write must consume exactly one application" + ); + } + #[tokio::test] async fn test_persist_rearm_failure_restores_current_without_continuation() { const SLOT: Slot = 500; @@ -9664,7 +9839,7 @@ mod tests { writes: writes.clone(), }); - svm.reschedule_override_for_next_slot(&instance, SLOT) + svm.reschedule_override_for_next_slot(&instance, SLOT, true) .expect_err("a failed read must surface as an error, not a silent skip"); assert_eq!(