From df5a374aa4a4a10726d3630fb3ea2ee675fd173b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 09:40:08 +0300 Subject: [PATCH 01/58] test(transport): cover shutdown deadline, cancellation, and backpressure paths Add tests for shutdown behaviour when the host callback blocks, when a started shutdown is cancelled, when an initialized module is stopped before becoming ready, and when a stop wakes a backpressured delivery. A drop guard releases the blocked callback so a failed assertion cannot hang later tests. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 114 +++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index cee6a353..9cfff836 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -591,3 +591,117 @@ async fn ready_followed_by_fault_is_not_successful_readiness() { assert!(transport.is_faulted()); assert!(!transport.is_ready()); } + +// Release opaque callbacks even when a regression assertion fails. +struct ShutdownRelease; +impl Drop for ShutdownRelease { + fn drop(&mut self) { + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + } +} + +async fn shutdown_entry() { + tokio::time::timeout(Duration::from_secs(2), async { + while !SHUTDOWN_ENTERED.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }).await.expect("S00"); +} + +#[tokio::test] +async fn ordinary_shutdown_deadline_preserves_state_until_callback_returns() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("S01"); + unsafe { (host.ready)(host.host_ctx); } + let stopping = tokio::spawn(transport.clone().stop(Duration::from_millis(100))); + shutdown_entry().await; + assert!(matches!(stopping.await.expect("S02"), Err(StopError::Started(_))), "S03"); + assert!(!transport.is_faulted(), "S04"); + assert!(transport.is_ready(), "S05"); + assert!(matches!(transport.clone().stop(Duration::from_millis(10)).await, Err(StopError::NotStarted(_))), "S06"); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "S07"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()).await.expect("S08"); + assert!(!transport.is_faulted(), "S09"); + unsafe { (host.ready)(host.host_ctx); } + assert!(!transport.is_ready(), "S10"); + assert!(matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), "S11"); + assert!(!transport.init_failed(), "S12"); +} + +#[tokio::test] +async fn cancelling_a_started_shutdown_keeps_physical_exclusion_without_faulting() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("cancel-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("S13"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + shutdown_entry().await; + stopping.abort(); + assert!(stopping.await.expect_err("S14").is_cancelled(), "S15"); + assert!(!transport.is_faulted(), "S16"); + assert!(matches!(transport.clone().stop(Duration::from_millis(10)).await, Err(StopError::NotStarted(_))), "S17"); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "S18"); + unsafe { (host.fault)(host.host_ctx, std::ptr::null(), 0); } + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()).await.expect("S19"); + unsafe { (host.ready)(host.host_ctx); } + assert!(transport.is_faulted(), "S20"); + assert!(!transport.is_ready(), "S21"); +} + +#[tokio::test] +async fn stopping_initialized_module_before_ready_releases_observers_and_queued_calls() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let (transport, host) = ModuleTransport::new("queued-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.send(call()).await.expect("S22"); + assert!(transport.init_started(), "S23"); + transport.stop_sync(Duration::ZERO); + tokio::time::timeout(Duration::from_secs(2), transport.wait_ready()).await.expect("S24"); + tokio::time::timeout(Duration::from_secs(2), async { + while transport.drain_started.load(Ordering::Acquire) { tokio::task::yield_now().await; } + }).await.expect("S25"); + assert!(!transport.is_faulted(), "S26"); + assert!(!transport.is_ready(), "S27"); + assert!(matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), "S28"); + assert!(!transport.init_failed(), "S29"); + let weak = Arc::downgrade(&transport); + drop(transport); + assert!(weak.upgrade().is_none(), "S30"); +} + +#[tokio::test] +async fn ordinary_stop_wakes_backpressure_without_turning_closure_into_a_fault() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + DELIVERY_CODE.store(TB_BACKPRESSURE, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("pressure-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.send(call()).await.expect("S31"); + unsafe { (host.ready)(host.host_ctx); } + tokio::time::timeout(Duration::from_secs(2), async { + while DELIVERIES.load(Ordering::Acquire) == 0 { tokio::task::yield_now().await; } + }).await.expect("S32"); + transport.stop_sync(Duration::ZERO); + // An explicit legacy wake also must not cause delivery to re-enter disposed code. + unsafe { (host.wake)(host.host_ctx); } + tokio::time::timeout(Duration::from_secs(2), async { + while transport.drain_started.load(Ordering::Acquire) { tokio::task::yield_now().await; } + }).await.expect("S33"); + assert!(!transport.is_faulted(), "S34"); + assert_eq!(DELIVERIES.load(Ordering::Acquire), 1, "S35"); + DELIVERY_CODE.store(TB_OK, Ordering::Release); +} From ca41ae11746b3680efccdbda57146520d3f3990a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 09:41:33 +0300 Subject: [PATCH 02/58] fix(transport): close module admission on normal stop Stopping a module now marks the transport as stopped under the pending queue lock, so sends, initialization, and the drainer all observe closure without being reported as a module fault. Queued messages are dropped and observers are woken once the opaque shutdown callback returns, keeping a caller timeout from being mistaken for physical completion. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 124 +++++++++++++++---- crates/tinybus/src/module/transport_tests.rs | 115 +++++++++++++---- 2 files changed, 190 insertions(+), 49 deletions(-) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 35097b42..9116ed3f 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -33,6 +33,7 @@ struct HostContext { config: StdMutex>, faulted: AtomicBool, init_failed: AtomicBool, + stopped: AtomicBool, init_started: AtomicBool, init_notify: Notify, ready: AtomicBool, @@ -50,7 +51,7 @@ pub(crate) struct ModuleTransport { label: String, initializer: StdMutex>, init_result: OnceCell>, - pending: Mutex>, + pending: StdMutex>, drain_started: AtomicBool, // Reconfiguration and shutdown both call into opaque module code. The // owned guard moves into the blocking task, so a host-side timeout cannot @@ -110,6 +111,7 @@ impl ModuleTransport { config: StdMutex::new(config), faulted: AtomicBool::new(false), init_failed: AtomicBool::new(false), + stopped: AtomicBool::new(false), init_started: AtomicBool::new(false), init_notify: Notify::new(), ready: AtomicBool::new(false), @@ -125,7 +127,7 @@ impl ModuleTransport { label, initializer: StdMutex::new(None), init_result: OnceCell::new(), - pending: Mutex::new(VecDeque::new()), + pending: StdMutex::new(VecDeque::new()), drain_started: AtomicBool::new(false), lifecycle: Arc::new(Mutex::new(())), stop_task_spawned: Arc::new(AtomicBool::new(false)), @@ -217,6 +219,25 @@ impl ModuleTransport { self.initialization_gate_changed.notify_waiters(); } + // Normal completion closes admission without inventing a module fault. The + // queue lock makes closure atomic with send admission and drainer retirement; + // no queued body survives disposal or starts a replacement drainer. + fn finish_stop(&self) { + { + let mut pending = self.pending.lock().expect("module pending lock"); + self.context.stopped.store(true, Ordering::Release); + pending.clear(); + } + self.context.init_notify.notify_waiters(); + self.context.ready_notify.notify_waiters(); + self.context.wake.notify_waiters(); + self.initialization_gate_changed.notify_waiters(); + } + + fn is_stopped(&self) -> bool { + self.context.stopped.load(Ordering::Acquire) + } + pub(crate) async fn ensure_initialized(&self) -> Result<()> { loop { let changed = self.initialization_gate_changed.notified(); @@ -225,7 +246,7 @@ impl ModuleTransport { .initialization_gate .lock() .expect("initialization gate lock"); - if self.is_faulted() { + if self.is_faulted() || self.is_stopped() { return Err(Error::ConnectionClosed); } match allowed { @@ -241,7 +262,18 @@ impl ModuleTransport { let result = self .init_result .get_or_init(|| async { - self.context.init_started.store(true, Ordering::Release); + { + // Pair starting opaque initialization with the never-started + // stop decision. A stopped admission cannot start it later. + let _gate = self + .initialization_gate + .lock() + .expect("initialization gate lock"); + if self.is_stopped() { + return Err("module transport is closed".to_string()); + } + self.context.init_started.store(true, Ordering::Release); + } self.context.init_notify.notify_waiters(); let Some(initializer) = self .initializer @@ -287,6 +319,7 @@ impl ModuleTransport { let notified = self.context.ready_notify.notified(); if self.context.ready.load(Ordering::Acquire) || self.context.faulted.load(Ordering::Acquire) + || self.is_stopped() { return; } @@ -295,7 +328,7 @@ impl ModuleTransport { } pub(crate) fn is_ready(&self) -> bool { - self.context.ready.load(Ordering::Acquire) && !self.is_faulted() + self.context.ready.load(Ordering::Acquire) && !self.is_faulted() && !self.is_stopped() } pub(crate) fn clear_config(&self) { @@ -373,7 +406,7 @@ impl ModuleTransport { pub(crate) async fn wait_initializing(&self) { loop { let notified = self.context.init_notify.notified(); - if self.init_started() || self.is_faulted() { + if self.init_started() || self.is_faulted() || self.is_stopped() { return; } notified.await; @@ -381,7 +414,7 @@ impl ModuleTransport { } async fn deliver_now(&self, message: Message) -> Result<()> { - if self.context.faulted.load(Ordering::Acquire) { + if self.is_faulted() || self.is_stopped() { return Err(Error::ConnectionClosed); } let bytes = serde_json::to_vec(&message)?; @@ -391,6 +424,9 @@ impl ModuleTransport { loop { let notified = self.context.wake.notified(); + if self.is_faulted() || self.is_stopped() { + return Err(Error::ConnectionClosed); + } let module = *self.module.lock().expect("module vtable lock"); let Some(module) = module else { return Err(Error::ConnectionClosed); @@ -408,6 +444,10 @@ impl ModuleTransport { .await .is_err() { + let _pending = self.pending.lock().expect("module pending lock"); + if self.is_stopped() { + return Err(Error::ConnectionClosed); + } self.context.faulted.store(true, Ordering::Release); self.context.ready_notify.notify_waiters(); self.context @@ -430,17 +470,22 @@ impl ModuleTransport { async fn drain_pending(self: Arc) { self.wait_ready().await; loop { - while !self.is_faulted() { - let message = self.pending.lock().await.pop_front(); + while !self.is_faulted() && !self.is_stopped() { + let message = self + .pending + .lock() + .expect("module pending lock") + .pop_front(); let Some(message) = message else { break; }; if self.deliver_now(message.clone()).await.is_err() { self.fault_pending(message).await; + self.drain_started.store(false, Ordering::Release); return; } } - if self.is_faulted() { + if self.is_faulted() || self.is_stopped() { self.drain_started.store(false, Ordering::Release); return; } @@ -451,7 +496,7 @@ impl ModuleTransport { // genuinely empty; the lock makes the check-and-clear atomic with a // concurrent push. let restart = { - let pending = self.pending.lock().await; + let pending = self.pending.lock().expect("module pending lock"); let empty = pending.is_empty(); if empty { self.drain_started.store(false, Ordering::Release); @@ -470,7 +515,12 @@ impl ModuleTransport { /// learn the delivery failed instead of waiting out their deadline. async fn fault_pending(&self, failed: Message) { let cleared = { - let mut pending = self.pending.lock().await; + let mut pending = self.pending.lock().expect("module pending lock"); + if self.is_stopped() { + pending.clear(); + return; + } + self.context.faulted.store(true, Ordering::Release); std::mem::take(&mut *pending) }; // Each of these calls incremented `inflight` when it was queued. The @@ -517,8 +567,19 @@ impl ModuleTransport { } pub(crate) fn stop_sync(&self, deadline: Duration) -> i32 { - self.terminate_waiters(); + { + let _gate = self + .initialization_gate + .lock() + .expect("initialization gate lock"); + if !self.init_started() { + self.finish_stop(); + } + } let code = self.shutdown_sync(deadline); + // A caller timeout is not physical completion. Running initialization + // and readiness observers are closed only after the opaque callback returns. + self.finish_stop(); self.context .inbound .lock() @@ -576,9 +637,18 @@ impl From for Error { #[async_trait] impl Transport for ModuleTransport { async fn send(&self, message: Message) -> Result<()> { + if self.is_stopped() { + return Err(Error::ConnectionClosed); + } if self.ensure_initialized().await.is_err() { - self.context.init_failed.store(true, Ordering::Release); - self.context.faulted.store(true, Ordering::Release); + { + let _pending = self.pending.lock().expect("module pending lock"); + if self.is_stopped() { + return Err(Error::ConnectionClosed); + } + self.context.init_failed.store(true, Ordering::Release); + self.context.faulted.store(true, Ordering::Release); + } self.context.ready_notify.notify_waiters(); if message.header.kind == MessageKind::MethodCall { let error = Error::ModuleUnavailable { @@ -603,16 +673,22 @@ impl Transport for ModuleTransport { return Err(Error::ConnectionClosed); } let is_call = message.header.kind == MessageKind::MethodCall; - if is_call { - self.context.inflight.fetch_add(1, Ordering::AcqRel); - } - if message.header.kind == MessageKind::MethodCall && !self.is_ready() { - self.pending.lock().await.push_back(message); - if !self.drain_started.swap(true, Ordering::AcqRel) { - let transport = self.self_ref.upgrade().ok_or(Error::ConnectionClosed)?; - tokio::spawn(transport.drain_pending()); + { + let mut pending = self.pending.lock().expect("module pending lock"); + if self.is_stopped() { + return Err(Error::ConnectionClosed); + } + if is_call { + self.context.inflight.fetch_add(1, Ordering::AcqRel); + } + if is_call && !self.is_ready() { + pending.push_back(message); + if !self.drain_started.swap(true, Ordering::AcqRel) { + let transport = self.self_ref.upgrade().ok_or(Error::ConnectionClosed)?; + tokio::spawn(transport.drain_pending()); + } + return Ok(()); } - return Ok(()); } let result = self.deliver_now(message).await; if result.is_err() && is_call { diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 9cfff836..f83054b4 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -605,12 +605,17 @@ async fn shutdown_entry() { while !SHUTDOWN_ENTERED.load(Ordering::Acquire) { tokio::task::yield_now().await; } - }).await.expect("S00"); + }) + .await + .expect("S00"); } #[tokio::test] async fn ordinary_shutdown_deadline_preserves_state_until_callback_returns() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; let _release = ShutdownRelease; SHUTDOWN_BLOCKED.store(true, Ordering::Release); SHUTDOWN_ENTERED.store(false, Ordering::Release); @@ -618,26 +623,47 @@ async fn ordinary_shutdown_deadline_preserves_state_until_callback_returns() { let (transport, host) = ModuleTransport::new("stop".into(), vec![]); transport.defer_initialize(initialize_ok, host); transport.ensure_initialized().await.expect("S01"); - unsafe { (host.ready)(host.host_ctx); } + unsafe { + (host.ready)(host.host_ctx); + } let stopping = tokio::spawn(transport.clone().stop(Duration::from_millis(100))); shutdown_entry().await; - assert!(matches!(stopping.await.expect("S02"), Err(StopError::Started(_))), "S03"); + assert!( + matches!(stopping.await.expect("S02"), Err(StopError::Started(_))), + "S03" + ); assert!(!transport.is_faulted(), "S04"); assert!(transport.is_ready(), "S05"); - assert!(matches!(transport.clone().stop(Duration::from_millis(10)).await, Err(StopError::NotStarted(_))), "S06"); + assert!( + matches!( + transport.clone().stop(Duration::from_millis(10)).await, + Err(StopError::NotStarted(_)) + ), + "S06" + ); assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "S07"); SHUTDOWN_BLOCKED.store(false, Ordering::Release); - let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()).await.expect("S08"); + let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()) + .await + .expect("S08"); assert!(!transport.is_faulted(), "S09"); - unsafe { (host.ready)(host.host_ctx); } + unsafe { + (host.ready)(host.host_ctx); + } assert!(!transport.is_ready(), "S10"); - assert!(matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), "S11"); + assert!( + matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), + "S11" + ); assert!(!transport.init_failed(), "S12"); } #[tokio::test] async fn cancelling_a_started_shutdown_keeps_physical_exclusion_without_faulting() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; let _release = ShutdownRelease; SHUTDOWN_BLOCKED.store(true, Ordering::Release); SHUTDOWN_ENTERED.store(false, Ordering::Release); @@ -650,32 +676,56 @@ async fn cancelling_a_started_shutdown_keeps_physical_exclusion_without_faulting stopping.abort(); assert!(stopping.await.expect_err("S14").is_cancelled(), "S15"); assert!(!transport.is_faulted(), "S16"); - assert!(matches!(transport.clone().stop(Duration::from_millis(10)).await, Err(StopError::NotStarted(_))), "S17"); + assert!( + matches!( + transport.clone().stop(Duration::from_millis(10)).await, + Err(StopError::NotStarted(_)) + ), + "S17" + ); assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "S18"); - unsafe { (host.fault)(host.host_ctx, std::ptr::null(), 0); } + unsafe { + (host.fault)(host.host_ctx, std::ptr::null(), 0); + } SHUTDOWN_BLOCKED.store(false, Ordering::Release); - let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()).await.expect("S19"); - unsafe { (host.ready)(host.host_ctx); } + let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()) + .await + .expect("S19"); + unsafe { + (host.ready)(host.host_ctx); + } assert!(transport.is_faulted(), "S20"); assert!(!transport.is_ready(), "S21"); } #[tokio::test] async fn stopping_initialized_module_before_ready_releases_observers_and_queued_calls() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; SHUTDOWN_BLOCKED.store(false, Ordering::Release); let (transport, host) = ModuleTransport::new("queued-stop".into(), vec![]); transport.defer_initialize(initialize_ok, host); transport.send(call()).await.expect("S22"); assert!(transport.init_started(), "S23"); transport.stop_sync(Duration::ZERO); - tokio::time::timeout(Duration::from_secs(2), transport.wait_ready()).await.expect("S24"); + tokio::time::timeout(Duration::from_secs(2), transport.wait_ready()) + .await + .expect("S24"); tokio::time::timeout(Duration::from_secs(2), async { - while transport.drain_started.load(Ordering::Acquire) { tokio::task::yield_now().await; } - }).await.expect("S25"); + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("S25"); assert!(!transport.is_faulted(), "S26"); assert!(!transport.is_ready(), "S27"); - assert!(matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), "S28"); + assert!( + matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), + "S28" + ); assert!(!transport.init_failed(), "S29"); let weak = Arc::downgrade(&transport); drop(transport); @@ -684,23 +734,38 @@ async fn stopping_initialized_module_before_ready_releases_observers_and_queued_ #[tokio::test] async fn ordinary_stop_wakes_backpressure_without_turning_closure_into_a_fault() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; SHUTDOWN_BLOCKED.store(false, Ordering::Release); DELIVERY_CODE.store(TB_BACKPRESSURE, Ordering::Release); DELIVERIES.store(0, Ordering::Release); let (transport, host) = ModuleTransport::new("pressure-stop".into(), vec![]); transport.defer_initialize(initialize_ok, host); transport.send(call()).await.expect("S31"); - unsafe { (host.ready)(host.host_ctx); } + unsafe { + (host.ready)(host.host_ctx); + } tokio::time::timeout(Duration::from_secs(2), async { - while DELIVERIES.load(Ordering::Acquire) == 0 { tokio::task::yield_now().await; } - }).await.expect("S32"); + while DELIVERIES.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("S32"); transport.stop_sync(Duration::ZERO); // An explicit legacy wake also must not cause delivery to re-enter disposed code. - unsafe { (host.wake)(host.host_ctx); } + unsafe { + (host.wake)(host.host_ctx); + } tokio::time::timeout(Duration::from_secs(2), async { - while transport.drain_started.load(Ordering::Acquire) { tokio::task::yield_now().await; } - }).await.expect("S33"); + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("S33"); assert!(!transport.is_faulted(), "S34"); assert_eq!(DELIVERIES.load(Ordering::Acquire), 1, "S35"); DELIVERY_CODE.store(TB_OK, Ordering::Release); From d2a1011e698f69f7e38e317947a0caae60d4588d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 09:42:30 +0300 Subject: [PATCH 03/58] fix(module): report closed transport as connection closed during init Initialization failures were collapsed into a single string error, so a transport that was stopped mid-init surfaced as a generic transport error instead of a connection-closed one. The failure is now tracked as a distinct closed or failed variant, and callers only treat the closed case as a shutdown when deciding whether to mark initialization as failed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 32 ++++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 9116ed3f..4e2ce516 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -50,7 +50,7 @@ pub(crate) struct ModuleTransport { context: &'static HostContext, label: String, initializer: StdMutex>, - init_result: OnceCell>, + init_result: OnceCell>, pending: StdMutex>, drain_started: AtomicBool, // Reconfiguration and shutdown both call into opaque module code. The @@ -77,6 +77,13 @@ struct DeferredInitializer { struct SendModuleVtable(TbModuleVtable); +#[derive(Clone)] +enum InitFailure { + Closed, + Failed(String), +} + + // The opaque host context is process-lifetime state and every callback is // required by the ABI to be thread-safe. unsafe impl Send for DeferredInitializer {} @@ -270,7 +277,7 @@ impl ModuleTransport { .lock() .expect("initialization gate lock"); if self.is_stopped() { - return Err("module transport is closed".to_string()); + return Err(InitFailure::Closed); } self.context.init_started.store(true, Ordering::Release); } @@ -284,7 +291,7 @@ impl ModuleTransport { return if self.module.lock().expect("module vtable lock").is_some() { Ok(()) } else { - Err("module has no initializer".to_string()) + Err(InitFailure::Failed("module has no initializer".to_string())) }; }; // Module init is opaque synchronous code. Keep it off Tokio's @@ -300,18 +307,23 @@ impl ModuleTransport { Ok(Ok(Ok(initialized))) => initialized, Ok(Ok(Err(reason))) => { self.clear_config(); - return Err(reason); + return Err(InitFailure::Failed(reason)); } - Ok(Err(_)) => return Err("module initialization panicked".to_string()), - Err(_) => return Err("module initialization exceeded its deadline".to_string()), + Ok(Err(_)) => return Err(InitFailure::Failed("module initialization panicked".to_string())), + Err(_) => return Err(InitFailure::Failed("module initialization exceeded its deadline".to_string())), }; self.clear_config(); self.initialize(module.0) - .map_err(|_| "module returned an invalid vtable".to_string())?; + .map_err(|_| InitFailure::Failed("module returned an invalid vtable".to_string()))?; Ok(()) }) .await; - result.clone().map_err(Error::transport) + match result { + Ok(()) if self.is_stopped() => Err(Error::ConnectionClosed), + Ok(()) => Ok(()), + Err(InitFailure::Closed) => Err(Error::ConnectionClosed), + Err(InitFailure::Failed(reason)) => Err(Error::transport(reason.clone())), + } } pub(crate) async fn wait_ready(&self) { @@ -640,10 +652,10 @@ impl Transport for ModuleTransport { if self.is_stopped() { return Err(Error::ConnectionClosed); } - if self.ensure_initialized().await.is_err() { + if let Err(error) = self.ensure_initialized().await { { let _pending = self.pending.lock().expect("module pending lock"); - if self.is_stopped() { + if self.is_stopped() && matches!(error, Error::ConnectionClosed) { return Err(Error::ConnectionClosed); } self.context.init_failed.store(true, Ordering::Release); From dc15c43ae5cdb253df4ebb7e6e69fbccda9b3c67 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 09:43:58 +0300 Subject: [PATCH 04/58] test(module): cover stop during initialization and shutdown races Add host and transport tests asserting that an ordinary stop never fabricates a faulted state, whether the module is blocked in its shutdown callback or waiting for initialization admission, while a genuine initializer failure during stop is still retained. Also reset the shared delivery code between tests and assert the pending queue drains on stop. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_tests.rs | 111 +++++++++++++++++++ crates/tinybus/src/module/transport.rs | 18 ++- crates/tinybus/src/module/transport_tests.rs | 67 +++++++++++ 3 files changed, 191 insertions(+), 5 deletions(-) diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 2327f787..643ac672 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -2884,3 +2884,114 @@ impl crate::Interface for NativeBlockingCallback { Ok(serde_json::json!(context.authenticated_sender().unwrap())) } } + +static BLOCKING_STOP_RECEIVER: OnceLock>>> = + OnceLock::new(); +static BLOCKING_STOP_STARTED: AtomicBool = AtomicBool::new(false); +static BLOCKING_STOP_CALLS: AtomicUsize = AtomicUsize::new(0); + +unsafe extern "C" fn blocking_shutdown(_: *mut std::ffi::c_void, _: u64) -> i32 { + BLOCKING_STOP_CALLS.fetch_add(1, Ordering::AcqRel); + BLOCKING_STOP_STARTED.store(true, Ordering::Release); + if let Some(receiver) = BLOCKING_STOP_RECEIVER + .get_or_init(|| StdMutex::new(None)) + .lock() + .expect("H00") + .take() + { + let _ = receiver.recv(); + } + TB_OK +} +unsafe extern "C" fn blocking_shutdown_init( + host: *const crate::module::abi::TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + let code = unsafe { lazy_echo_init(host, out) }; + if code == TB_OK { + unsafe { + (*out).shutdown = blocking_shutdown; + } + } + code +} +struct StopRelease(Option>); +impl Drop for StopRelease { + fn drop(&mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } +} + +#[tokio::test] +async fn ordinary_stop_never_fabricates_faulted_before_or_after_callback_completion() { + let _test_guard = FAKE_MODULE_TEST_LOCK.lock().await; + let host = ModuleHost::new(Broker::new()); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let release = StopRelease(Some(sender)); + *BLOCKING_STOP_RECEIVER + .get_or_init(|| StdMutex::new(None)) + .lock() + .expect("H01") = Some(receiver); + BLOCKING_STOP_STARTED.store(false, Ordering::Release); + BLOCKING_STOP_CALLS.store(0, Ordering::Release); + unsafe { + host.attach_raw( + "clock.so", + TbAbiDescriptor::current("clock", "0.1.0"), + manifest(), + blocking_shutdown_init, + ) + } + .expect("H02"); + tokio::time::timeout(Duration::from_secs(2), async { + while host.list()[0].state != ModuleState::Ready { + tokio::task::yield_now().await; + } + }) + .await + .expect("H03"); + let inner = host.inner.clone(); + let stopping = + tokio::spawn(async move { inner.stop("clock", Duration::from_millis(100)).await }); + tokio::time::timeout(Duration::from_secs(2), async { + while !BLOCKING_STOP_STARTED.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("H04"); + assert!(stopping.await.expect("H05").is_err(), "H06"); + assert_eq!(host.list()[0].state, ModuleState::Ready, "H07"); + let retry = host.inner.stop("clock", Duration::from_millis(10)).await; + assert!(matches!(retry, Err(Error::MethodFailed { .. })), "H08"); + assert_eq!(BLOCKING_STOP_CALLS.load(Ordering::Acquire), 1, "H09"); + // Hold the host observer's lock while the blocking callback returns. The + // real broker cannot commit peer detach until this assertion is finished. + // No await occurs under this lock, and the opaque callback needs no host lock. + { + let loaded = host.inner.loaded.lock().expect("H10"); + drop(release); + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while loaded[0].transport.is_ready() { + assert!(std::time::Instant::now() < deadline, "H11"); + std::thread::yield_now(); + } + assert!(!loaded[0].transport.is_faulted(), "H12"); + assert!( + !matches!( + loaded[0].snapshot().state, + ModuleState::Faulted { .. } | ModuleState::Failed { .. } + ), + "H13" + ); + } + tokio::time::timeout(Duration::from_secs(2), async { + while host.list()[0].state != ModuleState::Stopped { + tokio::task::yield_now().await; + } + }) + .await + .expect("H14"); +} diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 4e2ce516..23246283 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -83,7 +83,6 @@ enum InitFailure { Failed(String), } - // The opaque host context is process-lifetime state and every callback is // required by the ABI to be thread-safe. unsafe impl Send for DeferredInitializer {} @@ -309,12 +308,21 @@ impl ModuleTransport { self.clear_config(); return Err(InitFailure::Failed(reason)); } - Ok(Err(_)) => return Err(InitFailure::Failed("module initialization panicked".to_string())), - Err(_) => return Err(InitFailure::Failed("module initialization exceeded its deadline".to_string())), + Ok(Err(_)) => { + return Err(InitFailure::Failed( + "module initialization panicked".to_string(), + )); + } + Err(_) => { + return Err(InitFailure::Failed( + "module initialization exceeded its deadline".to_string(), + )); + } }; self.clear_config(); - self.initialize(module.0) - .map_err(|_| InitFailure::Failed("module returned an invalid vtable".to_string()))?; + self.initialize(module.0).map_err(|_| { + InitFailure::Failed("module returned an invalid vtable".to_string()) + })?; Ok(()) }) .await; diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index f83054b4..e1819499 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -597,6 +597,7 @@ struct ShutdownRelease; impl Drop for ShutdownRelease { fn drop(&mut self) { SHUTDOWN_BLOCKED.store(false, Ordering::Release); + DELIVERY_CODE.store(TB_OK, Ordering::Release); } } @@ -720,6 +721,7 @@ async fn stopping_initialized_module_before_ready_releases_observers_and_queued_ }) .await .expect("S25"); + assert!(transport.pending.lock().expect("S36").is_empty(), "S37"); assert!(!transport.is_faulted(), "S26"); assert!(!transport.is_ready(), "S27"); assert!( @@ -734,6 +736,7 @@ async fn stopping_initialized_module_before_ready_releases_observers_and_queued_ #[tokio::test] async fn ordinary_stop_wakes_backpressure_without_turning_closure_into_a_fault() { + let _release = ShutdownRelease; let _lock = VTABLE_TEST_LOCK .get_or_init(|| tokio::sync::Mutex::new(())) .lock() @@ -770,3 +773,67 @@ async fn ordinary_stop_wakes_backpressure_without_turning_closure_into_a_fault() assert_eq!(DELIVERIES.load(Ordering::Acquire), 1, "S35"); DELIVERY_CODE.store(TB_OK, Ordering::Release); } + +#[tokio::test] +async fn a_send_waiting_for_initialization_admission_closes_without_an_initializer_fault() { + let (transport, host) = ModuleTransport::new("admission-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.pause_initialization(); + let sending_transport = transport.clone(); + let sending = tokio::spawn(async move { sending_transport.send(call()).await }); + tokio::task::yield_now().await; + transport.stop_sync(Duration::ZERO); + assert!( + matches!( + tokio::time::timeout(Duration::from_secs(2), sending) + .await + .expect("S38") + .expect("S39"), + Err(Error::ConnectionClosed) + ), + "S40" + ); + assert!(!transport.init_started(), "S41"); + assert!(!transport.init_failed(), "S42"); + assert!(!transport.is_faulted(), "S43"); + assert!(transport.pending.lock().expect("S44").is_empty(), "S45"); + assert!(!transport.drain_started.load(Ordering::Acquire), "S46"); +} + +struct InitializerRelease(Option>); +impl Drop for InitializerRelease { + fn drop(&mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } +} + +#[tokio::test] +async fn an_actual_initializer_failure_during_stop_is_retained() { + let (transport, host) = ModuleTransport::new("failed-stop".into(), vec![]); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let release = InitializerRelease(Some(sender)); + let (entered, observed) = tokio::sync::oneshot::channel(); + transport.defer_initializer(host, move |_| { + let _ = entered.send(()); + let _ = receiver.recv(); + Err("module initialization failed".into()) + }); + let sending_transport = transport.clone(); + let sending = tokio::spawn(async move { sending_transport.send(call()).await }); + tokio::time::timeout(Duration::from_secs(2), observed) + .await + .expect("S47") + .expect("S48"); + transport.stop_sync(Duration::ZERO); + drop(release); + tokio::time::timeout(Duration::from_secs(2), sending) + .await + .expect("S49") + .expect("S50") + .expect("S51"); + assert!(transport.init_failed(), "S52"); + assert!(transport.is_faulted(), "S53"); + assert!(!transport.is_ready(), "S54"); +} From ddaea7d5f076872947d4fa9bb5ba6a0ed7c3f0d8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 09:45:29 +0300 Subject: [PATCH 05/58] fix(tinybus): preserve pending calls on non-connection delivery errors Delivery failures now carry the underlying error into fault_pending, which only clears queued calls when the module is stopped and the error is a connection close. Previously any failure while stopped discarded pending calls, and the stop path could finish before the module vtable was cleared. Comments were updated to describe admission rejection rather than shutdown. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 23246283..f1494db8 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -216,8 +216,8 @@ impl ModuleTransport { } } - // Terminal state is distinct from initialization: never fabricate READY or - // init_started to release observers during admission rollback or shutdown. + // Admission rejection is a genuine fault, unlike ordinary shutdown. Never + // fabricate READY or init_started to release a rejected observer. fn terminate_waiters(&self) { self.context.faulted.store(true, Ordering::Release); self.context.init_notify.notify_waiters(); @@ -499,8 +499,8 @@ impl ModuleTransport { let Some(message) = message else { break; }; - if self.deliver_now(message.clone()).await.is_err() { - self.fault_pending(message).await; + if let Err(error) = self.deliver_now(message.clone()).await { + self.fault_pending(message, error).await; self.drain_started.store(false, Ordering::Release); return; } @@ -533,10 +533,10 @@ impl ModuleTransport { /// Answer every still-queued call — and the one that just failed to /// deliver — with `ModuleUnavailable`, then fault the module, so callers /// learn the delivery failed instead of waiting out their deadline. - async fn fault_pending(&self, failed: Message) { + async fn fault_pending(&self, failed: Message, error: Error) { let cleared = { let mut pending = self.pending.lock().expect("module pending lock"); - if self.is_stopped() { + if self.is_stopped() && matches!(error, Error::ConnectionClosed) { pending.clear(); return; } @@ -592,7 +592,7 @@ impl ModuleTransport { .initialization_gate .lock() .expect("initialization gate lock"); - if !self.init_started() { + if !self.init_started() && self.module.lock().expect("module vtable lock").is_none() { self.finish_stop(); } } From 4bcce26d2d4d3696f2f81d0d318f507227d4e9c9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 09:46:40 +0300 Subject: [PATCH 06/58] fix(tinybus): reset stop marker only after acquiring lifecycle lock Move the stop_task_spawned reset to after the lifecycle lock is acquired so a retry that fails to take the lock no longer clears the marker set by the original running stop. The flag is now only replaced by the new lock owner right before the task is spawned. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index f1494db8..27cd1d4c 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -616,7 +616,6 @@ impl ModuleTransport { self: Arc, deadline: Duration, ) -> std::result::Result { - self.stop_task_spawned.store(false, Ordering::Release); let started = std::time::Instant::now(); let lifecycle = tokio::time::timeout(deadline, self.lifecycle.clone().lock_owned()) .await @@ -626,6 +625,10 @@ impl ModuleTransport { )) })?; let remaining = deadline.saturating_sub(started.elapsed()); + // A retry that cannot acquire the physical lock must not erase the + // original running stop's host transition. Only its new lock owner may + // replace that marker, immediately before handing the lock to the task. + self.stop_task_spawned.store(false, Ordering::Release); self.stop_task_spawned.store(true, Ordering::Release); let task = tokio::task::spawn_blocking(move || { let _lifecycle = lifecycle; From 89118b639fcb4f9509d59dad8cb76f64814774de Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 09:47:39 +0300 Subject: [PATCH 07/58] fix(transport): stop clearing the stop task marker before respawn The redundant store of false before setting stop_task_spawned to true could erase the original running stop's host transition, so it has been removed. Tests now cover concurrent pending sends racing a completed drainer and a delivery failure overlapping stop. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 1 - crates/tinybus/src/module/transport_tests.rs | 86 ++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 27cd1d4c..4b99de9b 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -628,7 +628,6 @@ impl ModuleTransport { // A retry that cannot acquire the physical lock must not erase the // original running stop's host transition. Only its new lock owner may // replace that marker, immediately before handing the lock to the task. - self.stop_task_spawned.store(false, Ordering::Release); self.stop_task_spawned.store(true, Ordering::Release); let task = tokio::task::spawn_blocking(move || { let _lifecycle = lifecycle; diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index e1819499..0b34afb3 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -5,6 +5,7 @@ use crate::{BusName, InterfaceName, MemberName, ObjectPath}; static DELIVERY_CODE: AtomicI32 = AtomicI32::new(TB_OK); static DELIVERIES: AtomicUsize = AtomicUsize::new(0); +static DELIVERY_BLOCKED: AtomicBool = AtomicBool::new(false); static SHUTDOWN_CODE: AtomicI32 = AtomicI32::new(TB_OK); static SHUTDOWN_CALLS: AtomicUsize = AtomicUsize::new(0); static SHUTDOWN_BLOCKED: AtomicBool = AtomicBool::new(false); @@ -16,6 +17,9 @@ static VTABLE_TEST_LOCK: std::sync::OnceLock> = std::sync unsafe extern "C" fn deliver(_: *mut c_void, _: *const u8, _: usize) -> i32 { DELIVERIES.fetch_add(1, Ordering::AcqRel); + while DELIVERY_BLOCKED.load(Ordering::Acquire) { + std::thread::yield_now(); + } DELIVERY_CODE.load(Ordering::Acquire) } @@ -597,6 +601,7 @@ struct ShutdownRelease; impl Drop for ShutdownRelease { fn drop(&mut self) { SHUTDOWN_BLOCKED.store(false, Ordering::Release); + DELIVERY_BLOCKED.store(false, Ordering::Release); DELIVERY_CODE.store(TB_OK, Ordering::Release); } } @@ -837,3 +842,84 @@ async fn an_actual_initializer_failure_during_stop_is_retained() { assert!(transport.is_faulted(), "S53"); assert!(!transport.is_ready(), "S54"); } + +#[tokio::test] +async fn concurrent_pending_sends_cannot_resurrect_a_completed_transport_drainer() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let (transport, host) = ModuleTransport::new("racing-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("S55"); + let barrier = Arc::new(tokio::sync::Barrier::new(17)); + let mut sending = Vec::new(); + for _ in 0..16 { + let sender = transport.clone(); + let barrier = barrier.clone(); + sending.push(tokio::spawn(async move { + barrier.wait().await; + sender.send(call()).await + })); + } + barrier.wait().await; + transport.stop_sync(Duration::ZERO); + for sending in sending { + assert!( + matches!( + sending.await.expect("S56"), + Ok(()) | Err(Error::ConnectionClosed) + ), + "S57" + ); + } + tokio::time::timeout(Duration::from_secs(2), async { + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("S58"); + assert!(transport.pending.lock().expect("S59").is_empty(), "S60"); + assert!(!transport.is_faulted(), "S61"); + assert!(!transport.init_failed(), "S62"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_actual_delivery_failure_overlapping_stop_is_retained() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + DELIVERY_CODE.store(TB_BAD_ARGUMENT, Ordering::Release); + DELIVERY_BLOCKED.store(true, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("delivery-failed-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.send(call()).await.expect("S63"); + unsafe { + (host.ready)(host.host_ctx); + } + tokio::time::timeout(Duration::from_secs(2), async { + while DELIVERIES.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("S64"); + transport.stop_sync(Duration::ZERO); + DELIVERY_BLOCKED.store(false, Ordering::Release); + tokio::time::timeout(Duration::from_secs(2), async { + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("S65"); + assert!(transport.is_faulted(), "S66"); + assert!(!transport.is_ready(), "S67"); +} From 30a4732fbbb3817d0345a80ef91e9347947da1a1 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 09:50:24 +0300 Subject: [PATCH 08/58] fix(module): reject stop and reinitialize on disposed transports Stopping or reinitializing a transport that has already been disposed now fails fast with ConnectionClosed instead of racing the lifecycle lock, and the host no longer clears its transition marker when a stop retry is refused. This keeps a disposed transport from being revived or from erasing the original stop's host transition. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host.rs | 4 +++- crates/tinybus/src/module/host_tests.rs | 13 +++++++++++++ crates/tinybus/src/module/transport.rs | 15 +++++++++++++++ crates/tinybus/src/module/transport_tests.rs | 17 +++++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/host.rs b/crates/tinybus/src/module/host.rs index eb9c9cd6..06869a74 100644 --- a/crates/tinybus/src/module/host.rs +++ b/crates/tinybus/src/module/host.rs @@ -1513,7 +1513,9 @@ impl ModuleControl for ModuleHostInner { match stopped { Ok(_) => {} Err(StopError::NotStarted(error)) => { - module.transition_from = None; + if !module.transport.stop_task_spawned() { + module.transition_from = None; + } return Err(error); } Err(StopError::Started(error)) => return Err(error), diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 643ac672..1fd3b73e 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -2979,6 +2979,19 @@ async fn ordinary_stop_never_fabricates_faulted_before_or_after_callback_complet std::thread::yield_now(); } assert!(!loaded[0].transport.is_faulted(), "H12"); + // A disposed transport refuses a retry immediately, before any new + // physical task or marker change, even while detach is held back. + let mut retry = std::pin::pin!(loaded[0].transport.clone().stop(Duration::from_secs(1))); + let mut context = std::task::Context::from_waker(std::task::Waker::noop()); + assert!( + matches!( + std::future::Future::poll(retry.as_mut(), &mut context), + std::task::Poll::Ready(Err(StopError::NotStarted(Error::ConnectionClosed))) + ), + "H15" + ); + assert!(loaded[0].transport.stop_task_spawned(), "H16"); + assert_eq!(BLOCKING_STOP_CALLS.load(Ordering::Acquire), 1, "H17"); assert!( !matches!( loaded[0].snapshot().state, diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 4b99de9b..0d8e7047 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -359,6 +359,9 @@ impl ModuleTransport { } pub(crate) async fn reinitialize(self: &Arc, bytes: crate::Secret) -> Result<()> { + if self.is_stopped() { + return Err(Error::ConnectionClosed); + } let started = std::time::Instant::now(); let lifecycle = tokio::time::timeout(MODULE_INIT_DEADLINE, self.lifecycle.clone().lock_owned()) @@ -368,6 +371,9 @@ impl ModuleTransport { .expect("static member name is valid"), timeout_ms: MODULE_INIT_DEADLINE.as_millis() as u64, })?; + if self.is_stopped() { + return Err(Error::ConnectionClosed); + } let module = self .module .lock() @@ -587,6 +593,9 @@ impl ModuleTransport { } pub(crate) fn stop_sync(&self, deadline: Duration) -> i32 { + if self.is_stopped() { + return TB_CLOSED; + } { let _gate = self .initialization_gate @@ -616,6 +625,9 @@ impl ModuleTransport { self: Arc, deadline: Duration, ) -> std::result::Result { + if self.is_stopped() { + return Err(StopError::NotStarted(Error::ConnectionClosed)); + } let started = std::time::Instant::now(); let lifecycle = tokio::time::timeout(deadline, self.lifecycle.clone().lock_owned()) .await @@ -624,6 +636,9 @@ impl ModuleTransport { "module lifecycle operation exceeded its deadline", )) })?; + if self.is_stopped() { + return Err(StopError::NotStarted(Error::ConnectionClosed)); + } let remaining = deadline.saturating_sub(started.elapsed()); // A retry that cannot acquire the physical lock must not erase the // original running stop's host transition. Only its new lock owner may diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 0b34afb3..c673389e 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -662,6 +662,23 @@ async fn ordinary_shutdown_deadline_preserves_state_until_callback_returns() { "S11" ); assert!(!transport.init_failed(), "S12"); + assert!( + matches!( + transport.clone().stop(Duration::from_secs(1)).await, + Err(StopError::NotStarted(Error::ConnectionClosed)) + ), + "S68" + ); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "S69"); + assert!( + matches!( + transport + .reinitialize(crate::Secret::new(b"{}".to_vec())) + .await, + Err(Error::ConnectionClosed) + ), + "S70" + ); } #[tokio::test] From 360b9c1025a0c8eba8da6b45f30e4ce08e91d05c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 09:51:11 +0300 Subject: [PATCH 09/58] test(transport): cover lifecycle waiters after shutdown disposal Add a regression test asserting that stop and reinitialize waiters blocked behind an in-flight shutdown fail with ConnectionClosed instead of re-entering the disposed transport, and that shutdown runs only once. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index c673389e..273b70a6 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -940,3 +940,51 @@ async fn an_actual_delivery_failure_overlapping_stop_is_retained() { assert!(transport.is_faulted(), "S66"); assert!(!transport.is_ready(), "S67"); } + +#[tokio::test] +async fn lifecycle_waiters_do_not_reenter_a_transport_disposed_by_the_prior_shutdown() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("lifecycle-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("S71"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_millis(100))); + shutdown_entry().await; + assert!( + matches!(stopping.await.expect("S72"), Err(StopError::Started(_))), + "S73" + ); + let mut retry = std::pin::pin!(transport.clone().stop(Duration::from_secs(2))); + let mut reinitializing = + std::pin::pin!(transport.reinitialize(crate::Secret::new(b"{}".to_vec()))); + let mut context = std::task::Context::from_waker(std::task::Waker::noop()); + assert!( + std::future::Future::poll(retry.as_mut(), &mut context).is_pending(), + "S74" + ); + assert!( + std::future::Future::poll(reinitializing.as_mut(), &mut context).is_pending(), + "S75" + ); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + assert!( + matches!( + retry.await, + Err(StopError::NotStarted(Error::ConnectionClosed)) + ), + "S76" + ); + assert!( + matches!(reinitializing.await, Err(Error::ConnectionClosed)), + "S77" + ); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "S78"); + assert!(transport.stop_task_spawned(), "S79"); + assert!(!transport.is_faulted(), "S80"); +} From 6a7bd8c01ba0cc43188c06dfcc48827f0bf4f7b4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 10:05:33 +0300 Subject: [PATCH 10/58] test(module): assert stop retry and state transition ordering Extend the ordinary stop test to drive a real broker and observe ModuleStateChanged signals, so the retry against a closed transport is checked to fail with ConnectionClosed while the loaded entry still reports Ready. The test now also verifies the emitted stopped event carries the expected ready-to-stopped transition. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_tests.rs | 51 ++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 1fd3b73e..7f960b23 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -2927,7 +2927,22 @@ impl Drop for StopRelease { #[tokio::test] async fn ordinary_stop_never_fabricates_faulted_before_or_after_callback_completion() { let _test_guard = FAKE_MODULE_TEST_LOCK.lock().await; - let host = ModuleHost::new(Broker::new()); + let bus = MemoryBus::new(); + let broker = Broker::new(); + let broker_task = broker.spawn(bus.clone()); + let host = ModuleHost::new(broker); + let observer = Connection::connect(bus.connect().await.expect("H18")) + .await + .expect("H19"); + let mut state_changes = observer + .add_match( + crate::router::MatchRule::parse( + "type=signal,interface=ai.tinyhumans.tinybus.Bus,member=ModuleStateChanged", + ) + .expect("H20"), + ) + .await + .expect("H21"); let (sender, receiver) = std::sync::mpsc::sync_channel(1); let release = StopRelease(Some(sender)); *BLOCKING_STOP_RECEIVER @@ -3000,6 +3015,25 @@ async fn ordinary_stop_never_fabricates_faulted_before_or_after_callback_complet "H13" ); } + // This is a current-thread runtime. Without an await or yield after dropping + // the loaded lock, the broker cannot process its ready detach task before + // this real Host retry completes synchronously against the closed transport. + { + let mut retry = std::pin::pin!(host.inner.stop("clock", Duration::from_secs(1))); + let mut context = std::task::Context::from_waker(std::task::Waker::noop()); + assert!( + matches!( + std::future::Future::poll(retry.as_mut(), &mut context), + std::task::Poll::Ready(Err(Error::ConnectionClosed)) + ), + "H22" + ); + } + assert_eq!( + host.inner.loaded.lock().expect("H23")[0].transition_from, + Some(ModuleState::Ready), + "H24" + ); tokio::time::timeout(Duration::from_secs(2), async { while host.list()[0].state != ModuleState::Stopped { tokio::task::yield_now().await; @@ -3007,4 +3041,19 @@ async fn ordinary_stop_never_fabricates_faulted_before_or_after_callback_complet }) .await .expect("H14"); + let stopped_event = tokio::time::timeout(Duration::from_secs(2), async { + loop { + let message = state_changes.recv().await.expect("H25"); + if message.body.get(2).and_then(serde_json::Value::as_str) == Some("stopped") { + break message; + } + } + }) + .await + .expect("H26"); + assert!( + stopped_event.body == serde_json::json!(["clock", "ready", "stopped", null]), + "H27" + ); + broker_task.abort(); } From 4dbb0582506b7b849759034d0f4590d7e12ee547 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 10:06:33 +0300 Subject: [PATCH 11/58] test(module): compare transition origin by pattern match The assertion on the recorded transition origin now clones the value and matches it against Some(ModuleState::Ready) instead of comparing with assert_eq!, avoiding a borrow of the guarded field across the assertion. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_tests.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 7f960b23..ef6db918 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -3029,11 +3029,10 @@ async fn ordinary_stop_never_fabricates_faulted_before_or_after_callback_complet "H22" ); } - assert_eq!( - host.inner.loaded.lock().expect("H23")[0].transition_from, - Some(ModuleState::Ready), - "H24" - ); + let origin = host.inner.loaded.lock().expect("H23")[0] + .transition_from + .clone(); + assert!(matches!(origin, Some(ModuleState::Ready)), "H24"); tokio::time::timeout(Duration::from_secs(2), async { while host.list()[0].state != ModuleState::Stopped { tokio::task::yield_now().await; From 1e3ef98f92081b84333cd3374f2ba007cdeb733c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 10:07:02 +0300 Subject: [PATCH 12/58] fix(tinybus): preserve existing transition_from on module restart The module host was overwriting transition_from with the previous state whenever a restart began, which discarded a transition already in progress. Using get_or_insert keeps the original value so the state machine reports the correct origin state. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/host.rs b/crates/tinybus/src/module/host.rs index 06869a74..e0073ea4 100644 --- a/crates/tinybus/src/module/host.rs +++ b/crates/tinybus/src/module/host.rs @@ -1499,7 +1499,7 @@ impl ModuleControl for ModuleHostInner { .lock() .expect("router lock") .revoke_callbacks_for_peer(&module.unique_name); - module.transition_from = Some(old); + module.transition_from.get_or_insert(old); module.lifecycle_busy = true; module.transport.clone() }; From 79b316c677432dfa7adeb90196f135c8b04c5054 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 10:54:21 +0300 Subject: [PATCH 13/58] test(transport): cover admission and drain during shutdown Add tests asserting that fresh signals and calls are rejected with ConnectionClosed once shutdown has actually begun, that queued readiness and backpressure retire before stop returns, and that an in-flight signal callback is drained even when stop times out and is cancelled. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 92 ++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 273b70a6..940f3520 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -988,3 +988,95 @@ async fn lifecycle_waiters_do_not_reenter_a_transport_disposed_by_the_prior_shut assert!(transport.stop_task_spawned(), "S79"); assert!(!transport.is_faulted(), "S80"); } + +#[tokio::test] +async fn fresh_signal_and_call_cannot_enter_an_actual_running_shutdown() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("admission".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("A01"); + unsafe { (host.ready)(host.host_ctx); } + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + shutdown_entry().await; + let mut signal = call(); + signal.header.kind = MessageKind::Signal; + let signal_result = transport.send(signal).await; + let call_result = transport.send(call()).await; + assert!(matches!(signal_result, Err(Error::ConnectionClosed)), "A02"); + assert!(matches!(call_result, Err(Error::ConnectionClosed)), "A03"); + assert_eq!(DELIVERIES.load(Ordering::Acquire), 0, "A04"); + assert!(!transport.is_faulted(), "A05"); + assert!(!transport.init_failed(), "A06"); + assert!(!transport.is_stopped(), "A07"); + assert!(transport.is_ready(), "A08"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + assert_eq!(stopping.await.expect("A09").expect("A10"), TB_OK, "A11"); +} + +#[tokio::test] +async fn queued_readiness_and_backpressure_retire_before_actual_shutdown_returns() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("queued-admission".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.send(call()).await.expect("A12"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + shutdown_entry().await; + tokio::time::timeout(Duration::from_millis(100), transport.wait_ready()).await.expect("A13"); + unsafe { (host.ready)(host.host_ctx); (host.wake)(host.host_ctx); } + tokio::time::timeout(Duration::from_millis(100), async { + while transport.drain_started.load(Ordering::Acquire) { tokio::task::yield_now().await; } + }).await.expect("A14"); + let empty = transport.pending.lock().expect("A15").is_empty(); + assert!(empty, "A16"); + assert_eq!(DELIVERIES.load(Ordering::Acquire), 0, "A17"); + assert!(!transport.is_ready(), "A18"); + assert!(!transport.is_faulted(), "A19"); + assert!(!transport.is_stopped(), "A20"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + stopping.await.expect("A21").expect("A22"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_signal_callback_is_drained_despite_stop_timeout_and_cancellation() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + DELIVERY_BLOCKED.store(true, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("callback-drain".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("A23"); + let mut signal = call(); signal.header.kind = MessageKind::Signal; + let sending_transport = transport.clone(); + let sending = tokio::spawn(async move { sending_transport.send(signal).await }); + tokio::time::timeout(Duration::from_secs(2), async { + while DELIVERIES.load(Ordering::Acquire) == 0 { tokio::task::yield_now().await; } + }).await.expect("A24"); + assert_eq!(transport.inflight(), 0, "A25"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_millis(50))); + assert!(matches!(stopping.await.expect("A26"), Err(StopError::Started(_))), "A27"); + assert!(!SHUTDOWN_ENTERED.load(Ordering::Acquire), "A28"); + assert!(!transport.is_stopped(), "A29"); + assert!(transport.stop_task_spawned(), "A30"); + assert!(matches!(transport.clone().stop(Duration::from_millis(10)).await, Err(StopError::NotStarted(_))), "A31"); + assert!(matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), "A32"); + DELIVERY_BLOCKED.store(false, Ordering::Release); + sending.await.expect("A33").expect("A34"); + shutdown_entry().await; + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "A35"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()).await.expect("A36"); + assert!(transport.is_stopped(), "A37"); + assert!(!transport.is_faulted(), "A38"); +} From d706b47be7d0fe1ebbf8213cca6284a0b1d1536a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 10:54:59 +0300 Subject: [PATCH 14/58] test(transport): cover admission during blocked shutdown Add two tests for module transport admission while an actual shutdown is in progress. One verifies that an installed vtable without the init flag closes fresh admission before teardown, and the other verifies that a backpressure retry closes without faulting the transport. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 54 ++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 940f3520..e3f64641 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1080,3 +1080,57 @@ async fn actual_signal_callback_is_drained_despite_stop_timeout_and_cancellation assert!(transport.is_stopped(), "A37"); assert!(!transport.is_faulted(), "A38"); } + +#[tokio::test] +async fn installed_vtable_without_init_flag_closes_fresh_admission_before_teardown() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("installed-admission".into(), vec![]); + let mut module = TbModuleVtable::default(); + unsafe { initialize_ok(&host, &mut module); } + transport.initialize(module).expect("A39"); + assert!(!transport.init_started(), "A40"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + shutdown_entry().await; + let result = transport.send(call()).await; + assert!(matches!(result, Err(Error::ConnectionClosed)), "A41"); + assert!(!transport.init_started(), "A42"); + assert!(!transport.is_stopped(), "A43"); + assert!(!transport.is_faulted(), "A44"); + assert!(!transport.init_failed(), "A45"); + assert_eq!(DELIVERIES.load(Ordering::Acquire), 0, "A46"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + stopping.await.expect("A47").expect("A48"); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "A49"); +} + +#[tokio::test] +async fn backpressure_retry_closes_without_fault_while_actual_shutdown_is_blocked() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + DELIVERY_CODE.store(TB_BACKPRESSURE, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("pressure-admission".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.send(call()).await.expect("A50"); + unsafe { (host.ready)(host.host_ctx); } + tokio::time::timeout(Duration::from_secs(2), async { + while DELIVERIES.load(Ordering::Acquire) == 0 { tokio::task::yield_now().await; } + }).await.expect("A51"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + shutdown_entry().await; + tokio::time::timeout(Duration::from_millis(100), async { + while transport.drain_started.load(Ordering::Acquire) { tokio::task::yield_now().await; } + }).await.expect("A52"); + unsafe { (host.wake)(host.host_ctx); } + assert_eq!(DELIVERIES.load(Ordering::Acquire), 1, "A53"); + assert!(!transport.is_faulted(), "A54"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + stopping.await.expect("A55").expect("A56"); +} From b9f21efe934d47ee3d73250b8ab39f672da0a8ed Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 10:56:05 +0300 Subject: [PATCH 15/58] fix(module): drain in-flight callbacks before module teardown Shutdown now closes admission separately from physical completion and waits for every reserved native callback to return before tearing down the module, so a stop timeout or cancelled caller can no longer free code underneath a running callback. Readiness signals arriving after admission closes are ignored, and queued confidential bodies are released outside internal locks. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 151 ++++++++++++++----- crates/tinybus/src/module/transport_tests.rs | 107 ++++++++++--- 2 files changed, 195 insertions(+), 63 deletions(-) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 0d8e7047..8b2256a9 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -3,7 +3,7 @@ use std::collections::VecDeque; use std::ffi::c_void; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex as StdMutex, Weak}; +use std::sync::{Arc, Condvar, Mutex as StdMutex, Weak}; use std::time::Duration; use async_trait::async_trait; @@ -34,6 +34,7 @@ struct HostContext { faulted: AtomicBool, init_failed: AtomicBool, stopped: AtomicBool, + admission_closed: AtomicBool, init_started: AtomicBool, init_notify: Notify, ready: AtomicBool, @@ -58,10 +59,57 @@ pub(crate) struct ModuleTransport { // let a second lifecycle callback overlap the still-running first one. lifecycle: Arc>, stop_task_spawned: Arc, + callbacks: CallbackGate, initialization_gate: StdMutex>, initialization_gate_changed: Notify, } +// Reservations cover physical callback execution, including signals. They are +// independent of unanswered method calls and remain owned by the actual caller. +#[derive(Default)] +struct CallbackState { + closed: bool, + active: usize, +} + +#[derive(Default)] +struct CallbackGate { + state: StdMutex, + drained: Condvar, +} + +struct CallbackLease<'a>(&'a CallbackGate); + +impl CallbackGate { + /// Reserve entry atomically with physical stop's admission closure. + fn enter(&self) -> Result> { + let mut state = self.state.lock().expect("module callback lock"); + if state.closed { + return Err(Error::ConnectionClosed); + } + state.active += 1; + Ok(CallbackLease(self)) + } + + /// Retain the shutdown worker until every reserved callback has returned. + fn wait_drained(&self) { + let mut state = self.state.lock().expect("module callback lock"); + while state.active != 0 { + state = self.drained.wait(state).expect("module callback lock"); + } + } +} + +impl Drop for CallbackLease<'_> { + fn drop(&mut self) { + let mut state = self.0.state.lock().expect("module callback lock"); + state.active -= 1; + if state.active == 0 { + self.0.drained.notify_all(); + } + } +} + // `module_ctx` is opaque and all access to it goes through callbacks whose ABI // contract requires thread safety. The vtable itself is immutable after init. unsafe impl Send for ModuleTransport {} @@ -118,6 +166,7 @@ impl ModuleTransport { faulted: AtomicBool::new(false), init_failed: AtomicBool::new(false), stopped: AtomicBool::new(false), + admission_closed: AtomicBool::new(false), init_started: AtomicBool::new(false), init_notify: Notify::new(), ready: AtomicBool::new(false), @@ -137,6 +186,7 @@ impl ModuleTransport { drain_started: AtomicBool::new(false), lifecycle: Arc::new(Mutex::new(())), stop_task_spawned: Arc::new(AtomicBool::new(false)), + callbacks: CallbackGate::default(), initialization_gate: StdMutex::new(Some(true)), initialization_gate_changed: Notify::new(), }); @@ -225,21 +275,41 @@ impl ModuleTransport { self.initialization_gate_changed.notify_waiters(); } - // Normal completion closes admission without inventing a module fault. The - // queue lock makes closure atomic with send admission and drainer retirement; - // no queued body survives disposal or starts a replacement drainer. - fn finish_stop(&self) { - { + /// Close admission before teardown, without reporting physical completion + /// or inventing a fault. The lifecycle owner alone invokes this operation. + fn close_admission(&self) { + let discarded = { + let _initialization = self + .initialization_gate + .lock() + .expect("initialization gate lock"); let mut pending = self.pending.lock().expect("module pending lock"); - self.context.stopped.store(true, Ordering::Release); - pending.clear(); - } + let mut callbacks = self.callbacks.state.lock().expect("module callback lock"); + callbacks.closed = true; + self.context.admission_closed.store(true, Ordering::Release); + std::mem::take(&mut *pending) + }; + // Queued confidential bodies are released outside internal locks. + drop(discarded); + self.context.init_notify.notify_waiters(); + self.context.ready_notify.notify_waiters(); + self.context.wake.notify_waiters(); + self.initialization_gate_changed.notify_waiters(); + } + + /// Publish physical completion only after the opaque shutdown returns. + fn finish_stop(&self) { + self.context.stopped.store(true, Ordering::Release); self.context.init_notify.notify_waiters(); self.context.ready_notify.notify_waiters(); self.context.wake.notify_waiters(); self.initialization_gate_changed.notify_waiters(); } + fn admission_closed(&self) -> bool { + self.context.admission_closed.load(Ordering::Acquire) + } + fn is_stopped(&self) -> bool { self.context.stopped.load(Ordering::Acquire) } @@ -252,7 +322,7 @@ impl ModuleTransport { .initialization_gate .lock() .expect("initialization gate lock"); - if self.is_faulted() || self.is_stopped() { + if self.is_faulted() || self.admission_closed() { return Err(Error::ConnectionClosed); } match allowed { @@ -275,7 +345,7 @@ impl ModuleTransport { .initialization_gate .lock() .expect("initialization gate lock"); - if self.is_stopped() { + if self.admission_closed() { return Err(InitFailure::Closed); } self.context.init_started.store(true, Ordering::Release); @@ -327,7 +397,7 @@ impl ModuleTransport { }) .await; match result { - Ok(()) if self.is_stopped() => Err(Error::ConnectionClosed), + Ok(()) if self.admission_closed() => Err(Error::ConnectionClosed), Ok(()) => Ok(()), Err(InitFailure::Closed) => Err(Error::ConnectionClosed), Err(InitFailure::Failed(reason)) => Err(Error::transport(reason.clone())), @@ -339,7 +409,7 @@ impl ModuleTransport { let notified = self.context.ready_notify.notified(); if self.context.ready.load(Ordering::Acquire) || self.context.faulted.load(Ordering::Acquire) - || self.is_stopped() + || self.admission_closed() { return; } @@ -359,7 +429,7 @@ impl ModuleTransport { } pub(crate) async fn reinitialize(self: &Arc, bytes: crate::Secret) -> Result<()> { - if self.is_stopped() { + if self.admission_closed() { return Err(Error::ConnectionClosed); } let started = std::time::Instant::now(); @@ -371,7 +441,7 @@ impl ModuleTransport { .expect("static member name is valid"), timeout_ms: MODULE_INIT_DEADLINE.as_millis() as u64, })?; - if self.is_stopped() { + if self.admission_closed() { return Err(Error::ConnectionClosed); } let module = self @@ -432,7 +502,7 @@ impl ModuleTransport { pub(crate) async fn wait_initializing(&self) { loop { let notified = self.context.init_notify.notified(); - if self.init_started() || self.is_faulted() || self.is_stopped() { + if self.init_started() || self.is_faulted() || self.admission_closed() { return; } notified.await; @@ -440,7 +510,7 @@ impl ModuleTransport { } async fn deliver_now(&self, message: Message) -> Result<()> { - if self.is_faulted() || self.is_stopped() { + if self.is_faulted() || self.admission_closed() { return Err(Error::ConnectionClosed); } let bytes = serde_json::to_vec(&message)?; @@ -450,17 +520,24 @@ impl ModuleTransport { loop { let notified = self.context.wake.notified(); - if self.is_faulted() || self.is_stopped() { + if self.is_faulted() || self.admission_closed() { return Err(Error::ConnectionClosed); } let module = *self.module.lock().expect("module vtable lock"); let Some(module) = module else { return Err(Error::ConnectionClosed); }; + let lease = self.callbacks.enter()?; + // Stop closes this gate and drains reservations before teardown; + // a caller disappearing cannot refund a running native callback. let code = unsafe { (module.deliver)(module.module_ctx, bytes.as_ptr(), bytes.len()) }; + drop(lease); match code { TB_OK => return Ok(()), TB_BACKPRESSURE => { + if self.admission_closed() { + return Err(Error::ConnectionClosed); + } // A module that stops draining its queue and never calls // `wake` must not park this delivery task — or the caller // behind it — for the process lifetime. Expiry faults the @@ -471,7 +548,7 @@ impl ModuleTransport { .is_err() { let _pending = self.pending.lock().expect("module pending lock"); - if self.is_stopped() { + if self.admission_closed() { return Err(Error::ConnectionClosed); } self.context.faulted.store(true, Ordering::Release); @@ -496,7 +573,7 @@ impl ModuleTransport { async fn drain_pending(self: Arc) { self.wait_ready().await; loop { - while !self.is_faulted() && !self.is_stopped() { + while !self.is_faulted() && !self.admission_closed() { let message = self .pending .lock() @@ -511,7 +588,7 @@ impl ModuleTransport { return; } } - if self.is_faulted() || self.is_stopped() { + if self.is_faulted() || self.admission_closed() { self.drain_started.store(false, Ordering::Release); return; } @@ -542,7 +619,7 @@ impl ModuleTransport { async fn fault_pending(&self, failed: Message, error: Error) { let cleared = { let mut pending = self.pending.lock().expect("module pending lock"); - if self.is_stopped() && matches!(error, Error::ConnectionClosed) { + if self.admission_closed() && matches!(error, Error::ConnectionClosed) { pending.clear(); return; } @@ -596,18 +673,12 @@ impl ModuleTransport { if self.is_stopped() { return TB_CLOSED; } - { - let _gate = self - .initialization_gate - .lock() - .expect("initialization gate lock"); - if !self.init_started() && self.module.lock().expect("module vtable lock").is_none() { - self.finish_stop(); - } - } - let code = self.shutdown_sync(deadline); - // A caller timeout is not physical completion. Running initialization - // and readiness observers are closed only after the opaque callback returns. + let started = std::time::Instant::now(); + self.close_admission(); + // This is the retained physical worker, not the caller's timeout. Never + // tear down code underneath an admitted callback or abort its lease. + self.callbacks.wait_drained(); + let code = self.shutdown_sync(deadline.saturating_sub(started.elapsed())); self.finish_stop(); self.context .inbound @@ -674,13 +745,13 @@ impl From for Error { #[async_trait] impl Transport for ModuleTransport { async fn send(&self, message: Message) -> Result<()> { - if self.is_stopped() { + if self.admission_closed() { return Err(Error::ConnectionClosed); } if let Err(error) = self.ensure_initialized().await { { let _pending = self.pending.lock().expect("module pending lock"); - if self.is_stopped() && matches!(error, Error::ConnectionClosed) { + if self.admission_closed() && matches!(error, Error::ConnectionClosed) { return Err(Error::ConnectionClosed); } self.context.init_failed.store(true, Ordering::Release); @@ -712,7 +783,7 @@ impl Transport for ModuleTransport { let is_call = message.header.kind == MessageKind::MethodCall; { let mut pending = self.pending.lock().expect("module pending lock"); - if self.is_stopped() { + if self.admission_closed() { return Err(Error::ConnectionClosed); } if is_call { @@ -859,8 +930,10 @@ unsafe extern "C" fn host_broker_routing(ctx: *mut c_void) -> u32 { unsafe extern "C" fn host_ready(ctx: *mut c_void) { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { if let Some(context) = unsafe { ctx.cast::().as_ref() } { - context.ready.store(true, Ordering::Release); - context.ready_notify.notify_waiters(); + if !context.admission_closed.load(Ordering::Acquire) { + context.ready.store(true, Ordering::Release); + context.ready_notify.notify_waiters(); + } } })); } diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index e3f64641..97aa591d 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -991,7 +991,10 @@ async fn lifecycle_waiters_do_not_reenter_a_transport_disposed_by_the_prior_shut #[tokio::test] async fn fresh_signal_and_call_cannot_enter_an_actual_running_shutdown() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; let _release = ShutdownRelease; SHUTDOWN_BLOCKED.store(true, Ordering::Release); SHUTDOWN_ENTERED.store(false, Ordering::Release); @@ -1000,7 +1003,9 @@ async fn fresh_signal_and_call_cannot_enter_an_actual_running_shutdown() { let (transport, host) = ModuleTransport::new("admission".into(), vec![]); transport.defer_initialize(initialize_ok, host); transport.ensure_initialized().await.expect("A01"); - unsafe { (host.ready)(host.host_ctx); } + unsafe { + (host.ready)(host.host_ctx); + } let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); shutdown_entry().await; let mut signal = call(); @@ -1020,7 +1025,10 @@ async fn fresh_signal_and_call_cannot_enter_an_actual_running_shutdown() { #[tokio::test] async fn queued_readiness_and_backpressure_retire_before_actual_shutdown_returns() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; let _release = ShutdownRelease; SHUTDOWN_BLOCKED.store(true, Ordering::Release); SHUTDOWN_ENTERED.store(false, Ordering::Release); @@ -1030,11 +1038,20 @@ async fn queued_readiness_and_backpressure_retire_before_actual_shutdown_returns transport.send(call()).await.expect("A12"); let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); shutdown_entry().await; - tokio::time::timeout(Duration::from_millis(100), transport.wait_ready()).await.expect("A13"); - unsafe { (host.ready)(host.host_ctx); (host.wake)(host.host_ctx); } + tokio::time::timeout(Duration::from_millis(100), transport.wait_ready()) + .await + .expect("A13"); + unsafe { + (host.ready)(host.host_ctx); + (host.wake)(host.host_ctx); + } tokio::time::timeout(Duration::from_millis(100), async { - while transport.drain_started.load(Ordering::Acquire) { tokio::task::yield_now().await; } - }).await.expect("A14"); + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("A14"); let empty = transport.pending.lock().expect("A15").is_empty(); assert!(empty, "A16"); assert_eq!(DELIVERIES.load(Ordering::Acquire), 0, "A17"); @@ -1047,7 +1064,10 @@ async fn queued_readiness_and_backpressure_retire_before_actual_shutdown_returns #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn actual_signal_callback_is_drained_despite_stop_timeout_and_cancellation() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; let _release = ShutdownRelease; SHUTDOWN_BLOCKED.store(true, Ordering::Release); SHUTDOWN_ENTERED.store(false, Ordering::Release); @@ -1057,33 +1077,55 @@ async fn actual_signal_callback_is_drained_despite_stop_timeout_and_cancellation let (transport, host) = ModuleTransport::new("callback-drain".into(), vec![]); transport.defer_initialize(initialize_ok, host); transport.ensure_initialized().await.expect("A23"); - let mut signal = call(); signal.header.kind = MessageKind::Signal; + let mut signal = call(); + signal.header.kind = MessageKind::Signal; let sending_transport = transport.clone(); let sending = tokio::spawn(async move { sending_transport.send(signal).await }); tokio::time::timeout(Duration::from_secs(2), async { - while DELIVERIES.load(Ordering::Acquire) == 0 { tokio::task::yield_now().await; } - }).await.expect("A24"); + while DELIVERIES.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("A24"); assert_eq!(transport.inflight(), 0, "A25"); let stopping = tokio::spawn(transport.clone().stop(Duration::from_millis(50))); - assert!(matches!(stopping.await.expect("A26"), Err(StopError::Started(_))), "A27"); + assert!( + matches!(stopping.await.expect("A26"), Err(StopError::Started(_))), + "A27" + ); assert!(!SHUTDOWN_ENTERED.load(Ordering::Acquire), "A28"); assert!(!transport.is_stopped(), "A29"); assert!(transport.stop_task_spawned(), "A30"); - assert!(matches!(transport.clone().stop(Duration::from_millis(10)).await, Err(StopError::NotStarted(_))), "A31"); - assert!(matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), "A32"); + assert!( + matches!( + transport.clone().stop(Duration::from_millis(10)).await, + Err(StopError::NotStarted(_)) + ), + "A31" + ); + assert!( + matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), + "A32" + ); DELIVERY_BLOCKED.store(false, Ordering::Release); sending.await.expect("A33").expect("A34"); shutdown_entry().await; assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "A35"); SHUTDOWN_BLOCKED.store(false, Ordering::Release); - let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()).await.expect("A36"); + let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()) + .await + .expect("A36"); assert!(transport.is_stopped(), "A37"); assert!(!transport.is_faulted(), "A38"); } #[tokio::test] async fn installed_vtable_without_init_flag_closes_fresh_admission_before_teardown() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; let _release = ShutdownRelease; SHUTDOWN_BLOCKED.store(true, Ordering::Release); SHUTDOWN_ENTERED.store(false, Ordering::Release); @@ -1091,7 +1133,9 @@ async fn installed_vtable_without_init_flag_closes_fresh_admission_before_teardo DELIVERIES.store(0, Ordering::Release); let (transport, host) = ModuleTransport::new("installed-admission".into(), vec![]); let mut module = TbModuleVtable::default(); - unsafe { initialize_ok(&host, &mut module); } + unsafe { + initialize_ok(&host, &mut module); + } transport.initialize(module).expect("A39"); assert!(!transport.init_started(), "A40"); let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); @@ -1110,7 +1154,10 @@ async fn installed_vtable_without_init_flag_closes_fresh_admission_before_teardo #[tokio::test] async fn backpressure_retry_closes_without_fault_while_actual_shutdown_is_blocked() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; let _release = ShutdownRelease; SHUTDOWN_BLOCKED.store(true, Ordering::Release); SHUTDOWN_ENTERED.store(false, Ordering::Release); @@ -1119,16 +1166,28 @@ async fn backpressure_retry_closes_without_fault_while_actual_shutdown_is_blocke let (transport, host) = ModuleTransport::new("pressure-admission".into(), vec![]); transport.defer_initialize(initialize_ok, host); transport.send(call()).await.expect("A50"); - unsafe { (host.ready)(host.host_ctx); } + unsafe { + (host.ready)(host.host_ctx); + } tokio::time::timeout(Duration::from_secs(2), async { - while DELIVERIES.load(Ordering::Acquire) == 0 { tokio::task::yield_now().await; } - }).await.expect("A51"); + while DELIVERIES.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("A51"); let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); shutdown_entry().await; tokio::time::timeout(Duration::from_millis(100), async { - while transport.drain_started.load(Ordering::Acquire) { tokio::task::yield_now().await; } - }).await.expect("A52"); - unsafe { (host.wake)(host.host_ctx); } + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("A52"); + unsafe { + (host.wake)(host.host_ctx); + } assert_eq!(DELIVERIES.load(Ordering::Acquire), 1, "A53"); assert!(!transport.is_faulted(), "A54"); SHUTDOWN_BLOCKED.store(false, Ordering::Release); From 51492aca89b7dd160c9a11e6ddd913157571cb85 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 10:57:13 +0300 Subject: [PATCH 16/58] fix(module): serialize ready callback against transport shutdown A ready gate mutex now guards the host ready callback and the shutdown path so a late ready notification can no longer race past admission closure and mark a stopping transport as ready. The lifecycle tests were updated to drive shutdown concurrently and assert that reinitialization fails fast with ConnectionClosed instead of staying pending. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 4 ++++ crates/tinybus/src/module/transport_tests.rs | 22 ++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 8b2256a9..0ce40f78 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -38,6 +38,7 @@ struct HostContext { init_started: AtomicBool, init_notify: Notify, ready: AtomicBool, + ready_gate: StdMutex<()>, broker_routing: AtomicBool, ready_notify: Notify, inflight: AtomicUsize, @@ -170,6 +171,7 @@ impl ModuleTransport { init_started: AtomicBool::new(false), init_notify: Notify::new(), ready: AtomicBool::new(false), + ready_gate: StdMutex::new(()), broker_routing: AtomicBool::new(false), ready_notify: Notify::new(), inflight: AtomicUsize::new(0), @@ -286,6 +288,7 @@ impl ModuleTransport { let mut pending = self.pending.lock().expect("module pending lock"); let mut callbacks = self.callbacks.state.lock().expect("module callback lock"); callbacks.closed = true; + let _ready = self.context.ready_gate.lock().expect("module ready lock"); self.context.admission_closed.store(true, Ordering::Release); std::mem::take(&mut *pending) }; @@ -930,6 +933,7 @@ unsafe extern "C" fn host_broker_routing(ctx: *mut c_void) -> u32 { unsafe extern "C" fn host_ready(ctx: *mut c_void) { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { if let Some(context) = unsafe { ctx.cast::().as_ref() } { + let _ready = context.ready_gate.lock().expect("module ready lock"); if !context.admission_closed.load(Ordering::Acquire) { context.ready.store(true, Ordering::Release); context.ready_notify.notify_waiters(); diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 97aa591d..6970e230 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -928,8 +928,18 @@ async fn an_actual_delivery_failure_overlapping_stop_is_retained() { }) .await .expect("S64"); - transport.stop_sync(Duration::ZERO); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + tokio::time::timeout(Duration::from_secs(2), async { + while !transport.admission_closed() { + tokio::task::yield_now().await; + } + }) + .await + .expect("A57"); + assert!(!SHUTDOWN_ENTERED.load(Ordering::Acquire), "A58"); DELIVERY_BLOCKED.store(false, Ordering::Release); + stopping.await.expect("A59").expect("A60"); tokio::time::timeout(Duration::from_secs(2), async { while transport.drain_started.load(Ordering::Acquire) { tokio::task::yield_now().await; @@ -969,7 +979,10 @@ async fn lifecycle_waiters_do_not_reenter_a_transport_disposed_by_the_prior_shut "S74" ); assert!( - std::future::Future::poll(reinitializing.as_mut(), &mut context).is_pending(), + matches!( + std::future::Future::poll(reinitializing.as_mut(), &mut context), + std::task::Poll::Ready(Err(Error::ConnectionClosed)) + ), "S75" ); SHUTDOWN_BLOCKED.store(false, Ordering::Release); @@ -980,10 +993,7 @@ async fn lifecycle_waiters_do_not_reenter_a_transport_disposed_by_the_prior_shut ), "S76" ); - assert!( - matches!(reinitializing.await, Err(Error::ConnectionClosed)), - "S77" - ); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "S78"); assert!(transport.stop_task_spawned(), "S79"); assert!(!transport.is_faulted(), "S80"); From 4daf0d4fa1ab3afdc30fcc7c09b6e9d6243cd4f5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 10:58:31 +0300 Subject: [PATCH 17/58] test(module): cover reserved callback admission during teardown Add tests asserting that a reserved native entry cannot start after admission closes even when the stop caller is cancelled, and that unanswered method calls do not hold physical callback leases. Document the shutdown barrier that waits for reserved deliver callbacks before entering the opaque shutdown callback. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 105 ++++++++++++++++++- docs/modules/module/abi.md | 11 ++ 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 6970e230..43c8acaa 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1073,7 +1073,7 @@ async fn queued_readiness_and_backpressure_retire_before_actual_shutdown_returns } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn actual_signal_callback_is_drained_despite_stop_timeout_and_cancellation() { +async fn actual_signal_callback_is_drained_despite_stop_timeout() { let _lock = VTABLE_TEST_LOCK .get_or_init(|| tokio::sync::Mutex::new(())) .lock() @@ -1203,3 +1203,106 @@ async fn backpressure_retry_closes_without_fault_while_actual_shutdown_is_blocke SHUTDOWN_BLOCKED.store(false, Ordering::Release); stopping.await.expect("A55").expect("A56"); } + +#[tokio::test] +async fn a_reserved_native_entry_cannot_start_after_teardown_even_if_stop_caller_is_cancelled() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("reserved-admission".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("A61"); + // Pause at the real check-to-native-entry boundary: the reservation already + // belongs to the callback caller, but native entry has not happened yet. + let lease = transport.callbacks.enter().expect("A62"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + tokio::time::timeout(Duration::from_secs(2), async { + while !transport.admission_closed() { + tokio::task::yield_now().await; + } + }) + .await + .expect("A63"); + stopping.abort(); + assert!(stopping.await.expect_err("A64").is_cancelled(), "A65"); + assert!(!SHUTDOWN_ENTERED.load(Ordering::Acquire), "A66"); + assert!(!transport.is_stopped(), "A67"); + assert!(transport.callbacks.enter().is_err(), "A68"); + assert!( + matches!( + transport.ensure_initialized().await, + Err(Error::ConnectionClosed) + ), + "A69" + ); + assert!( + matches!( + transport.reinitialize(crate::Secret::new(vec![])).await, + Err(Error::ConnectionClosed) + ), + "A70" + ); + assert!( + matches!( + transport.clone().stop(Duration::from_millis(10)).await, + Err(StopError::NotStarted(_)) + ), + "A71" + ); + // An admitted callback can still enter before teardown; shutdown must wait + // for its physical exit, even after the outer stop future disappears. + let module = *transport.module.lock().expect("A72"); + let module = module.expect("A73"); + let bytes = serde_json::to_vec(&call()).expect("A74"); + assert_eq!( + unsafe { (module.deliver)(module.module_ctx, bytes.as_ptr(), bytes.len()) }, + TB_OK, + "A75" + ); + assert!(!SHUTDOWN_ENTERED.load(Ordering::Acquire), "A76"); + drop(lease); + shutdown_entry().await; + assert_eq!(DELIVERIES.load(Ordering::Acquire), 1, "A77"); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "A78"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()) + .await + .expect("A79"); + assert!(transport.is_stopped(), "A80"); + assert!(!transport.is_faulted(), "A81"); +} + +#[tokio::test] +async fn unanswered_method_calls_are_not_physical_callback_leases() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let (transport, host) = ModuleTransport::new("unanswered".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("A82"); + unsafe { + (host.ready)(host.host_ctx); + } + transport.send(call()).await.expect("A83"); + assert_eq!(transport.inflight(), 1, "A84"); + assert_eq!( + transport + .clone() + .stop(Duration::from_secs(2)) + .await + .expect("A85"), + TB_OK, + "A86" + ); + assert_eq!(transport.inflight(), 1, "A87"); + assert!(transport.is_stopped(), "A88"); +} diff --git a/docs/modules/module/abi.md b/docs/modules/module/abi.md index 30b0b2e7..e208f2cf 100644 --- a/docs/modules/module/abi.md +++ b/docs/modules/module/abi.md @@ -64,6 +64,17 @@ separate copy of Tokio's thread-locals, so it cannot borrow the host runtime. The host-to-module `deliver` callback only performs a bounded `try_send` and never blocks a host thread. Queue capacity is reported as backpressure and the host retries after the module's `wake` callback. +Before shutdown, the physical lifecycle owner closes initialization and delivery +admission and discards queued messages. It then waits for every reserved +`deliver` callback to return before entering the opaque shutdown callback. +Each backpressure retry requires a new reservation. This barrier covers actual +callback execution, including signals; it does not wait for unanswered method +calls or detached module work. A caller timeout or cancellation retains the +physical worker, callback reservations, and lifecycle lock. A callback that +never returns prevents teardown rather than allowing shutdown underneath it. +Ordinary admission closure is distinct from a module fault and from physical +shutdown completion. An already reported Ready snapshot remains until physical +completion; late `ready` callbacks cannot reopen admission. The symmetric module-to-host `send` callback applies bounded backpressure by blocking the calling module thread until host queue capacity is available. Modules must not call it from a thread that has to remain responsive. From 6e4afbd87c02b2d5748926d70fb8a440487cd32d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:23:14 +0300 Subject: [PATCH 18/58] docs: add plan for native terminal authority Document the approach for making the native terminal the source of truth, covering the design decisions and migration steps needed before implementation begins. Auto-committed-on: dragonfly Co-authored-by: Medulla --- docs/plans/native-terminal-authority.md | 287 ++++++++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 docs/plans/native-terminal-authority.md diff --git a/docs/plans/native-terminal-authority.md b/docs/plans/native-terminal-authority.md new file mode 100644 index 00000000..c073b246 --- /dev/null +++ b/docs/plans/native-terminal-authority.md @@ -0,0 +1,287 @@ +# Native instance terminal authority implementation plan + +Status: proposed implementation; root reviews before source implementation. +Baseline: 4daf0d4fa1ab3afdc30fcc7c09b6e9d6243cd4f5; canonical comparison 1f25576. +Branch security-bus-7328, existing upstream PR #57. Preserve complete PR scope: +actual callback drain, stop admission, physical lifecycle lock, original stop +task/transition markers and ordinary non-Fault stop semantics. This document is +not TinySec App #263 acceptance or issue #7328 completion. + +Use existing superproject worktree only. All shell commands explicitly cd to +/home/enamakel/work/openhuman/worktrees/security-bus-7328/vendor/tinybus. +No nested worktree, second SDK PR, release, gitlink advance or host workaround. +Auto-commit checkpoints stay separate; never squash. + +## Contract and bounds + +Add optional safe per-instance terminal factory in tinybus-module and a size-gated +ABI-v1 host query. Rust closures/traits/Arc/worker handles stay module-local. +Existing C shutdown owns serialized physical cleanup and actual join. + +Separate terminal intent, physical gate fence, worker join and backing last-Drop. +Callback future drain, Connection closure, host deadline and runtime shutdown +prove no independent worker quiescence. Already admitted Publish may complete; +next phase must deny after terminal intent. Module physical gate checks immutable +owner scope/generation/receiver, local flags, live native query, then local flags +again. Query success is admission linearization; later intent cannot undo an OS +operation already granted. Query alone is not a publication permit. + +Normal stop commits intent only after acquiring actual lifecycle lock, before +handing it to retained physical task. Canceled/unacquired stop makes no commitment. +Actual fault, rollback and Host Drop commit immediately, even behind opaque C init +or reinit. Fault takes public-state precedence; normal stop never becomes Faulted. +Fault cleanup must not create ordinary transition_from or restart terminal modules. + +ModuleTransport::new already leaks HostContext and stores a static reference; +loader never dlcloses. Copy normalized callback plus context, never borrowed +vtable/config storage. New view adds no context allocation/retain-release ABI. +Context is terminal tombstone after Host Drop, never recycled/reopened. Callback +contract covers retained independent worker uses after runtime shutdown. +Custom unsafe C hosts must truthfully uphold this lifetime/issuer contract; the +SDK cannot authenticate a malicious in-process host cryptographically. Payloads, +foreign metadata, serde IDs and cached Live receipts never construct authority. + +Existing process-wide context/RuntimeState/library residue is unbounded. This +addition must qualify that preexisting gap; no bounded aggregate claim follows. +If strict new native admission requires a global bound, separately review finite +process-lifetime context quota reserved before exposure, retained after exposure, +with bounded per-slot dynamic content. No speculative pool rewrite or slot reuse. +One initializer/cleanup per instance is a distinct bound from total load count. +No crypto, confidential backing codec or retirement protocol enters this slice. + +## Proposed safe Rust surface (new names, not existing API) + +New crates/tinybus-module/src/terminal.rs and terminal_tests.rs; lib.rs reexports: + + enum TerminalReason { Stop, Fault, SetupFailed, HostClosed } + enum TerminalCompletion { PhysicallyStopped, Uncertain } + struct NativeInstanceLiveness { inner: Arc } + struct RuntimeHooks { + request: Box, + finish: Box TerminalCompletion + Send>, + } + +Fields private. RuntimeHooks::new accepts those callback bounds plus static. +NativeInstanceLiveness is Clone, has is_live() -> bool and request_fault(). +No public constructor, serde, raw pointer access or arbitrary Connection enrollment. +request_fault first HostCalls::fault, then local monotonic fault plus bounded +request outside panic hook, registry mutex and module phase gate. Never joins. +Independent worker catch_unwind invokes it after unwinding out of physical gate. +Host-origin faults are observed immediately through query even while local request +is serialized behind opaque lifecycle code. + +New macro arm terminal_setup_factory = factory, config = C, with linked equivalent. +Safe factory takes NativeInstanceLiveness and returns Result<(F, RuntimeHooks)>. +F: Fn(Connection, C) -> Fut + Send + Sync + static; +Fut: Future> + Send + static. +Validate host/output/count/config and capability before factory evaluation. +Create TerminalState before factory; store one immutable controller after return. +Factory constructs controller but launches worker only during initial setup after +registration. Before return, factory owns partial-failure disposal. SDK owns it +after return. Same Arc serves setup/reinit; activation checks view before +publishing owner/service. No hook replacement or terminal reset on reinit. + +Private TerminalState holds copied HostCalls, monotonic AtomicU32 intent, +AtomicBool request/finish claims, mutex Option, atomic completion. +Extract hook under lock, release lock, catch_unwind invoking. Poison recovery +must not create a second panic. Request panic closes/faults, no retry/refund; +finish panic leaves Uncertain. Callback is contractually bounded/no I/O/no join/ +no reentrant SDK calls: arbitrary trusted Rust cannot be forcibly made bounded. +Physical finish joins actual controller-owned worker and retains charged resources +until owner-defined actual destruction. SDK never infers destruction from counters. + +## Task 1: additive independently checked ABI tail + +Files: crates/tinybus/src/module/abi.rs, abi_tests.rs; +crates/tinybus-module/src/lib_tests.rs and adjusted host-vtable literals. + +Write failing tests first: routing_tail_survives_newer_optional_tail, +terminal_tail_requires_its_complete_field, prefix_hosts_normalize_both_tails. +Actual repr(C) prefix and old-routing structs allocate only declared bytes. +Cover null, below prefix, exact prefix, partial routing, complete old routing, +partial new callback, full None/Some, and larger storage. Run focused tests red. + +Append native_terminal_state: Option u32>. +ABI constants Live=1, StopIntent=2, Fault=4, PhysicallyStopped=8. Exactly 1 admits; +zero/unknown/combined/terminal flags deny. Terminal flags monotonic, never Live. +Physical bit means successful opaque shutdown return, and for opted-in modules +explicit finish proved worker join. It proves neither legacy worker completion +nor receiver backing destruction. Fault/uncertain shutdown never earns this bit. + +Normalization independently checks offset_of(field) + sizeof(field type) for +broker_routing and new callback; never sizeof enlarged whole vtable for old tail. +Frozen prefix/base size and ABI version stay unchanged. Document stronger callback +lifetime covering every retained view use. Run focused tests green. + +## Task 2: retain real deferred initialization before deadline abandonment + +Files: crates/tinybus/src/module/transport.rs and transport_tests.rs, +host.rs/host_tests.rs, host_callbacks.rs/host_callbacks_tests.rs. + +Existing DeferredInitializer::run returns SendModuleVtable through spawn_blocking; +ensure_initialized timeout drops JoinHandle, so eventual vtable can disappear +before self.initialize validates/stores it. Current stop_sync rollback can also +overlap opaque init. Necessary correction stays in these exact paths. + +First red test late_initialization_retains_real_vtable_for_cleanup: actual blocked +initializer, controlled host future timeout/drop, retained native worker/query, +release init returning valid vtable, exactly one real shutdown/join, no new phase. +Add rollback_cannot_overlap_opaque_initialization and late_invalid_vtable_never_runs. + +Retained blocking initializer owns Arc transport, actual owned lifecycle guard, +DeferredInitializer (and config borrow), resident library assumptions and eventual +vtable. Install via existing initialize validation before result publication to +separate waiter channel; JoinHandle result carries no exclusive physical owner. +Validate size/header/version/callback validity under existing initialize contract; +never invoke invalid/missing callbacks or fabricate a controller from partial ABI. +Failure with no usable returned vtable retains honest uncertainty for a foreign +initializer that launched work without returning cleanup; native query still denies. + +Host deadline/drop does not drop result owner or borrow. Timeout commits native +fault intent immediately, makes public failed/fault state terminal, and retains +one eventual cleanup. Clear config only after actual C return. Publish validated +vtable even when terminal, then release lifecycle guard and claim/schedule cleanup +once. Terminal vtable storage is cleanup access, never public resurrection. +No new initializer retries or unbounded spawn_blocking loops: one physical init, +one claim for cleanup; caller retries only observe same result/tombstone. +If init never returns, cleanup pending and real ownership survive indefinitely. + +## Task 3: actual host flags, fault cleanup and original stop invariants + +Same host files. Red tests fault_closes_native_phase_before_reinitializer_returns, +unacquired_stop_keeps_native_instance_live, abandoned_stop_keeps_original_cleanup, +host_drop_leaves_terminal_query_tombstone, and +fault_during_stop_keeps_fault_transition_and_single_shutdown. + +HostContext adds AtomicU32 terminal flags; commit_terminal clears Live and sets +StopIntent plus Fault when real fault. Query only Acquire loads leaked actual ctx, +no locks/allocation/parsing/product code. Install callback in ModuleTransport::new. +HostContext immutable Weak from Arc::new_cyclic enables host_fault +to schedule cleanup without retaining strong context cycle. + +Separate physical cleanup claim from existing stop_task_spawned. Normal stop +acquires lifecycle then claims/commits before spawn. Fault immediately commits, +then private schedule_terminal_cleanup retains Arc and queues one serialized +physical cleanup. Retained std thread using blocking_lock_owned permits Host Drop +cleanup independent of dropped Tokio runtime; never recurse into C shutdown from +host_fault, opaque callback, router/loaded lock or Tokio worker. Thread spawn failure +keeps terminal/uncertain ownership; no successful completion/reopen/refund. +Normal running task and fault share claim, never second join. stop_task_spawned +still denotes actual ordinary stop task, preserving StopError Started/NotStarted. + +Route host_fault, fault_pending, failed initialization/send, admission rollback +and Host Drop through same intent/claim. Gather cloned transports under lock, +commit intent before detach, release loaded/router/inbound locks before scheduling. +Remove synchronous rollback shutdown under lifecycle-bypassing Drop path. +Before-init rollback closes init gate without pretending worker joined. + +peer_detached checks actual fault before consuming normal transition_from; +private fault cleanup never creates that origin. Late stop_inner success preserves +actual Faulted/Failed precedence. Ordinary finish_stop stays non-fault. Keep every +existing #57 callback lease/drain/cancel/retry/original marker assertion unchanged. + +## Task 4: module-local hooks through runtime and fault paths + +Files: new terminal.rs/terminal_tests.rs/lib_terminal_tests.rs in tinybus-module; +lib.rs, lib_setup_factory_tests.rs, lib_lifecycle_tests.rs. +Declare sibling test modules, no inline tests. + +Red tests unavailable query rejects before factory; two instances isolated; +retained view after runtime shutdown; host fault precedes local request; +request/finish once; poisoned hook storage; finish panic stays uncertain; +terminal reinit never reruns setup; retained service refs cannot suppress cleanup. + +Implement API above. New private start_terminal_reconfigurable_module_factory_mode +and doc-hidden linked/unlinked entrypoints serve macro arms. Existing entrypoints +retain old behavior. start_module_runtime takes optional Arc; +transport, RuntimeState and setup monitor share it, existing callers pass None. +shutdown requests Stop before callback disposal, releases reinitializer mutex, +then runs finish synchronously on retained physical host cleanup worker before +runtime.shutdown_timeout. Indefinite join outlives deadline without TB_OK. +Uncertain/panic finish returns content-free failure, never physical completion. + +Caught method panic publishes host fault even if error-frame send fails, then +bounded request outside registry lock. Determine actual send lock boundary before +coding: if lock is held, retained controller task runs request after host intent. +Setup failure publishes host intent before waiting callback disposal, request and +finish run off runtime workers. Setup canceled by shutdown takes same once route. +No joining/hooks in global panic hook (location logging only). setup/reinit reject +terminal immediately and before activation. Never classify ordinary close as fault. + +## Task 5: actual compiled native proof and compatibility negatives + +New declared cdylibs crates/tinybus/examples/module_terminal.rs and +module_legacy_prefix.rs, explicit examples in crates/tinybus/Cargo.toml. +Legacy fixture uses locally frozen repr(C) prefix, not enlarged current layout. +Extend scripts/check-coverage.sh build/env validation and host_tests.rs loader +tests. Use existing staged_module hash allowlist/private copies; separate mapping +per test avoids module-global tracing collision. Include all fixtures in packages. + +Fixed test interface controls worker phase, held Publish, held reinit/init, +queued jobs, worker panic, failure before/after launch, receiver retention/Drop. +Use std channel/Condvar barriers and event acknowledgements, never sleeps. +After fault bus detaches: test-only exported C control functions, bound from the +actual staged library during init, release barriers and read actual completion. +Do not make production liveness constructors or payload control authority. + +Required red/green actual compiled tests: + +1. Hold opaque C reinit/init; actual host fault denies independent worker's next + physical phase before releasing opaque call. Cleanup stays pending, then once. +2. Old Publish lease survives stop timeout/caller cancellation; new phase denies; + original callback drain/task/lock and join retained. Release proves real join. + Retained receiver last-Drop is separate and never inferred from future drain. +3. Actual method/worker panic and poisoned bounded local gate: one close/finish, + denied queued callers, no blocking panic hook/reopen/refund. +4. Init/factory failure before launch, after registration/launch, rollback and + eventual valid vtable after abandoned waiter preserve real cleanup ownership. +5. Retained actual view after Host Drop/runtime shutdown queries terminal tombstone; + two instance isolation, no stale reuse or terminal reset on reinit/retry. +6. Compiled old module/new host; new terminal module/actual compiled prefix-only + host rejects before factory, ordinary legacy setup still loads. Truncated and + tampered prefix negatives must allocate genuinely frozen layout. JSON Live, + foreign metadata/serialized IDs cannot fabricate query or admission. + +Fixtures prove SDK lifecycle only, not TinySec charge or confidential last-Drop. + +## Task 6: docs, declarations and complete proof gates + +Update docs/modules/module/abi.md and README.md with actual optional tail/lifetime, +factory example, bounded request, physical join, post-runtime tombstone, legacy +denial and preexisting unbounded residue. Public Rust docs match actual API. + +Long gates use absolute superproject scripts/ci-cancel-aware.sh from explicit SDK +cwd. Never export CARGO_TARGET_DIR or send builds into temp directories. +Run focused red/green first and retain commands/full outputs/exit status: + +- cargo fmt --all -- --check +- cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +- cargo test --locked --workspace --all-features; preserve baseline 557 assertions +- cargo check --locked --workspace --all-targets --all-features +- cargo check --locked --package tinybus --no-default-features +- scripts/test-features.sh and Rust 1.85.0 enabled/disabled all-target checks +- cargo test --locked --workspace --doc --all-features +- Build all declared cdylibs, execute native ignored tests with explicit fixture + paths from both SDK cwd and superproject cwd via manifest path; fresh staging +- Pack tinybus/macros/module with compilation verification and real local registry + dependency strategy; --no-verify alone is not package compile proof +- Fresh scripts/check-coverage.sh, at least 90% meaningful compiled file lines, + no new ignore regex. Separate llvm-cov JSON verifies at least 80% actual changed + executable lines versus current canonical and full PR base, including excluded + host integration code in a dedicated report; no global percent substitution +- OpenHuman rust:layout/crate-chain/feature forwarding with native/module ABI off + and on, complete fixture/declaration checks and actual old/new ABI execution + +Review exact final diff and independently verify fresh proof before root updates +existing upstream #57 ready for review. No merge/release/pin/issue closure granted. + +## Source-backed review decisions before implementation + +This is an exact interface/ownership/TDD plan, not a fabricated complete source +replacement. Prove init/normal-stop/fault cleanup-claim handoff, actual method-panic +registry boundary, thread-spawn failure retention and invalid-init-vtable contract. +If source makes these contradictory, escalate exact conflict before green code; +never invent quiescence or silently weaken immediate native fault denial. +Resolve separately whether bounded global native admission is required: leaked +context pointer safety cannot satisfy it. No additional ABI lifetime allocation +is necessary for the approved actual SDK context contract. From 7e79fe5403f9546ce74a6a83a801beeabb4f345d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:29:06 +0300 Subject: [PATCH 19/58] feat(module): add optional native terminal state to host vtable Extend TbHostVtable with an optional native_terminal_state callback appended after broker_routing, leaving it unset when normalizing older hosts. Tests cover the frozen prefix and routing tail so the new field cannot be read from a truncated allocation. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_lifecycle_tests.rs | 1 + crates/tinybus-module/src/lib_tests.rs | 6 +- crates/tinybus/src/module/abi.rs | 3 + crates/tinybus/src/module/abi_tests.rs | 93 +++++++++++++++++++ crates/tinybus/src/module/transport.rs | 1 + 5 files changed, 103 insertions(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib_lifecycle_tests.rs b/crates/tinybus-module/src/lib_lifecycle_tests.rs index 89756fc8..c3b9a1d4 100644 --- a/crates/tinybus-module/src/lib_lifecycle_tests.rs +++ b/crates/tinybus-module/src/lib_lifecycle_tests.rs @@ -23,6 +23,7 @@ fn host() -> TbHostVtable { }, ready, broker_routing: None, + native_terminal_state: None, } } struct DropCounter(Arc); diff --git a/crates/tinybus-module/src/lib_tests.rs b/crates/tinybus-module/src/lib_tests.rs index 8892d848..294cac27 100644 --- a/crates/tinybus-module/src/lib_tests.rs +++ b/crates/tinybus-module/src/lib_tests.rs @@ -74,6 +74,7 @@ fn host(config: &[u8]) -> TbHostVtable { }, ready: host_ready, broker_routing: None, + native_terminal_state: None, } } @@ -739,6 +740,7 @@ fn an_older_host_prefix_is_copied_without_reading_the_additive_tail() { let ptr = std::ptr::from_ref(&prefix).cast::(); let normalized = unsafe { TbHostVtable::read_compatible(ptr) }.unwrap(); assert!(normalized.broker_routing.is_none()); + assert!(normalized.native_terminal_state.is_none()); assert_eq!( unsafe { parse_config::(ptr) }.unwrap(), serde_json::json!({}) @@ -752,9 +754,11 @@ fn an_older_host_prefix_is_copied_without_reading_the_additive_tail() { #[test] fn an_incomplete_host_tail_never_reads_the_callback() { let mut current = host(b"{}"); - current.size = size_of::() as u32 - 1; + current.size = (std::mem::offset_of!(TbHostVtable, broker_routing) + + size_of:: u32>>() - 1) as u32; let normalized = unsafe { TbHostVtable::read_compatible(¤t) }.unwrap(); assert!(normalized.broker_routing.is_none()); + assert!(normalized.native_terminal_state.is_none()); assert!(unsafe { TbHostVtable::read_compatible(std::ptr::null()) }.is_none()); current.size = tinybus::module::abi::TB_HOST_VTABLE_BASE_SIZE - 1; assert!(unsafe { TbHostVtable::read_compatible(¤t) }.is_none()); diff --git a/crates/tinybus/src/module/abi.rs b/crates/tinybus/src/module/abi.rs index b9ffeecb..70581ff0 100644 --- a/crates/tinybus/src/module/abi.rs +++ b/crates/tinybus/src/module/abi.rs @@ -166,6 +166,8 @@ pub struct TbHostVtable { /// The SDK trusts this callback under the unsafe module-init host contract; /// it is not authentication of arbitrary user-provided C vtables. pub broker_routing: Option u32>, + /// Optional native per-instance terminal state query. + pub native_terminal_state: Option u32>, } /// Size required from an older host before reading its frozen prefix. @@ -206,6 +208,7 @@ impl TbHostVtable { config: base.config, ready: base.ready, broker_routing, + native_terminal_state: None, }) } } diff --git a/crates/tinybus/src/module/abi_tests.rs b/crates/tinybus/src/module/abi_tests.rs index 6ebb313a..c2189932 100644 --- a/crates/tinybus/src/module/abi_tests.rs +++ b/crates/tinybus/src/module/abi_tests.rs @@ -41,3 +41,96 @@ fn an_uninitialized_module_vtable_refuses_calls() { TB_CLOSED ); } + +// These are the original ABI allocations, not truncated new vtables. Keeping +// their declarations independent detects accidental prefix or tail growth. +#[repr(C)] +struct FrozenHostPrefix { + size: u32, + reserved: u32, + ctx: *mut c_void, + send: unsafe extern "C" fn(*mut c_void, *const u8, usize) -> i32, + wake: unsafe extern "C" fn(*mut c_void), + log: unsafe extern "C" fn(*mut c_void, u32, *const u8, usize), + fault: unsafe extern "C" fn(*mut c_void, *const u8, usize), + config: TbSlice, + ready: unsafe extern "C" fn(*mut c_void), +} + +#[repr(C)] +struct FrozenRoutingHost { + prefix: FrozenHostPrefix, + routing: Option u32>, +} + +unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { TB_OK } +unsafe extern "C" fn wake(_: *mut c_void) {} +unsafe extern "C" fn log(_: *mut c_void, _: u32, _: *const u8, _: usize) {} +unsafe extern "C" fn fault(_: *mut c_void, _: *const u8, _: usize) {} +unsafe extern "C" fn query(ctx: *mut c_void) -> u32 { unsafe { ctx.cast::().read() } } + +fn frozen_prefix(size: usize) -> FrozenHostPrefix { + FrozenHostPrefix { + size: size as u32, reserved: 0, ctx: std::ptr::null_mut(), send, wake, log, fault, + config: TbSlice { ptr: std::ptr::null(), len: 0 }, ready: wake, + } +} + +#[test] +fn routing_tail_survives_newer_optional_tail() { + let value = 1u32; + let mut old = FrozenRoutingHost { + prefix: frozen_prefix(size_of::()), routing: Some(query), + }; + old.prefix.ctx = std::ptr::from_ref(&value).cast_mut().cast(); + let normalized = unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&old).cast()) }.unwrap(); + let routing = normalized.broker_routing.expect("complete original routing tail is retained"); + assert_eq!(unsafe { routing(normalized.host_ctx) }, 1); + assert!(normalized.native_terminal_state.is_none()); +} + +#[test] +fn prefix_hosts_normalize_both_tails() { + assert_eq!(ABI_REVISION, 1); + assert_eq!(size_of::(), TB_HOST_VTABLE_BASE_SIZE as usize); + assert_eq!(size_of::(), std::mem::offset_of!(TbHostVtable, broker_routing)); + assert!(unsafe { TbHostVtable::read_compatible(std::ptr::null()) }.is_none()); + // Only a u32 allocation is present for the rejected short header. + let short = TB_HOST_VTABLE_BASE_SIZE - 1; + assert!(unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&short).cast()) }.is_none()); + let prefix = frozen_prefix(size_of::()); + let normalized = unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&prefix).cast()) }.unwrap(); + assert!(normalized.broker_routing.is_none()); + assert!(normalized.native_terminal_state.is_none()); +} + +#[test] +fn terminal_tail_requires_its_complete_field() { + // Byte allocations are sized to the declared boundary and aligned to the + // vtable. No reference to a full vtable is created from shorter storage. + let routing_end = std::mem::offset_of!(TbHostVtable, broker_routing) + + size_of:: u32>>(); + let terminal_end = std::mem::offset_of!(TbHostVtable, native_terminal_state) + + size_of:: u32>>(); + for size in [routing_end - 1, routing_end, terminal_end - 1, terminal_end, terminal_end + 16] { + for terminal_present in [false, true] { + let layout = std::alloc::Layout::from_size_align(size, align_of::()).unwrap(); + let storage = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!storage.is_null()); + unsafe { storage.cast::().write(frozen_prefix(size)) }; + if size >= routing_end { + unsafe { storage.add(std::mem::offset_of!(TbHostVtable, broker_routing)) + .cast:: u32>>().write(Some(query)) }; + } + if size >= terminal_end { + unsafe { storage.add(std::mem::offset_of!(TbHostVtable, native_terminal_state)) + .cast:: u32>>() + .write(terminal_present.then_some(query)) }; + } + let normalized = unsafe { TbHostVtable::read_compatible(storage.cast()) }.unwrap(); + unsafe { std::alloc::dealloc(storage, layout) }; + assert_eq!(normalized.broker_routing.is_some(), size >= routing_end, "routing size {size}"); + assert_eq!(normalized.native_terminal_state.is_some(), size >= terminal_end && terminal_present, "terminal size {size}"); + } + } +} diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 0ce40f78..f7c38d62 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -209,6 +209,7 @@ impl ModuleTransport { config: config_slice, ready: host_ready, broker_routing: Some(host_broker_routing), + native_terminal_state: None, }; (transport, vtable) } From 88031728501f633cc2d82c21bf1a14a26f4f3b80 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:30:14 +0300 Subject: [PATCH 20/58] fix(module): read each optional host vtable tail field independently The host vtable reader previously copied the native terminal state callback only when the declared size covered the whole struct, so a host that supplied that field without any later tail lost it. Each optional callback is now gated on its own complete-field size check, and the terminal state bits are documented as constants with their admission and monotonicity rules. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_tests.rs | 3 +- crates/tinybus/src/module/abi.rs | 57 +++++++++++++-- crates/tinybus/src/module/abi_tests.rs | 96 ++++++++++++++++++++------ 3 files changed, 129 insertions(+), 27 deletions(-) diff --git a/crates/tinybus-module/src/lib_tests.rs b/crates/tinybus-module/src/lib_tests.rs index 294cac27..6db79c50 100644 --- a/crates/tinybus-module/src/lib_tests.rs +++ b/crates/tinybus-module/src/lib_tests.rs @@ -755,7 +755,8 @@ fn an_older_host_prefix_is_copied_without_reading_the_additive_tail() { fn an_incomplete_host_tail_never_reads_the_callback() { let mut current = host(b"{}"); current.size = (std::mem::offset_of!(TbHostVtable, broker_routing) - + size_of:: u32>>() - 1) as u32; + + size_of:: u32>>() + - 1) as u32; let normalized = unsafe { TbHostVtable::read_compatible(¤t) }.unwrap(); assert!(normalized.broker_routing.is_none()); assert!(normalized.native_terminal_state.is_none()); diff --git a/crates/tinybus/src/module/abi.rs b/crates/tinybus/src/module/abi.rs index 70581ff0..1d8d41cb 100644 --- a/crates/tinybus/src/module/abi.rs +++ b/crates/tinybus/src/module/abi.rs @@ -3,6 +3,7 @@ //! ABI revision one grows through optional vtable tails. Read the frozen prefix //! first and check its declared size before reading a tail; an older host has no //! broker-routing callback and therefore supplies no authenticated call context. +//! Each optional field has its own complete-field size check. use std::ffi::c_void; @@ -28,6 +29,20 @@ pub const TB_TIMEOUT: i32 = -4; /// A pointer, length, or frame was invalid. pub const TB_BAD_ARGUMENT: i32 = -5; +/// Exactly this value permits native per-instance admission. Zero, unknown +/// bits, or any combination with terminal bits must deny admission. A sampled +/// value is not a reusable permit for a later operation. +pub const TB_NATIVE_TERMINAL_LIVE: u32 = 1; +/// Terminal stop intent; once committed, the instance can never become live. +pub const TB_NATIVE_TERMINAL_STOP_INTENT: u32 = 2; +/// Actual fault intent, with public fault state taking precedence over stop. +pub const TB_NATIVE_TERMINAL_FAULT: u32 = 4; +/// Successful opaque shutdown return. For modules opting into native terminal +/// hooks, successful finish must also prove their independent worker joined. +/// This bit proves neither legacy worker completion nor receiver backing +/// destruction. Faulted or uncertain shutdown must never set it. +pub const TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED: u32 = 8; + /// Revision-one module initialization entrypoint. pub type TbModuleInit = unsafe extern "C" fn(*const TbHostVtable, *mut TbModuleVtable) -> i32; @@ -166,7 +181,24 @@ pub struct TbHostVtable { /// The SDK trusts this callback under the unsafe module-init host contract; /// it is not authentication of arbitrary user-provided C vtables. pub broker_routing: Option u32>, - /// Optional native per-instance terminal state query. + /// Optional live query for this native module instance, read only when + /// `size` covers this entire field. Absence means unsupported and must deny + /// native admission. Only exactly [`TB_NATIVE_TERMINAL_LIVE`] permits it; + /// zero, unknown bits, combined values and terminal flags deny. Terminal + /// intent is monotonic and must never return to Live. + /// + /// Modules copy this callback and `host_ctx`, never retain the borrowed + /// vtable or config storage. The host must keep the copied callback and + /// context valid and thread-safe for every retained view use, including + /// independent workers after runtime shutdown and host destruction. A + /// destroyed host leaves a terminal tombstone, never recycles or reopens + /// this context. Custom unsafe hosts must truthfully uphold this contract; + /// the SDK cannot authenticate an arbitrary in-process C host. + /// + /// Query success linearizes admission only; it neither grants publication + /// authority nor cancels an operation already admitted before later intent. + /// The SDK's existing process-lifetime context retention is unbounded; + /// this optional callback adds no aggregate resource bound. pub native_terminal_state: Option u32>, } @@ -178,12 +210,16 @@ impl TbHostVtable { /// /// Returns `None` for a null pointer or a size below /// [`TB_HOST_VTABLE_BASE_SIZE`]. A prefix-only or incomplete-tail host is - /// accepted with `broker_routing: None`; only a full tail is copied. + /// accepted. Each absent or incomplete optional callback normalizes to + /// `None`, independently of any later tail field. /// /// # Safety /// A non-null `host` must be aligned and readable for its first size field /// and every byte that size declares. Its callbacks must be valid and - /// thread-safe, and their context must outlive the module runtime. An + /// thread-safe, and their context must outlive the module runtime. If + /// `native_terminal_state` is supplied, its copied callback and context + /// must remain valid for every retained view use, even after runtime + /// shutdown or host destruction, as documented on that field. An /// affirmative routing callback (returning exactly 1) must truthfully /// guarantee exclusive broker delivery at the time each frame is queued. #[doc(hidden)] @@ -192,8 +228,17 @@ impl TbHostVtable { return None; } let base = unsafe { host.cast::().read() }; - let broker_routing = if base.size as usize >= size_of::() { - unsafe { std::ptr::addr_of!((*host).broker_routing).read() } + let query_size = size_of:: u32>>(); + let broker_routing = + if base.size as usize >= std::mem::offset_of!(Self, broker_routing) + query_size { + unsafe { std::ptr::addr_of!((*host).broker_routing).read() } + } else { + None + }; + let native_terminal_state = if base.size as usize + >= std::mem::offset_of!(Self, native_terminal_state) + query_size + { + unsafe { std::ptr::addr_of!((*host).native_terminal_state).read() } } else { None }; @@ -208,7 +253,7 @@ impl TbHostVtable { config: base.config, ready: base.ready, broker_routing, - native_terminal_state: None, + native_terminal_state, }) } } diff --git a/crates/tinybus/src/module/abi_tests.rs b/crates/tinybus/src/module/abi_tests.rs index c2189932..ddccf43d 100644 --- a/crates/tinybus/src/module/abi_tests.rs +++ b/crates/tinybus/src/module/abi_tests.rs @@ -63,16 +63,30 @@ struct FrozenRoutingHost { routing: Option u32>, } -unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { TB_OK } +unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { + TB_OK +} unsafe extern "C" fn wake(_: *mut c_void) {} unsafe extern "C" fn log(_: *mut c_void, _: u32, _: *const u8, _: usize) {} unsafe extern "C" fn fault(_: *mut c_void, _: *const u8, _: usize) {} -unsafe extern "C" fn query(ctx: *mut c_void) -> u32 { unsafe { ctx.cast::().read() } } +unsafe extern "C" fn query(ctx: *mut c_void) -> u32 { + unsafe { ctx.cast::().read() } +} fn frozen_prefix(size: usize) -> FrozenHostPrefix { FrozenHostPrefix { - size: size as u32, reserved: 0, ctx: std::ptr::null_mut(), send, wake, log, fault, - config: TbSlice { ptr: std::ptr::null(), len: 0 }, ready: wake, + size: size as u32, + reserved: 0, + ctx: std::ptr::null_mut(), + send, + wake, + log, + fault, + config: TbSlice { + ptr: std::ptr::null(), + len: 0, + }, + ready: wake, } } @@ -80,11 +94,15 @@ fn frozen_prefix(size: usize) -> FrozenHostPrefix { fn routing_tail_survives_newer_optional_tail() { let value = 1u32; let mut old = FrozenRoutingHost { - prefix: frozen_prefix(size_of::()), routing: Some(query), + prefix: frozen_prefix(size_of::()), + routing: Some(query), }; old.prefix.ctx = std::ptr::from_ref(&value).cast_mut().cast(); - let normalized = unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&old).cast()) }.unwrap(); - let routing = normalized.broker_routing.expect("complete original routing tail is retained"); + let normalized = + unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&old).cast()) }.unwrap(); + let routing = normalized + .broker_routing + .expect("complete original routing tail is retained"); assert_eq!(unsafe { routing(normalized.host_ctx) }, 1); assert!(normalized.native_terminal_state.is_none()); } @@ -92,14 +110,21 @@ fn routing_tail_survives_newer_optional_tail() { #[test] fn prefix_hosts_normalize_both_tails() { assert_eq!(ABI_REVISION, 1); - assert_eq!(size_of::(), TB_HOST_VTABLE_BASE_SIZE as usize); - assert_eq!(size_of::(), std::mem::offset_of!(TbHostVtable, broker_routing)); + assert_eq!( + size_of::(), + TB_HOST_VTABLE_BASE_SIZE as usize + ); + assert_eq!( + size_of::(), + std::mem::offset_of!(TbHostVtable, broker_routing) + ); assert!(unsafe { TbHostVtable::read_compatible(std::ptr::null()) }.is_none()); // Only a u32 allocation is present for the rejected short header. let short = TB_HOST_VTABLE_BASE_SIZE - 1; assert!(unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&short).cast()) }.is_none()); let prefix = frozen_prefix(size_of::()); - let normalized = unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&prefix).cast()) }.unwrap(); + let normalized = + unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&prefix).cast()) }.unwrap(); assert!(normalized.broker_routing.is_none()); assert!(normalized.native_terminal_state.is_none()); } @@ -112,25 +137,56 @@ fn terminal_tail_requires_its_complete_field() { + size_of:: u32>>(); let terminal_end = std::mem::offset_of!(TbHostVtable, native_terminal_state) + size_of:: u32>>(); - for size in [routing_end - 1, routing_end, terminal_end - 1, terminal_end, terminal_end + 16] { + let state = TB_NATIVE_TERMINAL_LIVE; + for size in [ + routing_end - 1, + routing_end, + terminal_end - 1, + terminal_end, + terminal_end + 16, + ] { for terminal_present in [false, true] { - let layout = std::alloc::Layout::from_size_align(size, align_of::()).unwrap(); + let layout = + std::alloc::Layout::from_size_align(size, align_of::()).unwrap(); let storage = unsafe { std::alloc::alloc_zeroed(layout) }; assert!(!storage.is_null()); - unsafe { storage.cast::().write(frozen_prefix(size)) }; + let mut prefix = frozen_prefix(size); + prefix.ctx = std::ptr::from_ref(&state).cast_mut().cast(); + unsafe { storage.cast::().write(prefix) }; if size >= routing_end { - unsafe { storage.add(std::mem::offset_of!(TbHostVtable, broker_routing)) - .cast:: u32>>().write(Some(query)) }; + unsafe { + storage + .add(std::mem::offset_of!(TbHostVtable, broker_routing)) + .cast:: u32>>() + .write(Some(query)) + }; } if size >= terminal_end { - unsafe { storage.add(std::mem::offset_of!(TbHostVtable, native_terminal_state)) - .cast:: u32>>() - .write(terminal_present.then_some(query)) }; + unsafe { + storage + .add(std::mem::offset_of!(TbHostVtable, native_terminal_state)) + .cast:: u32>>() + .write(terminal_present.then_some(query)) + }; } let normalized = unsafe { TbHostVtable::read_compatible(storage.cast()) }.unwrap(); unsafe { std::alloc::dealloc(storage, layout) }; - assert_eq!(normalized.broker_routing.is_some(), size >= routing_end, "routing size {size}"); - assert_eq!(normalized.native_terminal_state.is_some(), size >= terminal_end && terminal_present, "terminal size {size}"); + assert_eq!( + normalized.broker_routing.is_some(), + size >= routing_end, + "routing size {size}" + ); + assert_eq!( + normalized.native_terminal_state.is_some(), + size >= terminal_end && terminal_present, + "terminal size {size}" + ); + if let Some(query) = normalized.native_terminal_state { + assert_eq!( + unsafe { query(normalized.host_ctx) }, + TB_NATIVE_TERMINAL_LIVE + ); + } } } } From 7538e50f069f223c20faff8b037285640ae850be Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:30:52 +0300 Subject: [PATCH 21/58] docs(module): document optional host capability tails Adds documentation for the optional ABI capability tails, covering how optional fields are gated by size, the native query contract and its admission rules, the meaning of PhysicallyStopped, and the lifetime and thread-safety obligations a host must uphold for the query callback and context. Auto-committed-on: dragonfly Co-authored-by: Medulla --- docs/modules/module/abi.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/modules/module/abi.md b/docs/modules/module/abi.md index e208f2cf..11927a5f 100644 --- a/docs/modules/module/abi.md +++ b/docs/modules/module/abi.md @@ -25,6 +25,39 @@ The final host callback, `ready`, marks completion of asynchronous setup. This keeps a lazy module's reserved name routable during initialization without announcing ownership before its objects are serving. +## Optional host capability tails + +ABI revision 1 and the original host prefix remain frozen. Each optional field +is available only when `size` reaches that field's offset plus its complete +field size. The `broker_routing` callback therefore remains available from an +older routing-capable host even though it lacks the newer +`native_terminal_state: Option u32>` tail. +A null host or a size below the frozen prefix is rejected; absent and partial +optional fields normalize independently to `None`. + +The native query contract admits only exactly Live (`1`). Unsupported queries, +zero, unknown bits, combined values, StopIntent (`2`), Fault (`4`) and +PhysicallyStopped (`8`) deny native admission. Terminal intent is monotonic +and never returns to Live. Query success linearizes admission; a cached result +cannot authorize a later operation, and a query alone grants no publication +authority. Later intent cannot undo an already admitted operation. + +PhysicallyStopped means successful opaque shutdown return. For modules opting +into native terminal hooks it additionally requires explicit finish proving +independent worker join. It proves neither legacy worker completion nor receiver +backing destruction. Faulted or uncertain shutdown must not earn this bit. + +A module copies the normalized callback and context, never borrows the host +vtable or configuration storage. A host supplying the query must keep that +callback and context valid and thread-safe for every retained view use, +including independent workers after runtime shutdown or host destruction. +Host destruction must leave a terminal tombstone; this context must never be +recycled or reopened. Custom unsafe C hosts must truthfully uphold this +contract, which cannot cryptographically authenticate an in-process host. +This tail defines an optional ABI capability; ordinary SDK hosts currently +supply `None`. It introduces no retain/release ABI or aggregate resource bound: +the existing process-lifetime host-context and library retention is unbounded. + ## Symbols Every module exports exactly these revisioned symbols: From fe080e23d33cdc7ade15f104a392b7bc2c19120f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:32:04 +0300 Subject: [PATCH 22/58] test(module): allocate aligned storage for rejected host vtable The short-header rejection test now allocates a properly aligned buffer instead of pointing at a stack u32, so the read is not undefined behaviour and the assertion reflects the real ABI check. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/abi_tests.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/tinybus/src/module/abi_tests.rs b/crates/tinybus/src/module/abi_tests.rs index ddccf43d..e7020fc4 100644 --- a/crates/tinybus/src/module/abi_tests.rs +++ b/crates/tinybus/src/module/abi_tests.rs @@ -119,9 +119,16 @@ fn prefix_hosts_normalize_both_tails() { std::mem::offset_of!(TbHostVtable, broker_routing) ); assert!(unsafe { TbHostVtable::read_compatible(std::ptr::null()) }.is_none()); - // Only a u32 allocation is present for the rejected short header. + // The rejected host owns only its declared bytes, with vtable alignment. let short = TB_HOST_VTABLE_BASE_SIZE - 1; - assert!(unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&short).cast()) }.is_none()); + let layout = + std::alloc::Layout::from_size_align(short as usize, align_of::()).unwrap(); + let storage = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!storage.is_null()); + unsafe { storage.cast::().write(short) }; + let rejected = unsafe { TbHostVtable::read_compatible(storage.cast()) }.is_none(); + unsafe { std::alloc::dealloc(storage, layout) }; + assert!(rejected); let prefix = frozen_prefix(size_of::()); let normalized = unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&prefix).cast()) }.unwrap(); From f45bd2c2bef550381388c12e89b21aca36749090 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:40:01 +0300 Subject: [PATCH 23/58] test(transport): cover abandoned init waiter faulting Add a test that aborts the task awaiting initialization while a deferred initializer is still running, verifying the transport publishes a fault immediately and still runs the late initializer's cleanup exactly once. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 38 ++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 43c8acaa..95e6be54 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1306,3 +1306,41 @@ async fn unanswered_method_calls_are_not_physical_callback_leases() { assert_eq!(transport.inflight(), 1, "A87"); assert!(transport.is_stopped(), "A88"); } + +#[tokio::test] +async fn abandoning_initiating_waiter_faults_and_retains_late_initializer_cleanup() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + SHUTDOWN_CODE.store(TB_OK, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("abandoned-init".into(), b"secret".to_vec()); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + transport.defer_initializer(host, move |host| { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + assert_eq!(unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }, b"secret"); + let mut module = TbModuleVtable::default(); + assert_eq!(unsafe { initialize_ok(&host, &mut module) }, TB_OK); + Ok(module) + }); + let waiter = tokio::spawn({ + let transport = transport.clone(); + async move { transport.ensure_initialized().await } + }); + entered_rx.await.unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + // Always release foreign work even on the unfixed implementation. + let faulted = transport.is_faulted(); + release_tx.send(()).unwrap(); + assert!(faulted, "initiating waiter abandonment must publish fault immediately"); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if transport.is_stopped() { break; } + tokio::task::yield_now().await; + } + }).await.unwrap(); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1); + assert!(transport.ensure_initialized().await.is_err()); +} From 2b8746ae1fd8dc1b72fa830be6929083b8075e6c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:42:04 +0300 Subject: [PATCH 24/58] chore: files changed crates/tinybus/src/module/transport.rs Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 398 +++++++++++++++++++------ 1 file changed, 315 insertions(+), 83 deletions(-) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index f7c38d62..a689a844 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -2,12 +2,12 @@ use std::collections::VecDeque; use std::ffi::c_void; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; use std::sync::{Arc, Condvar, Mutex as StdMutex, Weak}; use std::time::Duration; use async_trait::async_trait; -use tokio::sync::{Mutex, Notify, OnceCell, mpsc}; +use tokio::sync::{Mutex, Notify, mpsc}; use crate::error::{Error, Result}; use crate::message::codec::MAX_FRAME_LEN; @@ -31,6 +31,11 @@ struct HostContext { inbound: StdMutex>>>, wake: Arc, config: StdMutex>, + owner: StdMutex>, + native_terminal_state: AtomicU32, + physical: StdMutex, + physical_changed: Condvar, + physical_notify: Notify, faulted: AtomicBool, init_failed: AtomicBool, stopped: AtomicBool, @@ -52,7 +57,6 @@ pub(crate) struct ModuleTransport { context: &'static HostContext, label: String, initializer: StdMutex>, - init_result: OnceCell>, pending: StdMutex>, drain_started: AtomicBool, // Reconfiguration and shutdown both call into opaque module code. The @@ -124,6 +128,38 @@ struct DeferredInitializer { host: TbHostVtable, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum InitState { Deferred, Scheduled, Entered, PublishedValid, PublishedRejected, CanceledBeforeEntry, ExternalValid } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CleanupState { Idle, Requested, Executing, Complete, Uncertain, NoCallbackRequired } + +struct PhysicalState { + init: InitState, + cleanup: CleanupState, + job: Option>, + result: Option>, + code: Option, +} + +struct PhysicalJob { + transport: Arc, + initializer: StdMutex>, + guard: StdMutex>>, + deadline: Duration, +} + +struct InitiatingWaiter<'a> { transport: &'a ModuleTransport, armed: bool } +impl Drop for InitiatingWaiter<'_> { + fn drop(&mut self) { + if self.armed { + let published = self.transport.context.physical.lock().expect("physical state lock").result.is_some(); + if !published && !self.transport.admission_closed() { + self.transport.request_terminal(true); + } + } + } +} + struct SendModuleVtable(TbModuleVtable); #[derive(Clone)] @@ -164,6 +200,14 @@ impl ModuleTransport { inbound: StdMutex::new(Some(inbound_tx)), wake: Arc::new(Notify::new()), config: StdMutex::new(config), + owner: StdMutex::new(Weak::new()), + native_terminal_state: AtomicU32::new(crate::module::abi::TB_NATIVE_TERMINAL_LIVE), + physical: StdMutex::new(PhysicalState { + init: InitState::Deferred, cleanup: CleanupState::Idle, + job: None, result: None, code: None, + }), + physical_changed: Condvar::new(), + physical_notify: Notify::new(), faulted: AtomicBool::new(false), init_failed: AtomicBool::new(false), stopped: AtomicBool::new(false), @@ -183,7 +227,6 @@ impl ModuleTransport { context, label, initializer: StdMutex::new(None), - init_result: OnceCell::new(), pending: StdMutex::new(VecDeque::new()), drain_started: AtomicBool::new(false), lifecycle: Arc::new(Mutex::new(())), @@ -192,6 +235,7 @@ impl ModuleTransport { initialization_gate: StdMutex::new(Some(true)), initialization_gate_changed: Notify::new(), }); + *context.owner.lock().expect("physical owner lock") = Arc::downgrade(&transport); let config = context.config.lock().expect("module config lock"); let config_slice = crate::module::abi::TbSlice { ptr: config.as_ptr(), @@ -209,7 +253,7 @@ impl ModuleTransport { config: config_slice, ready: host_ready, broker_routing: Some(host_broker_routing), - native_terminal_state: None, + native_terminal_state: Some(host_native_terminal_state), }; (transport, vtable) } @@ -222,6 +266,11 @@ impl ModuleTransport { return Err(Error::transport("module returned an incomplete vtable")); } *self.module.lock().expect("module vtable lock") = Some(module); + let mut physical = self.context.physical.lock().expect("physical state lock"); + if physical.init == InitState::Deferred { + physical.init = InitState::ExternalValid; + physical.result = Some(Ok(())); + } Ok(()) } @@ -272,7 +321,7 @@ impl ModuleTransport { // Admission rejection is a genuine fault, unlike ordinary shutdown. Never // fabricate READY or init_started to release a rejected observer. fn terminate_waiters(&self) { - self.context.faulted.store(true, Ordering::Release); + self.request_terminal(true); self.context.init_notify.notify_waiters(); self.context.ready_notify.notify_waiters(); self.initialization_gate_changed.notify_waiters(); @@ -339,73 +388,137 @@ impl ModuleTransport { } } - let result = self - .init_result - .get_or_init(|| async { - { - // Pair starting opaque initialization with the never-started - // stop decision. A stopped admission cannot start it later. - let _gate = self - .initialization_gate - .lock() - .expect("initialization gate lock"); - if self.admission_closed() { - return Err(InitFailure::Closed); - } - self.context.init_started.store(true, Ordering::Release); - } - self.context.init_notify.notify_waiters(); - let Some(initializer) = self - .initializer - .lock() - .expect("module initializer lock") - .take() - else { - return if self.module.lock().expect("module vtable lock").is_some() { - Ok(()) - } else { - Err(InitFailure::Failed("module has no initializer".to_string())) - }; - }; - // Module init is opaque synchronous code. Keep it off Tokio's - // worker threads and bound how long the broker waits; a timed - // out blocking task may remain wedged, so its borrowed config - // remains allocated rather than being invalidated underneath it. - let initialized = tokio::time::timeout( - MODULE_INIT_DEADLINE, - tokio::task::spawn_blocking(move || initializer.run()), - ) - .await; - let module = match initialized { - Ok(Ok(Ok(initialized))) => initialized, - Ok(Ok(Err(reason))) => { - self.clear_config(); - return Err(InitFailure::Failed(reason)); - } - Ok(Err(_)) => { - return Err(InitFailure::Failed( - "module initialization panicked".to_string(), - )); - } - Err(_) => { - return Err(InitFailure::Failed( - "module initialization exceeded its deadline".to_string(), - )); - } - }; - self.clear_config(); - self.initialize(module.0).map_err(|_| { - InitFailure::Failed("module returned an invalid vtable".to_string()) - })?; - Ok(()) - }) - .await; + let initiating = self.schedule_initializer()?; + let mut abandonment = InitiatingWaiter { transport: self, armed: initiating }; + let wait = async { + loop { + let changed = self.context.physical_notify.notified(); + let result = self.context.physical.lock().expect("physical state lock").result.clone(); + if let Some(result) = result { return result; } + changed.await; + } + }; + let result = match tokio::time::timeout(MODULE_INIT_DEADLINE, wait).await { + Ok(result) => { abandonment.armed = false; result } + Err(_) => { + if initiating && !self.admission_closed() { self.request_terminal(true); } + abandonment.armed = false; + return Err(Error::transport("module initialization exceeded its deadline")); + } + }; match result { - Ok(()) if self.admission_closed() => Err(Error::ConnectionClosed), + Ok(()) if self.admission_closed() || self.is_faulted() => Err(Error::ConnectionClosed), Ok(()) => Ok(()), Err(InitFailure::Closed) => Err(Error::ConnectionClosed), - Err(InitFailure::Failed(reason)) => Err(Error::transport(reason.clone())), + Err(InitFailure::Failed(reason)) => Err(Error::transport(reason)), + } + } + + fn notify_physical(&self) { + self.context.physical_changed.notify_all(); + self.context.physical_notify.notify_waiters(); + self.context.init_notify.notify_waiters(); + } + + // Scheduling is synchronous: the externally anchored owner precedes submission. + fn schedule_initializer(&self) -> Result { + let gate = self.initialization_gate.lock().expect("initialization gate lock"); + if *gate != Some(true) || self.is_faulted() || self.admission_closed() { + return Err(Error::ConnectionClosed); + } + let mut state = self.context.physical.lock().expect("physical state lock"); + if state.result.is_some() || state.init != InitState::Deferred { return Ok(false); } + let Some(initializer) = self.initializer.lock().expect("module initializer lock").take() else { + state.result = Some(Err(InitFailure::Failed("module has no initializer".into()))); + return Ok(false); + }; + let job = Arc::new(PhysicalJob { + transport: self.self_ref.upgrade().ok_or(Error::ConnectionClosed)?, + initializer: StdMutex::new(Some(initializer)), guard: StdMutex::new(None), + deadline: MODULE_INIT_DEADLINE, + }); + state.init = InitState::Scheduled; + state.job = Some(job.clone()); + self.context.init_started.store(true, Ordering::Release); + drop(state); + drop(gate); + self.notify_physical(); + Self::submit_job(job); + Ok(true) + } + + fn submit_job(job: Arc) { + let submitted = job.clone(); + if std::thread::Builder::new().name("tinybus-physical".into()) + .spawn(move || submitted.run()).is_err() + { + let transport = &job.transport; + transport.commit_terminal(true); + let mut state = transport.context.physical.lock().expect("physical state lock"); + state.cleanup = CleanupState::Uncertain; + state.result.get_or_insert(Err(InitFailure::Failed("module physical worker could not start".into()))); + drop(state); + transport.notify_physical(); + } + } + + pub(crate) fn commit_terminal(&self, fault: bool) { + use crate::module::abi::{TB_NATIVE_TERMINAL_FAULT, TB_NATIVE_TERMINAL_STOP_INTENT}; + if fault { self.context.faulted.store(true, Ordering::Release); } + self.context.native_terminal_state.fetch_update(Ordering::AcqRel, Ordering::Acquire, |flags| { + Some((flags & !crate::module::abi::TB_NATIVE_TERMINAL_LIVE) | + TB_NATIVE_TERMINAL_STOP_INTENT | if fault { TB_NATIVE_TERMINAL_FAULT } else { 0 }) + }).ok(); + self.close_admission(); + } + + pub(crate) fn request_terminal(&self, fault: bool) { + self.commit_terminal(fault); + let mut state = self.context.physical.lock().expect("physical state lock"); + if state.cleanup == CleanupState::Idle { state.cleanup = CleanupState::Requested; } + if state.job.is_some() || state.cleanup != CleanupState::Requested { return; } + if state.init == InitState::Deferred { + state.init = InitState::CanceledBeforeEntry; + state.cleanup = CleanupState::NoCallbackRequired; + state.result = Some(Err(InitFailure::Closed)); + drop(state); + self.initializer.lock().expect("module initializer lock").take(); + self.clear_config(); + self.finish_stop(); + self.notify_physical(); + return; + } + let Some(transport) = self.self_ref.upgrade() else { return; }; + let job = Arc::new(PhysicalJob { transport, initializer: StdMutex::new(None), + guard: StdMutex::new(None), deadline: MODULE_INIT_DEADLINE }); + state.job = Some(job.clone()); + drop(state); + Self::submit_job(job); + } + + pub(crate) fn eager_initialize(&self, init: crate::module::abi::TbModuleInit, host: TbHostVtable) -> Result<()> { + let guard = self.lifecycle.clone().try_lock_owned().map_err(|_| { + self.commit_terminal(true); + self.context.physical.lock().expect("physical state lock").cleanup = CleanupState::Uncertain; + Error::transport("module initialization lifecycle unavailable") + })?; + self.context.physical.lock().expect("physical state lock").init = InitState::Entered; + self.context.init_started.store(true, Ordering::Release); + let mut module = TbModuleVtable::default(); + let code = unsafe { init(&host, &mut module) }; + let result = if code == TB_OK { self.initialize(module) } else { Err(Error::transport("module initialization failed")) }; + self.clear_config(); + { + let mut state = self.context.physical.lock().expect("physical state lock"); + state.init = if result.is_ok() { InitState::ExternalValid } else { InitState::PublishedRejected }; + state.result = Some(if result.is_ok() { Ok(()) } else { Err(InitFailure::Failed("module returned an invalid vtable".into())) }); + if result.is_err() { state.cleanup = CleanupState::Uncertain; } } + self.notify_physical(); + drop(guard); + if result.is_err() { self.request_terminal(true); } + if self.is_faulted() || self.admission_closed() { return Err(Error::ConnectionClosed); } + result } pub(crate) async fn wait_ready(&self) { @@ -445,7 +558,7 @@ impl ModuleTransport { .expect("static member name is valid"), timeout_ms: MODULE_INIT_DEADLINE.as_millis() as u64, })?; - if self.admission_closed() { + if self.admission_closed() || self.is_faulted() { return Err(Error::ConnectionClosed); } let module = self @@ -683,6 +796,9 @@ impl ModuleTransport { // tear down code underneath an admitted callback or abort its lease. self.callbacks.wait_drained(); let code = self.shutdown_sync(deadline.saturating_sub(started.elapsed())); + if code == TB_OK { + self.context.native_terminal_state.fetch_or(crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, Ordering::AcqRel); + } self.finish_stop(); self.context .inbound @@ -715,23 +831,136 @@ impl ModuleTransport { return Err(StopError::NotStarted(Error::ConnectionClosed)); } let remaining = deadline.saturating_sub(started.elapsed()); - // A retry that cannot acquire the physical lock must not erase the - // original running stop's host transition. Only its new lock owner may - // replace that marker, immediately before handing the lock to the task. + self.commit_terminal(false); + let mut state = self.context.physical.lock().expect("physical state lock"); + if state.cleanup == CleanupState::Idle { state.cleanup = CleanupState::Requested; } + let existing = state.job.is_some(); + let mut acquired = Some(lifecycle); + let job = if existing { None } else { + let job = Arc::new(PhysicalJob { transport: self.clone(), + initializer: StdMutex::new(None), guard: StdMutex::new(acquired.take()), + deadline: remaining }); + state.job = Some(job.clone()); + Some(job) + }; self.stop_task_spawned.store(true, Ordering::Release); - let task = tokio::task::spawn_blocking(move || { - let _lifecycle = lifecycle; - self.stop_sync(remaining) - }); - tokio::time::timeout(remaining, task) - .await - .map_err(|_| { - StopError::Started(Error::failed("module shutdown exceeded its deadline")) - })? - .map_err(|_| StopError::Started(Error::failed("module shutdown task was cancelled"))) + drop(state); + // A Scheduled initializer must acquire the guard itself to prove no C entry. + if existing { drop(acquired.take()); } + if let Some(job) = job { Self::submit_job(job); } + let wait = async { + loop { + let changed = self.context.physical_notify.notified(); + let state = self.context.physical.lock().expect("physical state lock"); + let outcome = match state.cleanup { + CleanupState::Complete | CleanupState::NoCallbackRequired => Some(Ok(state.code.unwrap_or(TB_CLOSED))), + CleanupState::Uncertain => Some(Err(StopError::Started(Error::transport("module cleanup is uncertain")))), + _ => None, + }; + drop(state); + if let Some(outcome) = outcome { return outcome; } + changed.await; + } + }; + tokio::time::timeout(remaining, wait).await.map_err(|_| StopError::Started(Error::failed("module shutdown exceeded its deadline")))? + + } +} + +impl PhysicalJob { + fn run(self: Arc) { + let transport = &self.transport; + let initializer = self.initializer.lock().expect("physical initializer lock").take(); + if let Some(initializer) = initializer { + let guard = transport.lifecycle.clone().blocking_lock_owned(); + let entered = { + let mut state = transport.context.physical.lock().expect("physical state lock"); + if transport.admission_closed() || transport.is_faulted() { + state.init = InitState::CanceledBeforeEntry; + state.cleanup = CleanupState::NoCallbackRequired; + state.result = Some(Err(InitFailure::Closed)); + state.code = Some(TB_CLOSED); + state.job = None; + false + } else { state.init = InitState::Entered; true } + }; + if !entered { + drop(initializer); + transport.clear_config(); + transport.finish_stop(); + transport.notify_physical(); + return; + } + let output = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| initializer.run())); + let result = match output { + Ok(Ok(module)) => transport.initialize(module.0).map_err(|_| InitFailure::Failed("module returned an invalid vtable".into())), + Ok(Err(reason)) => Err(InitFailure::Failed(reason)), + Err(_) => Err(InitFailure::Failed("module initialization panicked".into())), + }; + transport.clear_config(); + if result.is_err() { transport.commit_terminal(true); } + let cleanup = { + let mut state = transport.context.physical.lock().expect("physical state lock"); + state.init = if result.is_ok() { InitState::PublishedValid } else { InitState::PublishedRejected }; + state.result = Some(result.clone()); + if result.is_err() { state.cleanup = CleanupState::Uncertain; } + if state.cleanup == CleanupState::Idle { state.job = None; } + state.cleanup == CleanupState::Requested + }; + transport.notify_physical(); + drop(guard); + if !cleanup { return; } + } else { + // Eager entry publishes before releasing its exclusion. Never hold + // lifecycle while waiting for an initializer which needs that lock. + let mut state = transport.context.physical.lock().expect("physical state lock"); + while matches!(state.init, InitState::Scheduled | InitState::Entered) && state.result.is_none() { + state = transport.context.physical_changed.wait(state).expect("physical state lock"); + } + } + let guard = self.guard.lock().expect("physical guard lock").take() + .unwrap_or_else(|| transport.lifecycle.clone().blocking_lock_owned()); + { + let mut state = transport.context.physical.lock().expect("physical state lock"); + if state.cleanup != CleanupState::Requested { return; } + if state.init == InitState::Deferred || state.init == InitState::CanceledBeforeEntry { + state.init = InitState::CanceledBeforeEntry; + state.cleanup = CleanupState::NoCallbackRequired; + state.result = Some(Err(InitFailure::Closed)); + state.code = Some(TB_CLOSED); + state.job = None; + drop(state); + transport.initializer.lock().expect("module initializer lock").take(); + transport.clear_config(); + transport.finish_stop(); + transport.notify_physical(); + return; + } + if transport.module.lock().expect("module vtable lock").is_none() { + state.cleanup = CleanupState::Uncertain; + drop(state); + transport.notify_physical(); + return; + } + state.cleanup = CleanupState::Executing; + } + let code = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| transport.stop_sync(self.deadline))); + let mut state = transport.context.physical.lock().expect("physical state lock"); + state.code = code.as_ref().ok().copied(); + state.cleanup = if matches!(code, Ok(TB_OK)) { CleanupState::Complete } else { CleanupState::Uncertain }; + if state.cleanup == CleanupState::Complete { state.job = None; } + drop(state); + transport.notify_physical(); + drop(guard); } } +unsafe extern "C" fn host_native_terminal_state(ctx: *mut c_void) -> u32 { + unsafe { ctx.cast::().as_ref() }.map_or( + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT | crate::module::abi::TB_NATIVE_TERMINAL_FAULT, + |context| context.native_terminal_state.load(Ordering::Acquire)) +} + #[derive(Debug)] pub(crate) enum StopError { NotStarted(Error), @@ -917,9 +1146,12 @@ unsafe extern "C" fn host_fault(ctx: *mut c_void, _: *const u8, _: usize) { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { if let Some(context) = unsafe { ctx.cast::().as_ref() } { context.faulted.store(true, Ordering::Release); + context.native_terminal_state.fetch_update(Ordering::AcqRel, Ordering::Acquire, |flags| Some((flags & !1) | 2 | 4)).ok(); context.init_notify.notify_waiters(); context.ready_notify.notify_waiters(); context.inbound.lock().expect("host inbound lock").take(); + let owner = context.owner.lock().expect("physical owner lock").upgrade(); + if let Some(owner) = owner { owner.request_terminal(true); } } })); } From 1d966ee6c74126d257d289a99d8632dc4e32799a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:42:49 +0300 Subject: [PATCH 25/58] refactor(module): centralize terminal transport state transitions Module initialization and teardown now go through transport helpers that record terminal state consistently, so a module that fails to initialize or faults is reported as failed or faulted instead of stopped. The terminal flag update was also rewritten as a compare-exchange loop to avoid losing concurrent fault or stop-intent bits. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host.rs | 19 ++++++---------- crates/tinybus/src/module/host_callbacks.rs | 17 +++++++------- crates/tinybus/src/module/transport.rs | 25 ++++++++++++++------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/crates/tinybus/src/module/host.rs b/crates/tinybus/src/module/host.rs index e0073ea4..de432a2c 100644 --- a/crates/tinybus/src/module/host.rs +++ b/crates/tinybus/src/module/host.rs @@ -1020,19 +1020,11 @@ impl ModuleHost { transport.defer_initialize(artifact.init, host_vtable); } else { admitted.state = ModuleState::Initializing; - let mut module_vtable = TbModuleVtable::default(); - let code = unsafe { (artifact.init)(&host_vtable, &mut module_vtable) }; - transport.clear_config(); - if code != TB_OK { + if transport.eager_initialize(artifact.init, host_vtable).is_err() { let error = Error::module_refused(path, "module initialization failed"); self.record_manifest_rejection(&error, manifest, RefusalClass::Failed); return Err(error); } - if transport.initialize(module_vtable).is_err() { - let error = Error::module_refused(path, "module returned an invalid vtable"); - self.record_manifest_rejection(&error, manifest, RefusalClass::Failed); - return Err(error); - } } self.attach_transport( @@ -1150,7 +1142,7 @@ impl ModuleHost { .lock() .expect("router lock") .detach_unique(&unique); - let _ = transport.stop_sync(Duration::from_millis(0)); + transport.request_terminal(true); let error = Error::module_refused(path, "module bus name is already owned"); self.record_manifest_rejection(&error, manifest, RefusalClass::Unresolved); return Err(error); @@ -1520,7 +1512,10 @@ impl ModuleControl for ModuleHostInner { } Err(StopError::Started(error)) => return Err(error), } - module.info.state = ModuleState::Stopped; + module.info.state = if module.transport.is_faulted() { + if module.transport.init_failed() { ModuleState::Failed { reason: "module initialization failed".into() } } + else { ModuleState::Faulted { reason: "module reported an unrecoverable fault".into() } } + } else { ModuleState::Stopped }; Ok(module.info.clone()) } @@ -1663,7 +1658,7 @@ impl ModuleControl for ModuleHostInner { let module = loaded .iter_mut() .find(|module| &module.unique_name == unique_name)?; - if let Some(old) = module.transition_from.take() { + if !module.transport.is_faulted() && let Some(old) = module.transition_from.take() { module.info.state = ModuleState::Stopped; return Some((module.info.name.clone(), old, ModuleState::Stopped)); } diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 1caff6db..8b4931c4 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -587,7 +587,7 @@ impl ModuleHost { }; if !ready { binding.revoke(); - let _ = transport.stop_sync(Duration::ZERO); + transport.request_terminal(true); let _ = transport.close().await; self.inner .loaded @@ -669,7 +669,7 @@ impl Drop for LoadRollback<'_> { } // A timed-out initializer may still run. Stop is best effort; // revocation/detach removes authority without claiming cessation. - let _ = transport.stop_sync(Duration::ZERO); + transport.request_terminal(true); self.host .inner .loaded @@ -682,13 +682,14 @@ impl Drop for LoadRollback<'_> { impl Drop for ModuleHostInner { fn drop(&mut self) { - for module in self.loaded.get_mut().expect("module list lock").iter() { - self.broker - .router - .lock() - .expect("router lock") - .revoke_callbacks_for_peer(&module.unique_name); + let modules = self.loaded.get_mut().expect("module list lock"); + let transports: Vec<_> = modules.iter().map(|module| module.transport.clone()).collect(); + for transport in &transports { transport.commit_terminal(false); } + { + let mut router = self.broker.router.lock().expect("router lock"); + for module in modules.iter() { router.revoke_callbacks_for_peer(&module.unique_name); } } + for transport in transports { transport.request_terminal(false); } } } diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index a689a844..6f63ba23 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -465,10 +465,15 @@ impl ModuleTransport { pub(crate) fn commit_terminal(&self, fault: bool) { use crate::module::abi::{TB_NATIVE_TERMINAL_FAULT, TB_NATIVE_TERMINAL_STOP_INTENT}; if fault { self.context.faulted.store(true, Ordering::Release); } - self.context.native_terminal_state.fetch_update(Ordering::AcqRel, Ordering::Acquire, |flags| { - Some((flags & !crate::module::abi::TB_NATIVE_TERMINAL_LIVE) | - TB_NATIVE_TERMINAL_STOP_INTENT | if fault { TB_NATIVE_TERMINAL_FAULT } else { 0 }) - }).ok(); + let flags = &self.context.native_terminal_state; + let mut old = flags.load(Ordering::Acquire); + loop { + let new = (old & !crate::module::abi::TB_NATIVE_TERMINAL_LIVE) | + TB_NATIVE_TERMINAL_STOP_INTENT | if fault { TB_NATIVE_TERMINAL_FAULT } else { 0 }; + match flags.compare_exchange_weak(old, new, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => break, Err(current) => old = current, + } + } self.close_admission(); } @@ -832,6 +837,7 @@ impl ModuleTransport { } let remaining = deadline.saturating_sub(started.elapsed()); self.commit_terminal(false); + let (existing, job, mut acquired) = { let mut state = self.context.physical.lock().expect("physical state lock"); if state.cleanup == CleanupState::Idle { state.cleanup = CleanupState::Requested; } let existing = state.job.is_some(); @@ -844,20 +850,22 @@ impl ModuleTransport { Some(job) }; self.stop_task_spawned.store(true, Ordering::Release); - drop(state); + (existing, job, acquired) + }; // A Scheduled initializer must acquire the guard itself to prove no C entry. if existing { drop(acquired.take()); } if let Some(job) = job { Self::submit_job(job); } let wait = async { loop { let changed = self.context.physical_notify.notified(); + let outcome = { let state = self.context.physical.lock().expect("physical state lock"); - let outcome = match state.cleanup { + match state.cleanup { CleanupState::Complete | CleanupState::NoCallbackRequired => Some(Ok(state.code.unwrap_or(TB_CLOSED))), CleanupState::Uncertain => Some(Err(StopError::Started(Error::transport("module cleanup is uncertain")))), _ => None, + } }; - drop(state); if let Some(outcome) = outcome { return outcome; } changed.await; } @@ -1146,7 +1154,8 @@ unsafe extern "C" fn host_fault(ctx: *mut c_void, _: *const u8, _: usize) { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { if let Some(context) = unsafe { ctx.cast::().as_ref() } { context.faulted.store(true, Ordering::Release); - context.native_terminal_state.fetch_update(Ordering::AcqRel, Ordering::Acquire, |flags| Some((flags & !1) | 2 | 4)).ok(); + context.native_terminal_state.fetch_or(crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT | crate::module::abi::TB_NATIVE_TERMINAL_FAULT, Ordering::AcqRel); + context.native_terminal_state.fetch_and(!crate::module::abi::TB_NATIVE_TERMINAL_LIVE, Ordering::AcqRel); context.init_notify.notify_waiters(); context.ready_notify.notify_waiters(); context.inbound.lock().expect("host inbound lock").take(); From 9867fb34f3d0d61cdbd49f7b08eeeccb9dc49d90 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:44:49 +0300 Subject: [PATCH 26/58] fix(module): report invalid vtable and release inbound on stop Module initialization failures now distinguish an invalid vtable from a generic refusal so callers get an accurate reason. Transport teardown releases the inbound queue and routes fault paths through request_terminal, ensuring the terminal state is published consistently and the failing result is returned to the caller. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host.rs | 5 +++-- crates/tinybus/src/module/transport.rs | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/tinybus/src/module/host.rs b/crates/tinybus/src/module/host.rs index de432a2c..32ca32c3 100644 --- a/crates/tinybus/src/module/host.rs +++ b/crates/tinybus/src/module/host.rs @@ -1020,8 +1020,9 @@ impl ModuleHost { transport.defer_initialize(artifact.init, host_vtable); } else { admitted.state = ModuleState::Initializing; - if transport.eager_initialize(artifact.init, host_vtable).is_err() { - let error = Error::module_refused(path, "module initialization failed"); + if let Err(failure) = transport.eager_initialize(artifact.init, host_vtable) { + let reason = if failure.to_string().contains("vtable") { "module returned an invalid vtable" } else { "module initialization failed" }; + let error = Error::module_refused(path, reason); self.record_manifest_rejection(&error, manifest, RefusalClass::Failed); return Err(error); } diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 6f63ba23..974d46aa 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -353,6 +353,7 @@ impl ModuleTransport { /// Publish physical completion only after the opaque shutdown returns. fn finish_stop(&self) { self.context.stopped.store(true, Ordering::Release); + self.context.inbound.lock().expect("host inbound lock").take(); self.context.init_notify.notify_waiters(); self.context.ready_notify.notify_waiters(); self.context.wake.notify_waiters(); @@ -521,7 +522,7 @@ impl ModuleTransport { } self.notify_physical(); drop(guard); - if result.is_err() { self.request_terminal(true); } + if result.is_err() { self.request_terminal(true); return result; } if self.is_faulted() || self.admission_closed() { return Err(Error::ConnectionClosed); } result } @@ -673,7 +674,8 @@ impl ModuleTransport { if self.admission_closed() { return Err(Error::ConnectionClosed); } - self.context.faulted.store(true, Ordering::Release); + drop(_pending); + self.request_terminal(true); self.context.ready_notify.notify_waiters(); self.context .inbound @@ -772,7 +774,7 @@ impl ModuleTransport { } } } - self.context.faulted.store(true, Ordering::Release); + self.request_terminal(true); self.context.ready_notify.notify_waiters(); self.context .inbound @@ -998,6 +1000,7 @@ impl Transport for ModuleTransport { self.context.init_failed.store(true, Ordering::Release); self.context.faulted.store(true, Ordering::Release); } + self.request_terminal(true); self.context.ready_notify.notify_waiters(); if message.header.kind == MessageKind::MethodCall { let error = Error::ModuleUnavailable { From f356d478d693e8ee06f2e070f90313939074934e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:47:08 +0300 Subject: [PATCH 27/58] fix(module): retain initializer owner when worker spawn is rejected When the physical worker thread cannot be spawned, the submitted job is dropped and its initializer closure would be lost, leaving cleanup uncertain. Anchor the job on the transport context so the actual owner survives host teardown, and add a test-only spawn rejection hook plus a controlled-init example to exercise the path. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/Cargo.toml | 5 + .../examples/module_controlled_init.rs | 70 +++++++ crates/tinybus/src/module/transport.rs | 30 ++- crates/tinybus/src/module/transport_tests.rs | 171 ++++++++++++++++++ 4 files changed, 272 insertions(+), 4 deletions(-) create mode 100644 crates/tinybus/examples/module_controlled_init.rs diff --git a/crates/tinybus/Cargo.toml b/crates/tinybus/Cargo.toml index b567ed1e..8f6452d7 100644 --- a/crates/tinybus/Cargo.toml +++ b/crates/tinybus/Cargo.toml @@ -135,3 +135,8 @@ required-features = ["modules"] name = "module_host_callback" crate-type = ["cdylib"] required-features = ["modules"] + +[[example]] +name = "module_controlled_init" +crate-type = ["cdylib"] +required-features = ["modules"] diff --git a/crates/tinybus/examples/module_controlled_init.rs b/crates/tinybus/examples/module_controlled_init.rs new file mode 100644 index 00000000..a17adbf4 --- /dev/null +++ b/crates/tinybus/examples/module_controlled_init.rs @@ -0,0 +1,70 @@ +//! Native ABI fixture with bounded, externally released test barriers. +use std::ffi::c_void; +use std::sync::{Arc, Condvar, Mutex}; +use tinybus::module::abi::*; + +#[repr(C)] +#[derive(Clone, Copy)] +struct Controls { + ctx: *mut c_void, + entered: unsafe extern "C" fn(*mut c_void, *const TbHostVtable), + joined: unsafe extern "C" fn(*mut c_void), +} +unsafe impl Send for Controls {} + +struct Instance { + controls: Controls, + stop: Arc<(Mutex, Condvar)>, + worker: Option>, +} + +#[unsafe(no_mangle)] +pub static TINYBUS_MODULE_ABI_V1: TbAbiDescriptor = + TbAbiDescriptor::current("controlled-init", env!("CARGO_PKG_VERSION")); + +#[unsafe(no_mangle)] +pub extern "C" fn tinybus_module_manifest_v1() -> TbSlice { + tinybus_module::manifest_slice(tinybus_module::ManifestDeclaration { + name: "controlled-init", version: env!("CARGO_PKG_VERSION"), + provides: &["ai.tinyhumans.ControlledInit"], methods: &[], signals: &[], + requires: &[], optional: &[], lazy: true, worker_threads: 1, + }) +} + +unsafe extern "C" fn deliver(_: *mut c_void, _: *const u8, _: usize) -> i32 { TB_OK } +unsafe extern "C" fn reinitialize(_: *mut c_void, _: *const u8, _: usize) -> i32 { TB_OK } +unsafe extern "C" fn shutdown(ctx: *mut c_void, _: u64) -> i32 { + let mut instance = unsafe { Box::from_raw(ctx.cast::()) }; + *instance.stop.0.lock().unwrap() = true; + instance.stop.1.notify_all(); + instance.worker.take().unwrap().join().unwrap(); + unsafe { (instance.controls.joined)(instance.controls.ctx); } + TB_OK +} + +/// # Safety +/// Host pointers are valid for the ABI call; config contains test Controls. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tinybus_module_init_v1(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { + let host = unsafe { &*host }; + if host.config.len != size_of::() { return TB_BAD_ARGUMENT; } + let controls = unsafe { host.config.ptr.cast::().read_unaligned() }; + let copied_config = unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }.to_vec(); + // The host-side barrier blocks actual opaque C entry, retaining its config + // borrow through fault publication and initiating waiter abandonment. + unsafe { (controls.entered)(controls.ctx, host); } + assert_eq!(unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }, copied_config); + let stop = Arc::new((Mutex::new(false), Condvar::new())); + let worker_stop = stop.clone(); + let worker = std::thread::spawn(move || { + let mut stopped = worker_stop.0.lock().unwrap(); + while !*stopped { stopped = worker_stop.1.wait(stopped).unwrap(); } + }); + let instance = Box::new(Instance { controls, stop, worker: Some(worker) }); + unsafe { *out = TbModuleVtable { + size: size_of::() as u32, _reserved: 0, + module_ctx: Box::into_raw(instance).cast(), deliver, shutdown, + reinitialize: Some(reinitialize), + }; } + TB_OK +} diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 974d46aa..18b609e3 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -67,6 +67,8 @@ pub(crate) struct ModuleTransport { callbacks: CallbackGate, initialization_gate: StdMutex>, initialization_gate_changed: Notify, + #[cfg(test)] + reject_worker_spawn: AtomicBool, } // Reservations cover physical callback execution, including signals. They are @@ -234,6 +236,8 @@ impl ModuleTransport { callbacks: CallbackGate::default(), initialization_gate: StdMutex::new(Some(true)), initialization_gate_changed: Notify::new(), + #[cfg(test)] + reject_worker_spawn: AtomicBool::new(false), }); *context.owner.lock().expect("physical owner lock") = Arc::downgrade(&transport); let config = context.config.lock().expect("module config lock"); @@ -450,9 +454,19 @@ impl ModuleTransport { fn submit_job(job: Arc) { let submitted = job.clone(); - if std::thread::Builder::new().name("tinybus-physical".into()) - .spawn(move || submitted.run()).is_err() - { + #[cfg(test)] + let rejected = job.transport.reject_worker_spawn.load(Ordering::Acquire); + #[cfg(not(test))] + let rejected = false; + let spawn = if rejected { + // Model Builder rejecting and dropping the submitted closure: only + // the independent context anchor can preserve its physical owner. + drop(submitted); + Err(std::io::Error::other("controlled worker rejection")) + } else { + std::thread::Builder::new().name("tinybus-physical".into()).spawn(move || submitted.run()) + }; + if spawn.is_err() { let transport = &job.transport; transport.commit_terminal(true); let mut state = transport.context.physical.lock().expect("physical state lock"); @@ -518,7 +532,15 @@ impl ModuleTransport { let mut state = self.context.physical.lock().expect("physical state lock"); state.init = if result.is_ok() { InitState::ExternalValid } else { InitState::PublishedRejected }; state.result = Some(if result.is_ok() { Ok(()) } else { Err(InitFailure::Failed("module returned an invalid vtable".into())) }); - if result.is_err() { state.cleanup = CleanupState::Uncertain; } + if result.is_err() { + state.cleanup = CleanupState::Uncertain; + if state.job.is_none() { + state.job = self.self_ref.upgrade().map(|transport| Arc::new(PhysicalJob { + transport, initializer: StdMutex::new(None), guard: StdMutex::new(None), + deadline: MODULE_INIT_DEADLINE, + })); + } + } } self.notify_physical(); drop(guard); diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 95e6be54..b5debac5 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1344,3 +1344,174 @@ async fn abandoning_initiating_waiter_faults_and_retains_late_initializer_cleanu assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1); assert!(transport.ensure_initialized().await.is_err()); } + +#[tokio::test] +async fn scheduled_initializer_fault_cancels_before_actual_entry() { + let (transport, host) = ModuleTransport::new("scheduled-fault".into(), vec![]); + let entries = Arc::new(AtomicUsize::new(0)); + transport.defer_initializer(host, { + let entries = entries.clone(); + move |_| { entries.fetch_add(1, Ordering::AcqRel); Err("unexpected C entry".into()) } + }); + let guard = transport.lifecycle.clone().lock_owned().await; + assert!(transport.schedule_initializer().unwrap()); + transport.request_terminal(true); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + assert_eq!(transport.context.physical.lock().unwrap().init, InitState::Scheduled); + drop(guard); + tokio::time::timeout(Duration::from_secs(2), async { + while !transport.is_stopped() { tokio::task::yield_now().await; } + }).await.unwrap(); + assert_eq!(entries.load(Ordering::Acquire), 0); + assert_eq!(transport.context.physical.lock().unwrap().cleanup, CleanupState::NoCallbackRequired); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); +} + +#[tokio::test] +async fn scheduled_initializer_is_adopted_by_original_ordinary_stop() { + let (transport, host) = ModuleTransport::new("scheduled-stop".into(), vec![]); + let entries = Arc::new(AtomicUsize::new(0)); + transport.defer_initializer(host, { + let entries = entries.clone(); + move |_| { entries.fetch_add(1, Ordering::AcqRel); Err("unexpected C entry".into()) } + }); + // Hold the job before submission so stop can acquire lifecycle first. + transport.reject_worker_spawn.store(true, Ordering::Release); + // Scheduling rejection itself must fault, so normal adoption is tested + // with an anchored job whose worker has not yet acquired lifecycle. + let initializer = transport.initializer.lock().unwrap().take().unwrap(); + let job = Arc::new(PhysicalJob { + transport: transport.clone(), initializer: StdMutex::new(Some(initializer)), + guard: StdMutex::new(None), deadline: MODULE_INIT_DEADLINE, + }); + { + let mut state = transport.context.physical.lock().unwrap(); + state.init = InitState::Scheduled; + state.job = Some(job.clone()); + } + let mut stopping = Box::pin(transport.clone().stop(Duration::from_secs(2))); + tokio::select! { result = &mut stopping => panic!("stop completed before owner: {result:?}"), + _ = async { while !transport.stop_task_spawned() { tokio::task::yield_now().await; } } => {} } + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 2); + transport.reject_worker_spawn.store(false, Ordering::Release); + ModuleTransport::submit_job(job); + assert_eq!(stopping.await.unwrap(), TB_CLOSED); + assert_eq!(entries.load(Ordering::Acquire), 0); + assert!(!transport.is_faulted()); +} + +#[tokio::test] +async fn rejected_spawn_retains_actual_initializer_owner_and_never_resubmits() { + struct Probe(Arc); + impl Drop for Probe { fn drop(&mut self) { self.0.store(true, Ordering::Release); } } + let dropped = Arc::new(AtomicBool::new(false)); + let (transport, host) = ModuleTransport::new("spawn-rejection".into(), vec![]); + transport.reject_worker_spawn.store(true, Ordering::Release); + let probe = Probe(dropped.clone()); + transport.defer_initializer(host, move |_| { drop(probe); Err("must never execute".into()) }); + assert!(transport.ensure_initialized().await.is_err()); + assert!(transport.is_faulted()); + let weak = Arc::downgrade(&transport); + assert_eq!(transport.context.physical.lock().unwrap().cleanup, CleanupState::Uncertain); + transport.reject_worker_spawn.store(false, Ordering::Release); + assert!(transport.ensure_initialized().await.is_err()); + drop(transport); + assert!(weak.upgrade().is_some(), "context anchor retains actual job through host teardown"); + assert!(!dropped.load(Ordering::Acquire), "closure rejection must not drop controller"); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct NativeControls { + ctx: *mut c_void, + entered: unsafe extern "C" fn(*mut c_void, *const TbHostVtable), + joined: unsafe extern "C" fn(*mut c_void), +} +struct NativeBarrier { + state: StdMutex<(Option, bool, bool)>, + changed: Condvar, + entered: tokio::sync::Notify, + joined: tokio::sync::Notify, +} +unsafe impl Send for NativeBarrier {} +unsafe impl Sync for NativeBarrier {} +unsafe extern "C" fn native_entered(ctx: *mut c_void, host: *const TbHostVtable) { + let barrier = unsafe { &*ctx.cast::() }; + let mut state = barrier.state.lock().unwrap(); + state.0 = Some(unsafe { *host }); + barrier.entered.notify_one(); + while !state.1 { state = barrier.changed.wait(state).unwrap(); } +} +unsafe extern "C" fn native_joined(ctx: *mut c_void) { + let barrier = unsafe { &*ctx.cast::() }; + let mut state = barrier.state.lock().unwrap(); + assert!(!state.2, "one actual native shutdown/join"); + state.2 = true; + barrier.joined.notify_one(); +} +fn native_fixture() -> (Arc, TbHostVtable, crate::module::abi::TbModuleInit, Arc) { + let path = std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").expect("controlled native fixture path"); + let artifact = crate::module::loader::load(std::path::Path::new(&path), false).unwrap(); + let barrier = Arc::new(NativeBarrier { + state: StdMutex::new((None, false, false)), changed: Condvar::new(), + entered: Notify::new(), joined: Notify::new(), + }); + let controls = NativeControls { ctx: Arc::as_ptr(&barrier).cast_mut().cast(), entered: native_entered, joined: native_joined }; + let config = unsafe { std::slice::from_raw_parts(std::ptr::from_ref(&controls).cast::(), size_of::()) }.to_vec(); + let (transport, host) = ModuleTransport::new("actual-native".into(), config); + (transport, host, artifact.init, barrier) +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_late_init_is_joined_after_initiator_abandonment() { + let (transport, host, init, barrier) = native_fixture(); + transport.defer_initialize(init, host); + let waiter = tokio::spawn({ + let transport = transport.clone(); async move { transport.ensure_initialized().await } + }); + barrier.entered.notified().await; + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + let view = barrier.state.lock().unwrap().0.unwrap(); + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 6); + assert!(!barrier.state.lock().unwrap().2); + { + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); + } + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + let _physical_exit = transport.lifecycle.lock().await; + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 14); + assert_eq!(transport.context.physical.lock().unwrap().cleanup, CleanupState::Complete); + assert!(transport.ensure_initialized().await.is_err()); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_eager_fault_denies_while_config_remains_borrowed_then_joins() { + let (transport, host, init, barrier) = native_fixture(); + let host = DeferredInitializer { initialize: Box::new(|_| unreachable!()), host }; + let eager = std::thread::spawn({ + let transport = transport.clone(); + move || transport.eager_initialize(init, host.host) + }); + barrier.entered.notified().await; + let view = barrier.state.lock().unwrap().0.unwrap(); + unsafe { (view.fault)(view.host_ctx, std::ptr::null(), 0); } + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 6); + assert_eq!(transport.context.config.lock().unwrap().len(), size_of::()); + assert!(!barrier.state.lock().unwrap().2); + { + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); + } + assert!(eager.join().unwrap().is_err()); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + let _physical_exit = transport.lifecycle.lock().await; + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 14); + assert!(transport.context.config.lock().unwrap().is_empty()); +} From c956e1ff699c11d38620928c28ec2a63daa578db Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:48:09 +0300 Subject: [PATCH 28/58] test: wire controlled-init module fixture into e2e and coverage scripts The e2e and coverage scripts now build, locate, and export the module_controlled_init example alongside the other loader fixtures, and the transport test moves the host owner into the eager-initialize thread so the borrow checker accepts it. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 2 +- scripts/check-coverage.sh | 7 ++++--- scripts/test-e2e.ps1 | 4 +++- scripts/test-e2e.sh | 7 ++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index b5debac5..0efed43b 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1496,7 +1496,7 @@ async fn actual_native_eager_fault_denies_while_config_remains_borrowed_then_joi let host = DeferredInitializer { initialize: Box::new(|_| unreachable!()), host }; let eager = std::thread::spawn({ let transport = transport.clone(); - move || transport.eager_initialize(init, host.host) + move || { let owner = host; transport.eager_initialize(init, owner.host) } }); barrier.entered.notified().await; let view = barrier.state.lock().unwrap().0.unwrap(); diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index 2e49d32b..adea2aaf 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." -cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --no-default-features --features modules,macros +cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --no-default-features --features modules,macros set -euo pipefail module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_clock.so' -print -quit) module_two=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_clock_two.so' -print -quit) @@ -9,7 +9,8 @@ wrong_target=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_w context_module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_context.so' -print -quit) context_eager_module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_context_eager.so' -print -quit) host_callback_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) -for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module"; do +controlled_init_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) +for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module"; do if [ -z "$artifact" ] || [ ! -f "$artifact" ]; then echo "a module loader fixture is missing" >&2 exit 1 @@ -33,4 +34,4 @@ printf '"%s" = "%s"\n' "$library" "$digest" > "$private/modules.toml" # process/network branches are not meaningful per-file coverage. The # test-support module is integration-test glue whose platform-specific # dynamic-loader and timeout paths run in the dedicated OS jobs instead. -TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 -- --include-ignored +TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$PWD/$controlled_init_module" cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 -- --include-ignored diff --git a/scripts/test-e2e.ps1 b/scripts/test-e2e.ps1 index 90592413..bbd3f966 100644 --- a/scripts/test-e2e.ps1 +++ b/scripts/test-e2e.ps1 @@ -19,10 +19,11 @@ Copy-Item target/debug/examples/module_clock.dll $private Copy-Item target/debug/examples/module_context.dll $private Copy-Item target/debug/examples/module_context_eager.dll $private Copy-Item target/debug/examples/module_host_callback.dll $private +Copy-Item target/debug/examples/module_controlled_init.dll $private Copy-Item target/debug/examples/module_wrong_target.dll $private # Admit every intentional fixture by its actual built bytes so later # checks, including the wrong-target gate, can exercise their contract. -$fixtureAllowlist = @('module_clock.dll', 'module_context.dll', 'module_context_eager.dll', 'module_host_callback.dll', 'module_wrong_target.dll') | ForEach-Object { +$fixtureAllowlist = @('module_clock.dll', 'module_context.dll', 'module_context_eager.dll', 'module_host_callback.dll', 'module_controlled_init.dll', 'module_wrong_target.dll') | ForEach-Object { $digest = (Get-FileHash -LiteralPath (Join-Path $private $_) -Algorithm SHA256).Hash.ToLowerInvariant() '"' + $_ + '" = "' + $digest + '"' } @@ -45,6 +46,7 @@ $env:TEMP = $private $env:TMP = $private $env:TINYBUS_TEST_CONTEXT_MODULE = (Get-Item "$private/module_context.dll").FullName $env:TINYBUS_TEST_CONTEXT_EAGER_MODULE = (Get-Item "$private/module_context_eager.dll").FullName +$env:TINYBUS_TEST_CONTROLLED_INIT_MODULE = (Get-Item "$private/module_controlled_init.dll").FullName $env:TINYBUS_TEST_HOST_CALLBACK_MODULE = (Get-Item "$private/module_host_callback.dll").FullName $env:TINYBUS_TEST_MODULE = $module $env:TINYBUS_TEST_MODULE_TWO = $moduleTwo diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh index bf97e755..72579373 100755 --- a/scripts/test-e2e.sh +++ b/scripts/test-e2e.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." -cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --no-default-features --features modules,macros +cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --no-default-features --features modules,macros cargo test --locked --no-default-features --features modules,macros,test-support test_support::tests case "${RUNNER_OS:-$(uname -s)}" in Windows|MINGW*|MSYS*|CYGWIN*) pwsh -NoProfile -File scripts/test-e2e.ps1 ;; @@ -13,7 +13,8 @@ wrong_target=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodul context_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_context.so' -o -name 'libmodule_context.dylib' \) -print -quit) context_eager_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_context_eager.so' -o -name 'libmodule_context_eager.dylib' \) -print -quit) host_callback_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) -for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module"; do +controlled_init_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) +for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module"; do if [ -z "$artifact" ] || [ ! -f "$artifact" ]; then echo "a module example artifact is missing; the loader tests would not exercise it" >&2 exit 1 @@ -30,7 +31,7 @@ else digest="$(shasum -a 256 "$private/$library" | awk '{print $1}')" fi printf '"%s" = "%s"\n' "$library" "$digest" > "$private/modules.toml" -TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" cargo test --locked --no-default-features --features modules,macros,test-support -- --ignored +TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$PWD/$controlled_init_module" cargo test --locked --no-default-features --features modules,macros,test-support -- --ignored cargo test --locked --no-default-features --features modules,macros admission_tests -- --include-ignored TINYBUS_TEST_UMASK=1 cargo test --locked --no-default-features --features modules,macros permissive_umask -- --test-threads=1 From dae53b3f8beaf65c6136b33aa036723021750268 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 11:50:02 +0300 Subject: [PATCH 29/58] fix(module): request terminal state after delivering the final error The terminal request was issued before the error reply was sent, so the reply could race with teardown. It is now requested after the reply is delivered on both the method-call and connection-closed paths, and tests wait for physical cleanup to complete before finishing. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 3 ++- crates/tinybus/src/module/transport_tests.rs | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 18b609e3..a60196ac 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -1022,7 +1022,6 @@ impl Transport for ModuleTransport { self.context.init_failed.store(true, Ordering::Release); self.context.faulted.store(true, Ordering::Release); } - self.request_terminal(true); self.context.ready_notify.notify_waiters(); if message.header.kind == MessageKind::MethodCall { let error = Error::ModuleUnavailable { @@ -1042,8 +1041,10 @@ impl Transport for ModuleTransport { let _ = sender.try_send(bytes); } } + self.request_terminal(true); return Ok(()); } + self.request_terminal(true); return Err(Error::ConnectionClosed); } let is_call = message.header.kind == MessageKind::MethodCall; diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 0efed43b..13aab826 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -274,6 +274,7 @@ async fn a_shutdown_task_timeout_is_distinguished_from_a_lifecycle_lock_timeout( )); assert!(SHUTDOWN_ENTERED.load(Ordering::Acquire)); SHUTDOWN_BLOCKED.store(false, Ordering::Release); + wait_physical_cleanup(&transport).await; } #[tokio::test] @@ -502,6 +503,7 @@ async fn delivery_maps_a_faulted_transport_a_missing_module_and_module_refusals( transport.deliver_now(call()).await, Err(Error::ConnectionClosed) )); + wait_physical_cleanup(&transport).await; } #[test] @@ -1515,3 +1517,16 @@ async fn actual_native_eager_fault_denies_while_config_remains_borrowed_then_joi assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 14); assert!(transport.context.config.lock().unwrap().is_empty()); } + + +async fn wait_physical_cleanup(transport: &ModuleTransport) { + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let changed = transport.context.physical_notify.notified(); + let terminal = matches!(transport.context.physical.lock().unwrap().cleanup, + CleanupState::Complete | CleanupState::Uncertain | CleanupState::NoCallbackRequired); + if terminal { return; } + changed.await; + } + }).await.unwrap(); +} From 1da2f24511173ea7eacdcdee9fac2f9d72ec7a8b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:02:03 +0300 Subject: [PATCH 30/58] test(transport): cover fault-owned stop and shutdown status Add tests asserting that a fault-owned job does not create an ordinary stop marker and that ordinary shutdown preserves a non-OK return code while physical cleanup stays uncertain. These lock in the separation between the legacy stopped flag and physical-success proof. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 13aab826..af4b7147 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1530,3 +1530,32 @@ async fn wait_physical_cleanup(transport: &ModuleTransport) { } }).await.unwrap(); } + +#[tokio::test] +async fn observing_fault_owned_job_does_not_create_ordinary_stop_marker() { + let (transport, host) = ModuleTransport::new("fault-owner".into(), vec![]); + transport.reject_worker_spawn.store(true, Ordering::Release); + transport.defer_initialize(initialize_ok, host); + assert!(transport.ensure_initialized().await.is_err()); + assert!(!transport.stop_task_spawned()); + assert!(matches!(transport.clone().stop(Duration::from_secs(1)).await, Err(StopError::NotStarted(_)))); + assert!(!transport.stop_task_spawned(), "fault-owned uncertainty cannot become an ordinary handoff"); +} + +#[tokio::test] +async fn ordinary_shutdown_retains_non_ok_return_without_physical_success() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let (transport, host) = ModuleTransport::new("shutdown-status".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.unwrap(); + SHUTDOWN_CODE.store(TB_TIMEOUT, Ordering::Release); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let result = transport.clone().stop(Duration::from_secs(1)).await; + SHUTDOWN_CODE.store(TB_OK, Ordering::Release); + assert_eq!(result.unwrap(), TB_TIMEOUT, "ordinary public return remains compatible"); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 2); + assert_eq!(transport.context.physical.lock().unwrap().cleanup, CleanupState::Uncertain); + assert!(transport.stop_task_spawned()); + assert!(transport.is_stopped(), "legacy stopped is separate from physical-success proof"); + assert!(!transport.is_faulted()); +} From a2d5474694b411f95983336a67ac00b695e38847 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:02:58 +0300 Subject: [PATCH 31/58] fix(tinybus): classify stop errors by whether the worker was handed off The stop path now tracks whether an ordinary handoff actually occurred and uses that to decide between Started and NotStarted errors, so a faulted or already-cleaned-up transport no longer reports a spurious start. Uncertain cleanup with a recorded exit code is also treated as a successful stop, and the worker closure is bound before the rejection branch so the anchor is dropped rather than the submitted closure. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index a60196ac..8a4a35e1 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -458,13 +458,14 @@ impl ModuleTransport { let rejected = job.transport.reject_worker_spawn.load(Ordering::Acquire); #[cfg(not(test))] let rejected = false; + let worker = move || submitted.run(); let spawn = if rejected { // Model Builder rejecting and dropping the submitted closure: only // the independent context anchor can preserve its physical owner. - drop(submitted); + drop(worker); Err(std::io::Error::other("controlled worker rejection")) } else { - std::thread::Builder::new().name("tinybus-physical".into()).spawn(move || submitted.run()) + std::thread::Builder::new().name("tinybus-physical".into()).spawn(worker) }; if spawn.is_err() { let transport = &job.transport; @@ -861,24 +862,27 @@ impl ModuleTransport { } let remaining = deadline.saturating_sub(started.elapsed()); self.commit_terminal(false); - let (existing, job, mut acquired) = { + let (_existing, job, mut acquired, ordinary_handoff) = { let mut state = self.context.physical.lock().expect("physical state lock"); if state.cleanup == CleanupState::Idle { state.cleanup = CleanupState::Requested; } let existing = state.job.is_some(); let mut acquired = Some(lifecycle); - let job = if existing { None } else { + let job = if existing || state.cleanup != CleanupState::Requested { None } else { let job = Arc::new(PhysicalJob { transport: self.clone(), initializer: StdMutex::new(None), guard: StdMutex::new(acquired.take()), deadline: remaining }); state.job = Some(job.clone()); Some(job) }; - self.stop_task_spawned.store(true, Ordering::Release); - (existing, job, acquired) + let ordinary_handoff = !self.is_faulted() && state.cleanup == CleanupState::Requested + && (job.is_some() || (existing && state.init == InitState::Scheduled)); + if ordinary_handoff { self.stop_task_spawned.store(true, Ordering::Release); } + (existing, job, acquired, ordinary_handoff) }; // A Scheduled initializer must acquire the guard itself to prove no C entry. - if existing { drop(acquired.take()); } + drop(acquired.take()); if let Some(job) = job { Self::submit_job(job); } + let classify = |error| if ordinary_handoff { StopError::Started(error) } else { StopError::NotStarted(error) }; let wait = async { loop { let changed = self.context.physical_notify.notified(); @@ -886,7 +890,10 @@ impl ModuleTransport { let state = self.context.physical.lock().expect("physical state lock"); match state.cleanup { CleanupState::Complete | CleanupState::NoCallbackRequired => Some(Ok(state.code.unwrap_or(TB_CLOSED))), - CleanupState::Uncertain => Some(Err(StopError::Started(Error::transport("module cleanup is uncertain")))), + CleanupState::Uncertain => Some(match state.code { + Some(code) => Ok(code), + None => Err(classify(Error::transport("module cleanup is uncertain"))), + }), _ => None, } }; @@ -894,7 +901,7 @@ impl ModuleTransport { changed.await; } }; - tokio::time::timeout(remaining, wait).await.map_err(|_| StopError::Started(Error::failed("module shutdown exceeded its deadline")))? + tokio::time::timeout(remaining, wait).await.map_err(|_| classify(Error::failed("module shutdown exceeded its deadline")))? } } From 580c0140dd4f3b5a8c73dad7ed22da689590846b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:04:58 +0300 Subject: [PATCH 32/58] test(module): cover rejected outputs and native fault paths Add test-only hooks that pause the physical worker before it takes the lifecycle lock and before initialization results are published, letting tests drive stop, fault, and deadline races deterministically. The new tests assert that rejected module outputs never reach untrusted callbacks, that a failed cleanup spawn keeps the original lifecycle guard, and that native faults during initialization, reinitialization, Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/Cargo.toml | 2 +- .../examples/module_controlled_init.rs | 9 +- crates/tinybus/src/module/transport.rs | 20 +- crates/tinybus/src/module/transport_tests.rs | 215 ++++++++++++++++-- 4 files changed, 220 insertions(+), 26 deletions(-) diff --git a/crates/tinybus/Cargo.toml b/crates/tinybus/Cargo.toml index 8f6452d7..b98b3681 100644 --- a/crates/tinybus/Cargo.toml +++ b/crates/tinybus/Cargo.toml @@ -82,7 +82,7 @@ test-support = ["modules", "macros"] [dev-dependencies] -tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time", "io-util"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time", "io-util", "test-util"] } tempfile = "3" [[bin]] diff --git a/crates/tinybus/examples/module_controlled_init.rs b/crates/tinybus/examples/module_controlled_init.rs index a17adbf4..507a47da 100644 --- a/crates/tinybus/examples/module_controlled_init.rs +++ b/crates/tinybus/examples/module_controlled_init.rs @@ -14,6 +14,7 @@ unsafe impl Send for Controls {} struct Instance { controls: Controls, + host: TbHostVtable, stop: Arc<(Mutex, Condvar)>, worker: Option>, } @@ -32,7 +33,11 @@ pub extern "C" fn tinybus_module_manifest_v1() -> TbSlice { } unsafe extern "C" fn deliver(_: *mut c_void, _: *const u8, _: usize) -> i32 { TB_OK } -unsafe extern "C" fn reinitialize(_: *mut c_void, _: *const u8, _: usize) -> i32 { TB_OK } +unsafe extern "C" fn reinitialize(ctx: *mut c_void, _: *const u8, _: usize) -> i32 { + let instance = unsafe { &*ctx.cast::() }; + unsafe { (instance.controls.entered)(instance.controls.ctx, &instance.host); } + TB_OK +} unsafe extern "C" fn shutdown(ctx: *mut c_void, _: u64) -> i32 { let mut instance = unsafe { Box::from_raw(ctx.cast::()) }; *instance.stop.0.lock().unwrap() = true; @@ -60,7 +65,7 @@ pub unsafe extern "C" fn tinybus_module_init_v1(host: *const TbHostVtable, out: let mut stopped = worker_stop.0.lock().unwrap(); while !*stopped { stopped = worker_stop.1.wait(stopped).unwrap(); } }); - let instance = Box::new(Instance { controls, stop, worker: Some(worker) }); + let instance = Box::new(Instance { controls, host: *host, stop, worker: Some(worker) }); unsafe { *out = TbModuleVtable { size: size_of::() as u32, _reserved: 0, module_ctx: Box::into_raw(instance).cast(), deliver, shutdown, diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 8a4a35e1..32779775 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -69,6 +69,10 @@ pub(crate) struct ModuleTransport { initialization_gate_changed: Notify, #[cfg(test)] reject_worker_spawn: AtomicBool, + #[cfg(test)] + before_worker_lock: StdMutex>>, + #[cfg(test)] + before_init_publication: StdMutex>>, } // Reservations cover physical callback execution, including signals. They are @@ -238,6 +242,10 @@ impl ModuleTransport { initialization_gate_changed: Notify::new(), #[cfg(test)] reject_worker_spawn: AtomicBool::new(false), + #[cfg(test)] + before_worker_lock: StdMutex::new(None), + #[cfg(test)] + before_init_publication: StdMutex::new(None), }); *context.owner.lock().expect("physical owner lock") = Arc::downgrade(&transport); let config = context.config.lock().expect("module config lock"); @@ -909,6 +917,11 @@ impl ModuleTransport { impl PhysicalJob { fn run(self: Arc) { let transport = &self.transport; + #[cfg(test)] + { + let hook = transport.before_worker_lock.lock().expect("worker test hook lock").take(); + if let Some(hook) = hook { hook(); } + } let initializer = self.initializer.lock().expect("physical initializer lock").take(); if let Some(initializer) = initializer { let guard = transport.lifecycle.clone().blocking_lock_owned(); @@ -937,6 +950,11 @@ impl PhysicalJob { Err(_) => Err(InitFailure::Failed("module initialization panicked".into())), }; transport.clear_config(); + #[cfg(test)] + { + let hook = transport.before_init_publication.lock().expect("publication test hook lock").take(); + if let Some(hook) = hook { hook(); } + } if result.is_err() { transport.commit_terminal(true); } let cleanup = { let mut state = transport.context.physical.lock().expect("physical state lock"); @@ -1219,4 +1237,4 @@ unsafe extern "C" fn host_ready(ctx: *mut c_void) { #[cfg(test)] #[path = "transport_tests.rs"] -mod tests; +pub(super) mod tests; diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index af4b7147..aabd3f50 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1377,26 +1377,19 @@ async fn scheduled_initializer_is_adopted_by_original_ordinary_stop() { let entries = entries.clone(); move |_| { entries.fetch_add(1, Ordering::AcqRel); Err("unexpected C entry".into()) } }); - // Hold the job before submission so stop can acquire lifecycle first. - transport.reject_worker_spawn.store(true, Ordering::Release); - // Scheduling rejection itself must fault, so normal adoption is tested - // with an anchored job whose worker has not yet acquired lifecycle. - let initializer = transport.initializer.lock().unwrap().take().unwrap(); - let job = Arc::new(PhysicalJob { - transport: transport.clone(), initializer: StdMutex::new(Some(initializer)), - guard: StdMutex::new(None), deadline: MODULE_INIT_DEADLINE, - }); - { - let mut state = transport.context.physical.lock().unwrap(); - state.init = InitState::Scheduled; - state.job = Some(job.clone()); - } + let (worker_started, worker_started_rx) = tokio::sync::oneshot::channel(); + let (worker_release, worker_release_rx) = std::sync::mpsc::channel(); + *transport.before_worker_lock.lock().unwrap() = Some(Box::new(move || { + worker_started.send(()).unwrap(); + worker_release_rx.recv().unwrap(); + })); + assert!(transport.schedule_initializer().unwrap()); + worker_started_rx.await.unwrap(); let mut stopping = Box::pin(transport.clone().stop(Duration::from_secs(2))); tokio::select! { result = &mut stopping => panic!("stop completed before owner: {result:?}"), _ = async { while !transport.stop_task_spawned() { tokio::task::yield_now().await; } } => {} } assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 2); - transport.reject_worker_spawn.store(false, Ordering::Release); - ModuleTransport::submit_job(job); + worker_release.send(()).unwrap(); assert_eq!(stopping.await.unwrap(), TB_CLOSED); assert_eq!(entries.load(Ordering::Acquire), 0); assert!(!transport.is_faulted()); @@ -1430,11 +1423,11 @@ struct NativeControls { entered: unsafe extern "C" fn(*mut c_void, *const TbHostVtable), joined: unsafe extern "C" fn(*mut c_void), } -struct NativeBarrier { - state: StdMutex<(Option, bool, bool)>, - changed: Condvar, - entered: tokio::sync::Notify, - joined: tokio::sync::Notify, +pub(crate) struct NativeBarrier { + pub(crate) state: StdMutex<(Option, bool, bool)>, + pub(crate) changed: Condvar, + pub(crate) entered: tokio::sync::Notify, + pub(crate) joined: tokio::sync::Notify, } unsafe impl Send for NativeBarrier {} unsafe impl Sync for NativeBarrier {} @@ -1452,7 +1445,7 @@ unsafe extern "C" fn native_joined(ctx: *mut c_void) { state.2 = true; barrier.joined.notify_one(); } -fn native_fixture() -> (Arc, TbHostVtable, crate::module::abi::TbModuleInit, Arc) { +pub(crate) fn native_fixture() -> (Arc, TbHostVtable, crate::module::abi::TbModuleInit, Arc) { let path = std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").expect("controlled native fixture path"); let artifact = crate::module::loader::load(std::path::Path::new(&path), false).unwrap(); let barrier = Arc::new(NativeBarrier { @@ -1559,3 +1552,181 @@ async fn ordinary_shutdown_retains_non_ok_return_without_physical_success() { assert!(transport.is_stopped(), "legacy stopped is separate from physical-success proof"); assert!(!transport.is_faulted()); } + +#[tokio::test] +async fn rejected_module_outputs_never_invoke_untrusted_callbacks() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + SHUTDOWN_CALLS.store(0, Ordering::Release); + for rejection in 0..4 { + let (transport, host) = ModuleTransport::new(format!("rejected-{rejection}"), vec![]); + transport.defer_initializer(host, move |host| { + let mut module = TbModuleVtable::default(); + unsafe { initialize_ok(&host, &mut module); } + match rejection { + 0 => module.size = TB_MODULE_VTABLE_BASE_SIZE - 1, + 1 => module.module_ctx = std::ptr::null_mut(), + 2 => module.reinitialize = None, + _ => return Err("non-OK output must remain inaccessible".into()), + } + Ok(module) + }); + assert!(transport.ensure_initialized().await.is_err()); + let physical = transport.context.physical.lock().unwrap(); + assert_eq!(physical.init, InitState::PublishedRejected); + assert_eq!(physical.cleanup, CleanupState::Uncertain); + assert!(physical.job.is_some()); + assert!(transport.module.lock().unwrap().is_none()); + drop(physical); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + } + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 0); +} + +#[tokio::test] +async fn rejected_ordinary_worker_submission_retains_original_lifecycle_guard() { + let (transport, host) = ModuleTransport::new("stop-spawn-rejection".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.unwrap(); + transport.reject_worker_spawn.store(true, Ordering::Release); + assert!(matches!(transport.clone().stop(Duration::from_secs(1)).await, Err(StopError::Started(_)))); + assert!(transport.stop_task_spawned()); + assert!(transport.lifecycle.try_lock().is_err(), "rejected closure cannot drop actual acquired guard"); + let weak = Arc::downgrade(&transport); + drop(transport); + let retained = weak.upgrade().expect("actual owner retained beyond host handle"); + assert_eq!(retained.context.physical.lock().unwrap().cleanup, CleanupState::Uncertain); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 2); + assert!(!retained.is_faulted(), "ordinary spawner failure is uncertainty, not module fault"); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_deadline_fault_retains_late_cleanup_without_wall_clock_sleep() { + let (transport, host, init, barrier) = native_fixture(); + transport.defer_initialize(init, host); + let waiter = tokio::spawn({ + let transport = transport.clone(); async move { transport.ensure_initialized().await } + }); + barrier.entered.notified().await; + tokio::time::pause(); + tokio::time::advance(MODULE_INIT_DEADLINE).await; + assert!(waiter.await.unwrap().is_err()); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + assert!(!barrier.state.lock().unwrap().2); + { + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); + } + // Resume the real timer only for the outer stuck-test safety deadline. + tokio::time::resume(); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + wait_physical_cleanup(&transport).await; + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_fault_between_return_and_publication_uses_same_owner() { + let (transport, host, init, barrier) = native_fixture(); + let (publication_tx, publication_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + *transport.before_init_publication.lock().unwrap() = Some(Box::new(move || { + publication_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + })); + transport.defer_initialize(init, host); + let waiter = tokio::spawn({ + let transport = transport.clone(); async move { transport.ensure_initialized().await } + }); + barrier.entered.notified().await; + { + let mut state = barrier.state.lock().unwrap(); state.1 = true; barrier.changed.notify_all(); + } + publication_rx.await.unwrap(); + let job = transport.context.physical.lock().unwrap().job.clone().unwrap(); + unsafe { (host.fault)(host.host_ctx, std::ptr::null(), 0); } + { + let state = transport.context.physical.lock().unwrap(); + assert!(Arc::ptr_eq(&job, state.job.as_ref().unwrap())); + assert_eq!(state.cleanup, CleanupState::Requested); + assert!(state.result.is_none(), "cleanup must not claim an empty publication"); + } + assert!(!barrier.state.lock().unwrap().2); + release_tx.send(()).unwrap(); + assert!(waiter.await.unwrap().is_err()); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + wait_physical_cleanup(&transport).await; + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_reinitializer_fault_denies_before_c_returns_and_joins_once() { + let (transport, host, init, barrier) = native_fixture(); + { + let mut state = barrier.state.lock().unwrap(); state.1 = true; + } + transport.defer_initialize(init, host); + transport.ensure_initialized().await.unwrap(); + barrier.entered.notified().await; + { + let mut state = barrier.state.lock().unwrap(); state.1 = false; + } + let reinit = tokio::spawn({ + let transport = transport.clone(); + async move { transport.reinitialize(crate::Secret::new(vec![])).await } + }); + barrier.entered.notified().await; + unsafe { (host.fault)(host.host_ctx, std::ptr::null(), 0); } + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + assert!(!barrier.state.lock().unwrap().2); + assert!(transport.lifecycle.try_lock().is_err(), "actual C reinit owns physical exclusion"); + { + let mut state = barrier.state.lock().unwrap(); state.1 = true; barrier.changed.notify_all(); + } + reinit.await.unwrap().unwrap(); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + wait_physical_cleanup(&transport).await; + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); + assert!(transport.reinitialize(crate::Secret::new(vec![])).await.is_err()); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_late_eager_publication_cannot_refund_failed_cleanup_spawn() { + let (transport, host, init, barrier) = native_fixture(); + transport.reject_worker_spawn.store(true, Ordering::Release); + let host_owner = DeferredInitializer { initialize: Box::new(|_| unreachable!()), host }; + let eager = std::thread::spawn({ + let transport = transport.clone(); + move || { let owner = host_owner; transport.eager_initialize(init, owner.host) } + }); + barrier.entered.notified().await; + unsafe { (host.fault)(host.host_ctx, std::ptr::null(), 0); } + let job = transport.context.physical.lock().unwrap().job.clone().unwrap(); + assert_eq!(transport.context.physical.lock().unwrap().cleanup, CleanupState::Uncertain); + { + let mut state = barrier.state.lock().unwrap(); state.1 = true; barrier.changed.notify_all(); + } + assert!(eager.join().unwrap().is_err()); + transport.reject_worker_spawn.store(false, Ordering::Release); + transport.request_terminal(true); + { + let state = transport.context.physical.lock().unwrap(); + assert_eq!(state.init, InitState::ExternalValid); + assert_eq!(state.cleanup, CleanupState::Uncertain); + assert!(Arc::ptr_eq(&job, state.job.as_ref().unwrap())); + } + assert!(transport.module.lock().unwrap().is_some(), "valid late output is cleanup access only"); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + assert!(!barrier.state.lock().unwrap().2, "rejected submission must never be silently retried"); + let weak = Arc::downgrade(&transport); + drop(transport); + drop(job); + assert!(weak.upgrade().is_some()); + // The deliberately unjoinable native instance retains its externally anchored + // controller too. This is one explicit uncertain test instance, not global + // leak-bound or secret-erasure proof. + std::mem::forget(barrier); +} From 601478ac18a24b08df86242da2fb9d1a4d94913d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:05:49 +0300 Subject: [PATCH 33/58] fix(module): make spawn-failure faulting configurable per job PhysicalJob now carries a fault_on_spawn_failure flag so that a failed worker spawn only commits a terminal fault when the caller asked for it. The recovery path that re-arms a job after a requested cleanup passes false, since a spawn failure there is expected and should not fault the module. The ABI doc for TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED was also rewritten to clarify that the bit only records an accepted TB_OK return Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/abi.rs | 10 ++++++---- crates/tinybus/src/module/transport.rs | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/tinybus/src/module/abi.rs b/crates/tinybus/src/module/abi.rs index 1d8d41cb..07d33db4 100644 --- a/crates/tinybus/src/module/abi.rs +++ b/crates/tinybus/src/module/abi.rs @@ -37,10 +37,12 @@ pub const TB_NATIVE_TERMINAL_LIVE: u32 = 1; pub const TB_NATIVE_TERMINAL_STOP_INTENT: u32 = 2; /// Actual fault intent, with public fault state taking precedence over stop. pub const TB_NATIVE_TERMINAL_FAULT: u32 = 4; -/// Successful opaque shutdown return. For modules opting into native terminal -/// hooks, successful finish must also prove their independent worker joined. -/// This bit proves neither legacy worker completion nor receiver backing -/// destruction. Faulted or uncertain shutdown must never set it. +/// An accepted opaque shutdown callback actually returned TB_OK. Fault and +/// stop intent remain set and continue to deny admission even with this bit. +/// A legacy callback's TB_OK alone proves neither independent worker completion +/// nor receiver backing destruction. For native terminal hooks, TB_OK must +/// additionally require actual worker/resource finish without sticky uncertainty +/// or fault; that stronger module contract is separate from this host evidence. pub const TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED: u32 = 8; /// Revision-one module initialization entrypoint. diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 32779775..05fe347e 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -152,6 +152,7 @@ struct PhysicalJob { initializer: StdMutex>, guard: StdMutex>>, deadline: Duration, + fault_on_spawn_failure: bool, } struct InitiatingWaiter<'a> { transport: &'a ModuleTransport, armed: bool } @@ -449,6 +450,7 @@ impl ModuleTransport { transport: self.self_ref.upgrade().ok_or(Error::ConnectionClosed)?, initializer: StdMutex::new(Some(initializer)), guard: StdMutex::new(None), deadline: MODULE_INIT_DEADLINE, + fault_on_spawn_failure: true, }); state.init = InitState::Scheduled; state.job = Some(job.clone()); @@ -477,7 +479,7 @@ impl ModuleTransport { }; if spawn.is_err() { let transport = &job.transport; - transport.commit_terminal(true); + transport.commit_terminal(job.fault_on_spawn_failure); let mut state = transport.context.physical.lock().expect("physical state lock"); state.cleanup = CleanupState::Uncertain; state.result.get_or_insert(Err(InitFailure::Failed("module physical worker could not start".into()))); @@ -519,7 +521,7 @@ impl ModuleTransport { } let Some(transport) = self.self_ref.upgrade() else { return; }; let job = Arc::new(PhysicalJob { transport, initializer: StdMutex::new(None), - guard: StdMutex::new(None), deadline: MODULE_INIT_DEADLINE }); + guard: StdMutex::new(None), deadline: MODULE_INIT_DEADLINE, fault_on_spawn_failure: fault }); state.job = Some(job.clone()); drop(state); Self::submit_job(job); @@ -546,7 +548,7 @@ impl ModuleTransport { if state.job.is_none() { state.job = self.self_ref.upgrade().map(|transport| Arc::new(PhysicalJob { transport, initializer: StdMutex::new(None), guard: StdMutex::new(None), - deadline: MODULE_INIT_DEADLINE, + deadline: MODULE_INIT_DEADLINE, fault_on_spawn_failure: true, })); } } @@ -878,7 +880,7 @@ impl ModuleTransport { let job = if existing || state.cleanup != CleanupState::Requested { None } else { let job = Arc::new(PhysicalJob { transport: self.clone(), initializer: StdMutex::new(None), guard: StdMutex::new(acquired.take()), - deadline: remaining }); + deadline: remaining, fault_on_spawn_failure: false }); state.job = Some(job.clone()); Some(job) }; From 64543cfc99bcc096a6db75d0ec513cdae6c449ec Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:07:27 +0300 Subject: [PATCH 34/58] fix(module): commit terminal state on failed load and rollback Failed module loads and reservation rollbacks now mark the transport terminal before revoking the binding, so a rejected instance cannot be reused and its cleanup runs exactly once. Tests cover host drop, reservation rollback, and faults during ordinary stop against a real native module. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host.rs | 1 + crates/tinybus/src/module/host_callbacks.rs | 2 + crates/tinybus/src/module/host_tests.rs | 87 ++++++++++++++++++++ crates/tinybus/src/module/transport_tests.rs | 6 +- 4 files changed, 95 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/host.rs b/crates/tinybus/src/module/host.rs index 32ca32c3..eb096492 100644 --- a/crates/tinybus/src/module/host.rs +++ b/crates/tinybus/src/module/host.rs @@ -1137,6 +1137,7 @@ impl ModuleHost { { Ok(change) => change, Err(_) => { + transport.commit_terminal(true); self.inner .broker .router diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 8b4931c4..49c6c29b 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -586,6 +586,7 @@ impl ModuleHost { false }; if !ready { + transport.commit_terminal(true); binding.revoke(); transport.request_terminal(true); let _ = transport.close().await; @@ -638,6 +639,7 @@ impl Drop for LoadRollback<'_> { if self.committed { return; } + if let Some(transport) = &self.transport { transport.commit_terminal(true); } self.binding.revoke(); self.binding.transport.close_outbound(); self.binding diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index ef6db918..76b09720 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -3056,3 +3056,90 @@ async fn ordinary_stop_never_fabricates_faulted_before_or_after_callback_complet ); broker_task.abort(); } + +async fn controlled_native_transport() -> ( + Arc, crate::module::abi::TbHostVtable, + Arc, LoadedArtifact, PathBuf, +) { + let (transport, view, init, barrier) = super::super::transport::tests::native_fixture(); + barrier.state.lock().unwrap().1 = true; + transport.defer_initialize(init, view); + transport.ensure_initialized().await.unwrap(); + barrier.entered.notified().await; + let path = PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); + let artifact = loader::load(&path, false).unwrap(); + (transport, view, barrier, artifact, path) +} + +fn attach_controlled_native(host: &ModuleHost, transport: Arc, artifact: LoadedArtifact, path: &Path) -> Result { + let mut admitted = host.validate(path, &artifact.descriptor, &artifact.manifest)?; + admitted.state = ModuleState::Initializing; + host.attach_transport(path, admitted, artifact.manifest, transport, Activation { + lazy_init: false, lazy_load: false, descriptor_info: None, pinned_sha256: None, + }) +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_host_drop_keeps_terminal_tombstone_and_other_instance_live() { + let (transport, view, barrier, artifact, path) = controlled_native_transport().await; + let (other, other_view, other_barrier, _, _) = controlled_native_transport().await; + let host = ModuleHost::new(Broker::new()); + attach_controlled_native(&host, transport.clone(), artifact, &path).unwrap(); + drop(host); + assert_ne!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 1); + assert_eq!(unsafe { other_view.native_terminal_state.unwrap()(other_view.host_ctx) }, 1); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + let stopped = transport.clone().stop(Duration::from_secs(2)).await; + assert!(matches!(stopped, Err(StopError::NotStarted(_)))); + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 10); + drop(transport); + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 10); + other.request_terminal(false); + tokio::time::timeout(Duration::from_secs(2), other_barrier.joined.notified()).await.unwrap(); + assert!(barrier.state.lock().unwrap().2); + assert!(other_barrier.state.lock().unwrap().2); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_reservation_rollback_retains_one_cleanup_and_fault_tombstone() { + let (first, first_view, first_barrier, artifact, path) = controlled_native_transport().await; + let (rejected, view, barrier, other_artifact, _) = controlled_native_transport().await; + let host = ModuleHost::new(Broker::new()); + attach_controlled_native(&host, first, artifact, &path).unwrap(); + assert!(attach_controlled_native(&host, rejected.clone(), other_artifact, &path).is_err()); + assert_ne!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 1); + assert_eq!(unsafe { first_view.native_terminal_state.unwrap()(first_view.host_ctx) }, 1); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + let _observed = rejected.clone().stop(Duration::from_secs(2)).await; + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 14); + assert!(rejected.ensure_initialized().await.is_err()); + drop(rejected); + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 14); + drop(host); + tokio::time::timeout(Duration::from_secs(2), first_barrier.joined.notified()).await.unwrap(); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_fault_during_ordinary_stop_keeps_fault_state_and_single_join() { + let (transport, view, barrier, artifact, path) = controlled_native_transport().await; + let host = ModuleHost::new(Broker::new()); + let info = attach_controlled_native(&host, transport.clone(), artifact, &path).unwrap(); + let (release, release_rx) = std::sync::mpsc::channel(); + *barrier.finish_gate.lock().unwrap() = Some(release_rx); + let stopping = tokio::spawn({ + let host = host.clone(); async move { host.stop(info.name, Duration::from_secs(2)).await } + }); + barrier.joined.notified().await; + assert!(transport.stop_task_spawned()); + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 2, "joined worker is not yet successful opaque return"); + unsafe { (view.fault)(view.host_ctx, std::ptr::null(), 0); } + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 6); + release.send(()).unwrap(); + let result = stopping.await.unwrap().unwrap(); + assert!(matches!(result.state, ModuleState::Faulted { .. }), "late ordinary completion cannot replace actual fault"); + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 14); + assert!(barrier.state.lock().unwrap().2); +} diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index aabd3f50..41b717e5 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1428,6 +1428,7 @@ pub(crate) struct NativeBarrier { pub(crate) changed: Condvar, pub(crate) entered: tokio::sync::Notify, pub(crate) joined: tokio::sync::Notify, + pub(crate) finish_gate: StdMutex>>, } unsafe impl Send for NativeBarrier {} unsafe impl Sync for NativeBarrier {} @@ -1444,13 +1445,16 @@ unsafe extern "C" fn native_joined(ctx: *mut c_void) { assert!(!state.2, "one actual native shutdown/join"); state.2 = true; barrier.joined.notify_one(); + drop(state); + let gate = barrier.finish_gate.lock().unwrap().take(); + if let Some(gate) = gate { gate.recv().unwrap(); } } pub(crate) fn native_fixture() -> (Arc, TbHostVtable, crate::module::abi::TbModuleInit, Arc) { let path = std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").expect("controlled native fixture path"); let artifact = crate::module::loader::load(std::path::Path::new(&path), false).unwrap(); let barrier = Arc::new(NativeBarrier { state: StdMutex::new((None, false, false)), changed: Condvar::new(), - entered: Notify::new(), joined: Notify::new(), + entered: Notify::new(), joined: Notify::new(), finish_gate: StdMutex::new(None), }); let controls = NativeControls { ctx: Arc::as_ptr(&barrier).cast_mut().cast(), entered: native_entered, joined: native_joined }; let config = unsafe { std::slice::from_raw_parts(std::ptr::from_ref(&controls).cast::(), size_of::()) }.to_vec(); From 9d2bf12d7d8e8be7c135f6660042debf7ee17df8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:09:20 +0300 Subject: [PATCH 35/58] fix(module): wrap test host in Arc for native fault test The native fault-during-stop test now constructs the module host inside an Arc so it matches the shared ownership the host API expects. The native terminal authority plan was rewritten to describe the retained initialization, publication and terminal cleanup protocol in more detail. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_tests.rs | 2 +- docs/plans/native-terminal-authority.md | 203 +++++++++++++++--------- 2 files changed, 132 insertions(+), 73 deletions(-) diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 76b09720..9e225f3f 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -3125,7 +3125,7 @@ async fn actual_native_reservation_rollback_retains_one_cleanup_and_fault_tombst #[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] async fn actual_native_fault_during_ordinary_stop_keeps_fault_state_and_single_join() { let (transport, view, barrier, artifact, path) = controlled_native_transport().await; - let host = ModuleHost::new(Broker::new()); + let host = Arc::new(ModuleHost::new(Broker::new())); let info = attach_controlled_native(&host, transport.clone(), artifact, &path).unwrap(); let (release, release_rx) = std::sync::mpsc::channel(); *barrier.finish_gate.lock().unwrap() = Some(release_rx); diff --git a/docs/plans/native-terminal-authority.md b/docs/plans/native-terminal-authority.md index c073b246..23b88719 100644 --- a/docs/plans/native-terminal-authority.md +++ b/docs/plans/native-terminal-authority.md @@ -106,79 +106,127 @@ ABI constants Live=1, StopIntent=2, Fault=4, PhysicallyStopped=8. Exactly 1 admi zero/unknown/combined/terminal flags deny. Terminal flags monotonic, never Live. Physical bit means successful opaque shutdown return, and for opted-in modules explicit finish proved worker join. It proves neither legacy worker completion -nor receiver backing destruction. Fault/uncertain shutdown never earns this bit. +nor receiver backing destruction. The generic host sets the bit only after an +accepted opaque callback actually returns TB_OK; StopIntent and Fault remain +monotonic and continue denying admission (a legacy fault cleanup can return +flags 14). This is not opted-in hook acceptance. For new native terminal hooks, +TB_OK additionally requires actual independent worker/SDK resource finish and +no sticky uncertainty or fault, under the separate Task 4 protocol. Normalization independently checks offset_of(field) + sizeof(field type) for broker_routing and new callback; never sizeof enlarged whole vtable for old tail. Frozen prefix/base size and ABI version stay unchanged. Document stronger callback lifetime covering every retained view use. Run focused tests green. -## Task 2: retain real deferred initialization before deadline abandonment +## Task 2: retain initialization and publication independently of waiters Files: crates/tinybus/src/module/transport.rs and transport_tests.rs, -host.rs/host_tests.rs, host_callbacks.rs/host_callbacks_tests.rs. - -Existing DeferredInitializer::run returns SendModuleVtable through spawn_blocking; -ensure_initialized timeout drops JoinHandle, so eventual vtable can disappear -before self.initialize validates/stores it. Current stop_sync rollback can also -overlap opaque init. Necessary correction stays in these exact paths. - -First red test late_initialization_retains_real_vtable_for_cleanup: actual blocked -initializer, controlled host future timeout/drop, retained native worker/query, -release init returning valid vtable, exactly one real shutdown/join, no new phase. -Add rollback_cannot_overlap_opaque_initialization and late_invalid_vtable_never_runs. - -Retained blocking initializer owns Arc transport, actual owned lifecycle guard, -DeferredInitializer (and config borrow), resident library assumptions and eventual -vtable. Install via existing initialize validation before result publication to -separate waiter channel; JoinHandle result carries no exclusive physical owner. -Validate size/header/version/callback validity under existing initialize contract; -never invoke invalid/missing callbacks or fabricate a controller from partial ABI. -Failure with no usable returned vtable retains honest uncertainty for a foreign -initializer that launched work without returning cleanup; native query still denies. - -Host deadline/drop does not drop result owner or borrow. Timeout commits native -fault intent immediately, makes public failed/fault state terminal, and retains -one eventual cleanup. Clear config only after actual C return. Publish validated -vtable even when terminal, then release lifecycle guard and claim/schedule cleanup -once. Terminal vtable storage is cleanup access, never public resurrection. -No new initializer retries or unbounded spawn_blocking loops: one physical init, -one claim for cleanup; caller retries only observe same result/tombstone. -If init never returns, cleanup pending and real ownership survive indefinitely. - -## Task 3: actual host flags, fault cleanup and original stop invariants - -Same host files. Red tests fault_closes_native_phase_before_reinitializer_returns, -unacquired_stop_keeps_native_instance_live, abandoned_stop_keeps_original_cleanup, -host_drop_leaves_terminal_query_tombstone, and -fault_during_stop_keeps_fault_transition_and_single_shutdown. - -HostContext adds AtomicU32 terminal flags; commit_terminal clears Live and sets -StopIntent plus Fault when real fault. Query only Acquire loads leaked actual ctx, -no locks/allocation/parsing/product code. Install callback in ModuleTransport::new. -HostContext immutable Weak from Arc::new_cyclic enables host_fault -to schedule cleanup without retaining strong context cycle. - -Separate physical cleanup claim from existing stop_task_spawned. Normal stop -acquires lifecycle then claims/commits before spawn. Fault immediately commits, -then private schedule_terminal_cleanup retains Arc and queues one serialized -physical cleanup. Retained std thread using blocking_lock_owned permits Host Drop -cleanup independent of dropped Tokio runtime; never recurse into C shutdown from -host_fault, opaque callback, router/loaded lock or Tokio worker. Thread spawn failure -keeps terminal/uncertain ownership; no successful completion/reopen/refund. -Normal running task and fault share claim, never second join. stop_task_spawned -still denotes actual ordinary stop task, preserving StopError Started/NotStarted. - -Route host_fault, fault_pending, failed initialization/send, admission rollback -and Host Drop through same intent/claim. Gather cloned transports under lock, -commit intent before detach, release loaded/router/inbound locks before scheduling. -Remove synchronous rollback shutdown under lifecycle-bypassing Drop path. -Before-init rollback closes init gate without pretending worker joined. - -peer_detached checks actual fault before consuming normal transition_from; -private fault cleanup never creates that origin. Late stop_inner success preserves -actual Faulted/Failed precedence. Ordinary finish_stop stays non-fault. Keep every -existing #57 callback lease/drain/cancel/retry/original marker assertion unchanged. +host.rs/host_tests.rs, host_callbacks.rs/host_callbacks_tests.rs; declared +module_controlled_init native fixture and its existing CI staging. + +Use one mutex-protected PhysicalState in the existing process-lifetime +HostContext. InitState is Deferred, Scheduled, Entered, PublishedValid, +PublishedRejected, CanceledBeforeEntry or ExternalValid. CleanupState is Idle, +Requested, Executing, Complete, Uncertain or NoCallbackRequired. These are +orthogonal states: Requested does not consume the shutdown invocation, and an +empty vtable slot never proves physical success. + +Before thread submission, install one externally anchored Arc +owning Arc, at most one initializer, and any acquired owned +lifecycle guard. The context anchor survives rejected/dropped spawn closures, +Host Drop and Tokio runtime teardown. Never retry failed submission or reset +Uncertain. This is a finite per-instance ownership bound, not a global leak +bound. Result predicates use Notify and Condvar loops, not notification counts. + +The non-async scheduling helper takes initialization-gate then PhysicalState, +rechecks terminal admission, takes the initializer once and records Scheduled +before awaiting physical completion. OnceCell does not own initializer scheduling +or eventual vtable publication. Only the initiating waiter has an armed Drop +guard: deadline expiry or abandonment before publication immediately commits +Fault and Requested. Observation waiters cannot cancel the physical owner. +Already-committed ordinary cancellation does not invent Fault. + +The retained std thread acquires the actual owned lifecycle guard without +holding host state locks. Before C entry it briefly checks PhysicalState: +terminal Scheduled becomes CanceledBeforeEntry/NoCallbackRequired without any +opaque call; live Scheduled becomes Entered. Entered retains config until real +C return regardless of a host deadline or dropped waiter. After C return, +validate only the existing contract: module size >= base size, non-null context, +and Some(reinitialize) if the full module tail is declared. No fictitious +header/version validation, null testing of mandatory Rust function pointers, +or executable-address authentication is added. Non-OK output is ignored; +structurally rejected output invokes no callbacks and retains Uncertain. + +Store accepted vtables under lifecycle exclusion even after terminal intent, +for cleanup-only access. Clear config after the real borrow ends and publish the +immutable result. Under PhysicalState, Requested keeps the SAME initializer +anchor for cleanup; live publication retires the anchor atomically before +lifecycle release; rejection retains uncertainty. Requested cleanup releases +then reacquires lifecycle, never waits for its own guard or submits a second +cleanup worker. Reinit must recheck terminal after obtaining lifecycle. + +Eager activation uses the same Entered/publication predicates. Its fresh, +unexposed transport is demonstrably unlocked: try_lock_owned before opaque C +entry, fail content-free/uncertain on unexpected contention, retain caller Arc, +config and guard through actual return and validation/publication, then release +exclusion. Cleanup waits both eager and deferred Entered publication before +acquiring lifecycle. Late accepted eager output is cleanup only and cannot +attach or reactivate. Failed cleanup submission remains Uncertain even if +eager output is valid later; no refund or resubmission. + +Prove with controlled channels/Condvar barriers and an actual compiled fixture: +blocked native init followed by initiating deadline/drop and late single join; +fault while Scheduled (C never entered); fault after C return before publication +(same owner, no empty-success claim); actual rejected closure/controller retention; +invalid size/null context/missing declared reinit/non-OK misleading output with +zero callbacks; eager fault while config remains borrowed and late single join. + +## Task 3: actual host flags and retained terminal cleanup + +HostContext supplies the actual AtomicU32 query through the optional ABI tail: +Live=1, StopIntent=2, Fault=4, PhysicallyStopped=8. Only exactly Live permits a +phase. Query performs an Acquire load on the leaked context without locks, +allocation, parsing or product logic. Weak allows host_fault to +upgrade the actual owner without an unconditional strong transport cycle. + +Commit native StopIntent (and actual Fault) immediately before physical cleanup +scheduling. Fault precedence is monotonic. host_fault releases inbound locks +before scheduling and never invokes C recursively, joins or waits. Rollback and +Host Drop commit all flags before revocation, release router/loaded locks and +route through retained cleanup. No lifecycle-bypassing synchronous Drop cleanup. + +Normal stop remains deadline-bound while obtaining the actual owned lifecycle +guard. Before acquisition it owns no ordinary intent, job or stop_task_spawned +marker; cancellation/deadline is NotStarted and remains live absent real fault. +After acquisition commit and install/adopt an actual owner synchronously before +the next await. If an already-submitted initializer is Scheduled, ordinary stop +adopts that SAME job, records the original handoff, releases its acquired guard, +then observes completion; the existing worker obtains lifecycle, sees terminal +and proves CanceledBeforeEntry. Never store the guard where that worker needs it +or create a second worker. A running Entered initializer remains excluded; an +ordinary lock deadline does not cancel it or fabricate Fault. + +Fault-owned jobs are observed rather than claimed as new ordinary handoffs. +Original stop_task_spawned and transition_from.get_or_insert(old) survive retry +and caller cancellation. Preserve Started/NotStarted distinctions and ordinary +non-Fault behavior, including an actual non-OK shutdown return. Failed ordinary +submission retains the actual acquired guard in the external anchor and explicit +Uncertain without inventing module Fault or silently retrying. + +Cleanup waits initialization publication before lifecycle acquisition, then +rechecks terminal/cleanup state, closes the actual callback gate and drains its +physical leases. Only accepted cleanup access can claim Requested -> Executing. +Hold physical exclusion and retained job until the one opaque shutdown returns. +Only actual TB_OK permits Complete and the physical-success bit; failed return +or panic is Uncertain. Legacy public Stopped is separate bookkeeping and never +proves join or backing destruction. Successful cleanup can release the anchor; +uncertainty remains inspectable, with no reset/restart. + +peer_detached checks actual Fault before consuming ordinary transition_from. +Late stop_inner completion preserves Faulted/Failed. Native tests cover blocked +actual reinit fault/query denial before return, ordinary callback lease retention, +Host Drop/rollback tombstones, instance isolation and one real join. Keep every +original #57 callback drain, physical lock, retry and original marker assertion. ## Task 4: module-local hooks through runtime and fault paths @@ -200,13 +248,24 @@ then runs finish synchronously on retained physical host cleanup worker before runtime.shutdown_timeout. Indefinite join outlives deadline without TB_OK. Uncertain/panic finish returns content-free failure, never physical completion. -Caught method panic publishes host fault even if error-frame send fails, then -bounded request outside registry lock. Determine actual send lock boundary before -coding: if lock is held, retained controller task runs request after host intent. -Setup failure publishes host intent before waiting callback disposal, request and -finish run off runtime workers. Setup canceled by shutdown takes same once route. -No joining/hooks in global panic hook (location logging only). setup/reinit reject -terminal immediately and before activation. Never classify ordinary close as fault. +Task 4 remains separate, unimplemented scope. Its startup guard owns returned +hooks until runtime/output publication; factory Err/panic before return owns +partial disposal itself. Request callback is an independent Arc and finish a +separate take-once slot. Publish intent, claim bounded request outside locks, +signal request completion even on panic; physical finish waits request-done +off runtime, callback-gate and reinitializer locks. Pre-registration fault +persists and invokes bounded request once at registration. Ordinary +terminal-caused setup cancellation must remain non-Fault, while genuine +Err/panic still upgrades Fault even racing ordinary stop. + +The accepted host prerequisite does not establish catch-time fault publication, +module-local worker/resource finish, fixed retained job bounds or confidential +payload/backing last-Drop. Task 4's own accepted specification must prove those +contracts at the actual CatchUnwind boundary before payload destruction; no +enqueue/send-success inference, extra panic task or global panic hook join. +A caught registry-free callback boundary must publish fault/request before +returning its error. New physical-success evidence remains distinct from legacy +Stopped and receiver backing destruction. ## Task 5: actual compiled native proof and compatibility negatives From 51de1b312fd3dcd59684030ba7592de2aeaa7c74 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:10:50 +0300 Subject: [PATCH 36/58] fix(module): linearize terminal publication with init abandonment Initialization abandonment now checks and publishes terminal state under the physical state lock, so a concurrent commit cannot be overwritten and the host lock order is preserved. The initiating waiter's drop path and the init deadline path both route through the new helper, and the terminal flag publication is split out from admission closing. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_tests.rs | 2 +- crates/tinybus/src/module/transport.rs | 26 +++++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 9e225f3f..129b38fc 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -3130,7 +3130,7 @@ async fn actual_native_fault_during_ordinary_stop_keeps_fault_state_and_single_j let (release, release_rx) = std::sync::mpsc::channel(); *barrier.finish_gate.lock().unwrap() = Some(release_rx); let stopping = tokio::spawn({ - let host = host.clone(); async move { host.stop(info.name, Duration::from_secs(2)).await } + let host = host.clone(); async move { host.inner.stop_inner(&info.name, Duration::from_secs(2)).await } }); barrier.joined.notified().await; assert!(transport.stop_task_spawned()); diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 05fe347e..9d1ef59b 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -159,10 +159,7 @@ struct InitiatingWaiter<'a> { transport: &'a ModuleTransport, armed: bool } impl Drop for InitiatingWaiter<'_> { fn drop(&mut self) { if self.armed { - let published = self.transport.context.physical.lock().expect("physical state lock").result.is_some(); - if !published && !self.transport.admission_closed() { - self.transport.request_terminal(true); - } + self.transport.abandon_initialization(); } } } @@ -415,7 +412,7 @@ impl ModuleTransport { let result = match tokio::time::timeout(MODULE_INIT_DEADLINE, wait).await { Ok(result) => { abandonment.armed = false; result } Err(_) => { - if initiating && !self.admission_closed() { self.request_terminal(true); } + if initiating { self.abandon_initialization(); } abandonment.armed = false; return Err(Error::transport("module initialization exceeded its deadline")); } @@ -488,7 +485,7 @@ impl ModuleTransport { } } - pub(crate) fn commit_terminal(&self, fault: bool) { + fn publish_terminal_flags(&self, fault: bool) { use crate::module::abi::{TB_NATIVE_TERMINAL_FAULT, TB_NATIVE_TERMINAL_STOP_INTENT}; if fault { self.context.faulted.store(true, Ordering::Release); } let flags = &self.context.native_terminal_state; @@ -500,6 +497,23 @@ impl ModuleTransport { Ok(_) => break, Err(current) => old = current, } } + } + + fn abandon_initialization(&self) { + { + let mut state = self.context.physical.lock().expect("physical state lock"); + // Linearize publication versus the initiating waiter's abandonment. + // Atomic-only cause publication does not invert the host lock order. + if state.result.is_some() || self.context.native_terminal_state.load(Ordering::Acquire) + != crate::module::abi::TB_NATIVE_TERMINAL_LIVE { return; } + self.publish_terminal_flags(true); + if state.cleanup == CleanupState::Idle { state.cleanup = CleanupState::Requested; } + } + self.request_terminal(true); + } + + pub(crate) fn commit_terminal(&self, fault: bool) { + self.publish_terminal_flags(fault); self.close_admission(); } From 4b69ef13d4b9988635a649fb3fd60b5a09e2b5f4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:13:05 +0300 Subject: [PATCH 37/58] test(transport): cover cancellation and eager rejection paths Rework the scheduled-initializer test to abort a waiter instead of scheduling directly, asserting that ordinary cancellation cannot invent a Fault state, and add an observer-abandonment check to the late-init join test. Replace the inline rejection closures with named initializer functions and add coverage for eager rejected outputs, which must keep cleanup uncertain and never invoke returned callbacks, plus an ignored Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 107 ++++++++++++++++--- 1 file changed, 94 insertions(+), 13 deletions(-) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 41b717e5..e64bc734 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1383,12 +1383,16 @@ async fn scheduled_initializer_is_adopted_by_original_ordinary_stop() { worker_started.send(()).unwrap(); worker_release_rx.recv().unwrap(); })); - assert!(transport.schedule_initializer().unwrap()); + let waiter = tokio::spawn({ + let transport = transport.clone(); async move { transport.ensure_initialized().await } + }); worker_started_rx.await.unwrap(); let mut stopping = Box::pin(transport.clone().stop(Duration::from_secs(2))); tokio::select! { result = &mut stopping => panic!("stop completed before owner: {result:?}"), _ = async { while !transport.stop_task_spawned() { tokio::task::yield_now().await; } } => {} } - assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 2); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 2, "ordinary cancellation cannot invent Fault"); worker_release.send(()).unwrap(); assert_eq!(stopping.await.unwrap(), TB_CLOSED); assert_eq!(entries.load(Ordering::Acquire), 0); @@ -1471,6 +1475,13 @@ async fn actual_native_late_init_is_joined_after_initiator_abandonment() { let transport = transport.clone(); async move { transport.ensure_initialized().await } }); barrier.entered.notified().await; + let mut observation = Box::pin(transport.ensure_initialized()); + std::future::poll_fn(|cx| { + assert!(std::future::Future::poll(observation.as_mut(), cx).is_pending()); + std::task::Poll::Ready(()) + }).await; + drop(observation); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 1, "observer abandonment cannot cancel initiating owner"); waiter.abort(); assert!(waiter.await.unwrap_err().is_cancelled()); let view = barrier.state.lock().unwrap().0.unwrap(); @@ -1563,17 +1574,13 @@ async fn rejected_module_outputs_never_invoke_untrusted_callbacks() { SHUTDOWN_CALLS.store(0, Ordering::Release); for rejection in 0..4 { let (transport, host) = ModuleTransport::new(format!("rejected-{rejection}"), vec![]); - transport.defer_initializer(host, move |host| { - let mut module = TbModuleVtable::default(); - unsafe { initialize_ok(&host, &mut module); } - match rejection { - 0 => module.size = TB_MODULE_VTABLE_BASE_SIZE - 1, - 1 => module.module_ctx = std::ptr::null_mut(), - 2 => module.reinitialize = None, - _ => return Err("non-OK output must remain inaccessible".into()), - } - Ok(module) - }); + let init: crate::module::abi::TbModuleInit = match rejection { + 0 => initialize_short_output, + 1 => initialize_null_context_output, + 2 => initialize_missing_declared_reinit, + _ => initialize_non_ok_misleading_output, + }; + transport.defer_initialize(init, host); assert!(transport.ensure_initialized().await.is_err()); let physical = transport.context.physical.lock().unwrap(); assert_eq!(physical.init, InitState::PublishedRejected); @@ -1734,3 +1741,77 @@ async fn actual_native_late_eager_publication_cannot_refund_failed_cleanup_spawn // leak-bound or secret-erasure proof. std::mem::forget(barrier); } + + +unsafe extern "C" fn initialize_short_output(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { + unsafe { initialize_ok(host, out); (*out).size = TB_MODULE_VTABLE_BASE_SIZE - 1; } + TB_OK +} +unsafe extern "C" fn initialize_null_context_output(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { + unsafe { initialize_ok(host, out); (*out).module_ctx = std::ptr::null_mut(); } + TB_OK +} +unsafe extern "C" fn initialize_missing_declared_reinit(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { + unsafe { initialize_ok(host, out); (*out).reinitialize = None; } + TB_OK +} +unsafe extern "C" fn initialize_non_ok_misleading_output(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { + unsafe { initialize_ok(host, out); } + TB_PANICKED +} + +#[tokio::test] +async fn eager_rejected_outputs_keep_uncertainty_and_never_call_returned_callbacks() { + let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + SHUTDOWN_CALLS.store(0, Ordering::Release); + for init in [initialize_short_output as crate::module::abi::TbModuleInit, + initialize_null_context_output, initialize_missing_declared_reinit, + initialize_non_ok_misleading_output] { + let (transport, host) = ModuleTransport::new("eager-invalid".into(), b"borrowed".to_vec()); + assert!(transport.eager_initialize(init, host).is_err()); + let state = transport.context.physical.lock().unwrap(); + assert_eq!(state.init, InitState::PublishedRejected); + assert_eq!(state.cleanup, CleanupState::Uncertain); + assert!(state.job.is_some()); + assert!(transport.module.lock().unwrap().is_none()); + drop(state); + assert!(transport.context.config.lock().unwrap().is_empty()); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + } + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 0); +} + +#[test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +fn actual_native_initializer_owner_outlives_runtime_and_host_handle_drop() { + let (transport, host, init, barrier) = native_fixture(); + transport.defer_initialize(init, host); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let transport = transport.clone(); + tokio::spawn(async move { let _ = transport.ensure_initialized().await; }); + barrier.entered.notified().await; + }); + let weak = Arc::downgrade(&transport); + drop(runtime); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + drop(transport); + let retained = weak.upgrade().expect("real initializer owner survives runtime and host drop"); + assert!(!barrier.state.lock().unwrap().2); + { + let mut state = barrier.state.lock().unwrap(); state.1 = true; barrier.changed.notify_all(); + } + // The runtime is gone. Wait on the retained physical predicate/Condvar, + // never on a Tokio task, deadline or notification as completion proof. + let mut state = retained.context.physical.lock().unwrap(); + while state.cleanup != CleanupState::Complete { + let (next, timeout) = retained.context.physical_changed.wait_timeout(state, Duration::from_secs(2)).unwrap(); + state = next; + assert!(!timeout.timed_out(), "actual retained native cleanup must finish"); + } + drop(state); + assert!(barrier.state.lock().unwrap().2); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); + drop(retained); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); +} From 15d4e082e78dd32520b251efe2b7d5befc75cb1b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:15:16 +0300 Subject: [PATCH 38/58] chore: files changed crates/tinybus/examples/module_controlled_init.rs,crates/tinybus/src/module/hos Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../examples/module_controlled_init.rs | 71 ++- crates/tinybus/src/module/host.rs | 25 +- crates/tinybus/src/module/host_callbacks.rs | 21 +- crates/tinybus/src/module/host_tests.rs | 117 ++++- crates/tinybus/src/module/transport.rs | 475 ++++++++++++++---- crates/tinybus/src/module/transport_tests.rs | 459 +++++++++++++---- 6 files changed, 907 insertions(+), 261 deletions(-) diff --git a/crates/tinybus/examples/module_controlled_init.rs b/crates/tinybus/examples/module_controlled_init.rs index 507a47da..7c0a4498 100644 --- a/crates/tinybus/examples/module_controlled_init.rs +++ b/crates/tinybus/examples/module_controlled_init.rs @@ -26,16 +26,26 @@ pub static TINYBUS_MODULE_ABI_V1: TbAbiDescriptor = #[unsafe(no_mangle)] pub extern "C" fn tinybus_module_manifest_v1() -> TbSlice { tinybus_module::manifest_slice(tinybus_module::ManifestDeclaration { - name: "controlled-init", version: env!("CARGO_PKG_VERSION"), - provides: &["ai.tinyhumans.ControlledInit"], methods: &[], signals: &[], - requires: &[], optional: &[], lazy: true, worker_threads: 1, + name: "controlled-init", + version: env!("CARGO_PKG_VERSION"), + provides: &["ai.tinyhumans.ControlledInit"], + methods: &[], + signals: &[], + requires: &[], + optional: &[], + lazy: true, + worker_threads: 1, }) } -unsafe extern "C" fn deliver(_: *mut c_void, _: *const u8, _: usize) -> i32 { TB_OK } +unsafe extern "C" fn deliver(_: *mut c_void, _: *const u8, _: usize) -> i32 { + TB_OK +} unsafe extern "C" fn reinitialize(ctx: *mut c_void, _: *const u8, _: usize) -> i32 { let instance = unsafe { &*ctx.cast::() }; - unsafe { (instance.controls.entered)(instance.controls.ctx, &instance.host); } + unsafe { + (instance.controls.entered)(instance.controls.ctx, &instance.host); + } TB_OK } unsafe extern "C" fn shutdown(ctx: *mut c_void, _: u64) -> i32 { @@ -43,33 +53,58 @@ unsafe extern "C" fn shutdown(ctx: *mut c_void, _: u64) -> i32 { *instance.stop.0.lock().unwrap() = true; instance.stop.1.notify_all(); instance.worker.take().unwrap().join().unwrap(); - unsafe { (instance.controls.joined)(instance.controls.ctx); } + unsafe { + (instance.controls.joined)(instance.controls.ctx); + } TB_OK } /// # Safety /// Host pointers are valid for the ABI call; config contains test Controls. #[unsafe(no_mangle)] -pub unsafe extern "C" fn tinybus_module_init_v1(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { +pub unsafe extern "C" fn tinybus_module_init_v1( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { let host = unsafe { &*host }; - if host.config.len != size_of::() { return TB_BAD_ARGUMENT; } + if host.config.len != size_of::() { + return TB_BAD_ARGUMENT; + } let controls = unsafe { host.config.ptr.cast::().read_unaligned() }; - let copied_config = unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }.to_vec(); + let copied_config = + unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }.to_vec(); // The host-side barrier blocks actual opaque C entry, retaining its config // borrow through fault publication and initiating waiter abandonment. - unsafe { (controls.entered)(controls.ctx, host); } - assert_eq!(unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }, copied_config); + unsafe { + (controls.entered)(controls.ctx, host); + } + assert_eq!( + unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }, + copied_config + ); let stop = Arc::new((Mutex::new(false), Condvar::new())); let worker_stop = stop.clone(); let worker = std::thread::spawn(move || { let mut stopped = worker_stop.0.lock().unwrap(); - while !*stopped { stopped = worker_stop.1.wait(stopped).unwrap(); } + while !*stopped { + stopped = worker_stop.1.wait(stopped).unwrap(); + } + }); + let instance = Box::new(Instance { + controls, + host: *host, + stop, + worker: Some(worker), }); - let instance = Box::new(Instance { controls, host: *host, stop, worker: Some(worker) }); - unsafe { *out = TbModuleVtable { - size: size_of::() as u32, _reserved: 0, - module_ctx: Box::into_raw(instance).cast(), deliver, shutdown, - reinitialize: Some(reinitialize), - }; } + unsafe { + *out = TbModuleVtable { + size: size_of::() as u32, + _reserved: 0, + module_ctx: Box::into_raw(instance).cast(), + deliver, + shutdown, + reinitialize: Some(reinitialize), + }; + } TB_OK } diff --git a/crates/tinybus/src/module/host.rs b/crates/tinybus/src/module/host.rs index eb096492..b86add27 100644 --- a/crates/tinybus/src/module/host.rs +++ b/crates/tinybus/src/module/host.rs @@ -1021,7 +1021,11 @@ impl ModuleHost { } else { admitted.state = ModuleState::Initializing; if let Err(failure) = transport.eager_initialize(artifact.init, host_vtable) { - let reason = if failure.to_string().contains("vtable") { "module returned an invalid vtable" } else { "module initialization failed" }; + let reason = if failure.to_string().contains("vtable") { + "module returned an invalid vtable" + } else { + "module initialization failed" + }; let error = Error::module_refused(path, reason); self.record_manifest_rejection(&error, manifest, RefusalClass::Failed); return Err(error); @@ -1515,9 +1519,18 @@ impl ModuleControl for ModuleHostInner { Err(StopError::Started(error)) => return Err(error), } module.info.state = if module.transport.is_faulted() { - if module.transport.init_failed() { ModuleState::Failed { reason: "module initialization failed".into() } } - else { ModuleState::Faulted { reason: "module reported an unrecoverable fault".into() } } - } else { ModuleState::Stopped }; + if module.transport.init_failed() { + ModuleState::Failed { + reason: "module initialization failed".into(), + } + } else { + ModuleState::Faulted { + reason: "module reported an unrecoverable fault".into(), + } + } + } else { + ModuleState::Stopped + }; Ok(module.info.clone()) } @@ -1660,7 +1673,9 @@ impl ModuleControl for ModuleHostInner { let module = loaded .iter_mut() .find(|module| &module.unique_name == unique_name)?; - if !module.transport.is_faulted() && let Some(old) = module.transition_from.take() { + if !module.transport.is_faulted() + && let Some(old) = module.transition_from.take() + { module.info.state = ModuleState::Stopped; return Some((module.info.name.clone(), old, ModuleState::Stopped)); } diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 49c6c29b..932829bf 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -639,7 +639,9 @@ impl Drop for LoadRollback<'_> { if self.committed { return; } - if let Some(transport) = &self.transport { transport.commit_terminal(true); } + if let Some(transport) = &self.transport { + transport.commit_terminal(true); + } self.binding.revoke(); self.binding.transport.close_outbound(); self.binding @@ -685,13 +687,22 @@ impl Drop for LoadRollback<'_> { impl Drop for ModuleHostInner { fn drop(&mut self) { let modules = self.loaded.get_mut().expect("module list lock"); - let transports: Vec<_> = modules.iter().map(|module| module.transport.clone()).collect(); - for transport in &transports { transport.commit_terminal(false); } + let transports: Vec<_> = modules + .iter() + .map(|module| module.transport.clone()) + .collect(); + for transport in &transports { + transport.commit_terminal(false); + } { let mut router = self.broker.router.lock().expect("router lock"); - for module in modules.iter() { router.revoke_callbacks_for_peer(&module.unique_name); } + for module in modules.iter() { + router.revoke_callbacks_for_peer(&module.unique_name); + } + } + for transport in transports { + transport.request_terminal(false); } - for transport in transports { transport.request_terminal(false); } } } diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 129b38fc..16c79265 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -3058,8 +3058,11 @@ async fn ordinary_stop_never_fabricates_faulted_before_or_after_callback_complet } async fn controlled_native_transport() -> ( - Arc, crate::module::abi::TbHostVtable, - Arc, LoadedArtifact, PathBuf, + Arc, + crate::module::abi::TbHostVtable, + Arc, + LoadedArtifact, + PathBuf, ) { let (transport, view, init, barrier) = super::super::transport::tests::native_fixture(); barrier.state.lock().unwrap().1 = true; @@ -3071,12 +3074,26 @@ async fn controlled_native_transport() -> ( (transport, view, barrier, artifact, path) } -fn attach_controlled_native(host: &ModuleHost, transport: Arc, artifact: LoadedArtifact, path: &Path) -> Result { +fn attach_controlled_native( + host: &ModuleHost, + transport: Arc, + artifact: LoadedArtifact, + path: &Path, +) -> Result { let mut admitted = host.validate(path, &artifact.descriptor, &artifact.manifest)?; admitted.state = ModuleState::Initializing; - host.attach_transport(path, admitted, artifact.manifest, transport, Activation { - lazy_init: false, lazy_load: false, descriptor_info: None, pinned_sha256: None, - }) + host.attach_transport( + path, + admitted, + artifact.manifest, + transport, + Activation { + lazy_init: false, + lazy_load: false, + descriptor_info: None, + pinned_sha256: None, + }, + ) } #[tokio::test] @@ -3087,16 +3104,32 @@ async fn actual_native_host_drop_keeps_terminal_tombstone_and_other_instance_liv let host = ModuleHost::new(Broker::new()); attach_controlled_native(&host, transport.clone(), artifact, &path).unwrap(); drop(host); - assert_ne!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 1); - assert_eq!(unsafe { other_view.native_terminal_state.unwrap()(other_view.host_ctx) }, 1); - tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + assert_ne!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 1 + ); + assert_eq!( + unsafe { other_view.native_terminal_state.unwrap()(other_view.host_ctx) }, + 1 + ); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); let stopped = transport.clone().stop(Duration::from_secs(2)).await; assert!(matches!(stopped, Err(StopError::NotStarted(_)))); - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 10); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 10 + ); drop(transport); - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 10); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 10 + ); other.request_terminal(false); - tokio::time::timeout(Duration::from_secs(2), other_barrier.joined.notified()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), other_barrier.joined.notified()) + .await + .unwrap(); assert!(barrier.state.lock().unwrap().2); assert!(other_barrier.state.lock().unwrap().2); } @@ -3109,16 +3142,32 @@ async fn actual_native_reservation_rollback_retains_one_cleanup_and_fault_tombst let host = ModuleHost::new(Broker::new()); attach_controlled_native(&host, first, artifact, &path).unwrap(); assert!(attach_controlled_native(&host, rejected.clone(), other_artifact, &path).is_err()); - assert_ne!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 1); - assert_eq!(unsafe { first_view.native_terminal_state.unwrap()(first_view.host_ctx) }, 1); - tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + assert_ne!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 1 + ); + assert_eq!( + unsafe { first_view.native_terminal_state.unwrap()(first_view.host_ctx) }, + 1 + ); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); let _observed = rejected.clone().stop(Duration::from_secs(2)).await; - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 14); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 14 + ); assert!(rejected.ensure_initialized().await.is_err()); drop(rejected); - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 14); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 14 + ); drop(host); - tokio::time::timeout(Duration::from_secs(2), first_barrier.joined.notified()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), first_barrier.joined.notified()) + .await + .unwrap(); } #[tokio::test] @@ -3130,16 +3179,36 @@ async fn actual_native_fault_during_ordinary_stop_keeps_fault_state_and_single_j let (release, release_rx) = std::sync::mpsc::channel(); *barrier.finish_gate.lock().unwrap() = Some(release_rx); let stopping = tokio::spawn({ - let host = host.clone(); async move { host.inner.stop_inner(&info.name, Duration::from_secs(2)).await } + let host = host.clone(); + async move { + host.inner + .stop_inner(&info.name, Duration::from_secs(2)) + .await + } }); barrier.joined.notified().await; assert!(transport.stop_task_spawned()); - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 2, "joined worker is not yet successful opaque return"); - unsafe { (view.fault)(view.host_ctx, std::ptr::null(), 0); } - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 6); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2, + "joined worker is not yet successful opaque return" + ); + unsafe { + (view.fault)(view.host_ctx, std::ptr::null(), 0); + } + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 6 + ); release.send(()).unwrap(); let result = stopping.await.unwrap().unwrap(); - assert!(matches!(result.state, ModuleState::Faulted { .. }), "late ordinary completion cannot replace actual fault"); - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 14); + assert!( + matches!(result.state, ModuleState::Faulted { .. }), + "late ordinary completion cannot replace actual fault" + ); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 14 + ); assert!(barrier.state.lock().unwrap().2); } diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 9d1ef59b..67e9429e 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -68,7 +68,7 @@ pub(crate) struct ModuleTransport { initialization_gate: StdMutex>, initialization_gate_changed: Notify, #[cfg(test)] - reject_worker_spawn: AtomicBool, + pub(crate) reject_worker_spawn: AtomicBool, #[cfg(test)] before_worker_lock: StdMutex>>, #[cfg(test)] @@ -135,9 +135,24 @@ struct DeferredInitializer { } #[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum InitState { Deferred, Scheduled, Entered, PublishedValid, PublishedRejected, CanceledBeforeEntry, ExternalValid } +enum InitState { + Deferred, + Scheduled, + Entered, + PublishedValid, + PublishedRejected, + CanceledBeforeEntry, + ExternalValid, +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum CleanupState { Idle, Requested, Executing, Complete, Uncertain, NoCallbackRequired } +enum CleanupState { + Idle, + Requested, + Executing, + Complete, + Uncertain, + NoCallbackRequired, +} struct PhysicalState { init: InitState, @@ -153,9 +168,14 @@ struct PhysicalJob { guard: StdMutex>>, deadline: Duration, fault_on_spawn_failure: bool, + submitted: AtomicBool, + started: AtomicBool, } -struct InitiatingWaiter<'a> { transport: &'a ModuleTransport, armed: bool } +struct InitiatingWaiter<'a> { + transport: &'a ModuleTransport, + armed: bool, +} impl Drop for InitiatingWaiter<'_> { fn drop(&mut self) { if self.armed { @@ -207,8 +227,11 @@ impl ModuleTransport { owner: StdMutex::new(Weak::new()), native_terminal_state: AtomicU32::new(crate::module::abi::TB_NATIVE_TERMINAL_LIVE), physical: StdMutex::new(PhysicalState { - init: InitState::Deferred, cleanup: CleanupState::Idle, - job: None, result: None, code: None, + init: InitState::Deferred, + cleanup: CleanupState::Idle, + job: None, + result: None, + code: None, }), physical_changed: Condvar::new(), physical_notify: Notify::new(), @@ -363,7 +386,11 @@ impl ModuleTransport { /// Publish physical completion only after the opaque shutdown returns. fn finish_stop(&self) { self.context.stopped.store(true, Ordering::Release); - self.context.inbound.lock().expect("host inbound lock").take(); + self.context + .inbound + .lock() + .expect("host inbound lock") + .take(); self.context.init_notify.notify_waiters(); self.context.ready_notify.notify_waiters(); self.context.wake.notify_waiters(); @@ -400,21 +427,39 @@ impl ModuleTransport { } let initiating = self.schedule_initializer()?; - let mut abandonment = InitiatingWaiter { transport: self, armed: initiating }; + let mut abandonment = InitiatingWaiter { + transport: self, + armed: initiating, + }; let wait = async { loop { let changed = self.context.physical_notify.notified(); - let result = self.context.physical.lock().expect("physical state lock").result.clone(); - if let Some(result) = result { return result; } + let result = self + .context + .physical + .lock() + .expect("physical state lock") + .result + .clone(); + if let Some(result) = result { + return result; + } changed.await; } }; let result = match tokio::time::timeout(MODULE_INIT_DEADLINE, wait).await { - Ok(result) => { abandonment.armed = false; result } + Ok(result) => { + abandonment.armed = false; + result + } Err(_) => { - if initiating { self.abandon_initialization(); } + if initiating { + self.abandon_initialization(); + } abandonment.armed = false; - return Err(Error::transport("module initialization exceeded its deadline")); + return Err(Error::transport( + "module initialization exceeded its deadline", + )); } }; match result { @@ -433,21 +478,34 @@ impl ModuleTransport { // Scheduling is synchronous: the externally anchored owner precedes submission. fn schedule_initializer(&self) -> Result { - let gate = self.initialization_gate.lock().expect("initialization gate lock"); + let gate = self + .initialization_gate + .lock() + .expect("initialization gate lock"); if *gate != Some(true) || self.is_faulted() || self.admission_closed() { return Err(Error::ConnectionClosed); } let mut state = self.context.physical.lock().expect("physical state lock"); - if state.result.is_some() || state.init != InitState::Deferred { return Ok(false); } - let Some(initializer) = self.initializer.lock().expect("module initializer lock").take() else { + if state.result.is_some() || state.init != InitState::Deferred { + return Ok(false); + } + let Some(initializer) = self + .initializer + .lock() + .expect("module initializer lock") + .take() + else { state.result = Some(Err(InitFailure::Failed("module has no initializer".into()))); return Ok(false); }; let job = Arc::new(PhysicalJob { transport: self.self_ref.upgrade().ok_or(Error::ConnectionClosed)?, - initializer: StdMutex::new(Some(initializer)), guard: StdMutex::new(None), + initializer: StdMutex::new(Some(initializer)), + guard: StdMutex::new(None), deadline: MODULE_INIT_DEADLINE, fault_on_spawn_failure: true, + submitted: AtomicBool::new(false), + started: AtomicBool::new(false), }); state.init = InitState::Scheduled; state.job = Some(job.clone()); @@ -460,6 +518,11 @@ impl ModuleTransport { } fn submit_job(job: Arc) { + // Submission is spent even if the builder drops its closure. The + // external anchor remains; no caller can silently resubmit this job. + if job.submitted.swap(true, Ordering::AcqRel) { + return; + } let submitted = job.clone(); #[cfg(test)] let rejected = job.transport.reject_worker_spawn.load(Ordering::Acquire); @@ -472,14 +535,22 @@ impl ModuleTransport { drop(worker); Err(std::io::Error::other("controlled worker rejection")) } else { - std::thread::Builder::new().name("tinybus-physical".into()).spawn(worker) + std::thread::Builder::new() + .name("tinybus-physical".into()) + .spawn(worker) }; if spawn.is_err() { let transport = &job.transport; transport.commit_terminal(job.fault_on_spawn_failure); - let mut state = transport.context.physical.lock().expect("physical state lock"); + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); state.cleanup = CleanupState::Uncertain; - state.result.get_or_insert(Err(InitFailure::Failed("module physical worker could not start".into()))); + state.result.get_or_insert(Err(InitFailure::Failed( + "module physical worker could not start".into(), + ))); drop(state); transport.notify_physical(); } @@ -487,14 +558,18 @@ impl ModuleTransport { fn publish_terminal_flags(&self, fault: bool) { use crate::module::abi::{TB_NATIVE_TERMINAL_FAULT, TB_NATIVE_TERMINAL_STOP_INTENT}; - if fault { self.context.faulted.store(true, Ordering::Release); } + if fault { + self.context.faulted.store(true, Ordering::Release); + } let flags = &self.context.native_terminal_state; let mut old = flags.load(Ordering::Acquire); loop { - let new = (old & !crate::module::abi::TB_NATIVE_TERMINAL_LIVE) | - TB_NATIVE_TERMINAL_STOP_INTENT | if fault { TB_NATIVE_TERMINAL_FAULT } else { 0 }; + let new = (old & !crate::module::abi::TB_NATIVE_TERMINAL_LIVE) + | TB_NATIVE_TERMINAL_STOP_INTENT + | if fault { TB_NATIVE_TERMINAL_FAULT } else { 0 }; match flags.compare_exchange_weak(old, new, Ordering::AcqRel, Ordering::Acquire) { - Ok(_) => break, Err(current) => old = current, + Ok(_) => break, + Err(current) => old = current, } } } @@ -504,10 +579,16 @@ impl ModuleTransport { let mut state = self.context.physical.lock().expect("physical state lock"); // Linearize publication versus the initiating waiter's abandonment. // Atomic-only cause publication does not invert the host lock order. - if state.result.is_some() || self.context.native_terminal_state.load(Ordering::Acquire) - != crate::module::abi::TB_NATIVE_TERMINAL_LIVE { return; } + if state.result.is_some() + || self.context.native_terminal_state.load(Ordering::Acquire) + != crate::module::abi::TB_NATIVE_TERMINAL_LIVE + { + return; + } self.publish_terminal_flags(true); - if state.cleanup == CleanupState::Idle { state.cleanup = CleanupState::Requested; } + if state.cleanup == CleanupState::Idle { + state.cleanup = CleanupState::Requested; + } } self.request_terminal(true); } @@ -520,57 +601,111 @@ impl ModuleTransport { pub(crate) fn request_terminal(&self, fault: bool) { self.commit_terminal(fault); let mut state = self.context.physical.lock().expect("physical state lock"); - if state.cleanup == CleanupState::Idle { state.cleanup = CleanupState::Requested; } - if state.job.is_some() || state.cleanup != CleanupState::Requested { return; } + if state.cleanup == CleanupState::Idle { + state.cleanup = CleanupState::Requested; + } + if state.job.is_some() || state.cleanup != CleanupState::Requested { + return; + } if state.init == InitState::Deferred { state.init = InitState::CanceledBeforeEntry; state.cleanup = CleanupState::NoCallbackRequired; state.result = Some(Err(InitFailure::Closed)); drop(state); - self.initializer.lock().expect("module initializer lock").take(); + self.initializer + .lock() + .expect("module initializer lock") + .take(); self.clear_config(); self.finish_stop(); self.notify_physical(); return; } - let Some(transport) = self.self_ref.upgrade() else { return; }; - let job = Arc::new(PhysicalJob { transport, initializer: StdMutex::new(None), - guard: StdMutex::new(None), deadline: MODULE_INIT_DEADLINE, fault_on_spawn_failure: fault }); + let Some(transport) = self.self_ref.upgrade() else { + return; + }; + let job = Arc::new(PhysicalJob { + transport, + initializer: StdMutex::new(None), + guard: StdMutex::new(None), + deadline: MODULE_INIT_DEADLINE, + fault_on_spawn_failure: fault, + submitted: AtomicBool::new(false), + started: AtomicBool::new(false), + }); state.job = Some(job.clone()); drop(state); Self::submit_job(job); } - pub(crate) fn eager_initialize(&self, init: crate::module::abi::TbModuleInit, host: TbHostVtable) -> Result<()> { + pub(crate) fn eager_initialize( + &self, + init: crate::module::abi::TbModuleInit, + host: TbHostVtable, + ) -> Result<()> { let guard = self.lifecycle.clone().try_lock_owned().map_err(|_| { self.commit_terminal(true); - self.context.physical.lock().expect("physical state lock").cleanup = CleanupState::Uncertain; + self.context + .physical + .lock() + .expect("physical state lock") + .cleanup = CleanupState::Uncertain; Error::transport("module initialization lifecycle unavailable") })?; - self.context.physical.lock().expect("physical state lock").init = InitState::Entered; + self.context + .physical + .lock() + .expect("physical state lock") + .init = InitState::Entered; self.context.init_started.store(true, Ordering::Release); let mut module = TbModuleVtable::default(); let code = unsafe { init(&host, &mut module) }; - let result = if code == TB_OK { self.initialize(module) } else { Err(Error::transport("module initialization failed")) }; + let result = if code == TB_OK { + self.initialize(module) + } else { + Err(Error::transport("module initialization failed")) + }; self.clear_config(); { let mut state = self.context.physical.lock().expect("physical state lock"); - state.init = if result.is_ok() { InitState::ExternalValid } else { InitState::PublishedRejected }; - state.result = Some(if result.is_ok() { Ok(()) } else { Err(InitFailure::Failed("module returned an invalid vtable".into())) }); + state.init = if result.is_ok() { + InitState::ExternalValid + } else { + InitState::PublishedRejected + }; + state.result.get_or_insert(if result.is_ok() { + Ok(()) + } else { + Err(InitFailure::Failed( + "module returned an invalid vtable".into(), + )) + }); if result.is_err() { state.cleanup = CleanupState::Uncertain; if state.job.is_none() { - state.job = self.self_ref.upgrade().map(|transport| Arc::new(PhysicalJob { - transport, initializer: StdMutex::new(None), guard: StdMutex::new(None), - deadline: MODULE_INIT_DEADLINE, fault_on_spawn_failure: true, - })); + state.job = self.self_ref.upgrade().map(|transport| { + Arc::new(PhysicalJob { + transport, + initializer: StdMutex::new(None), + guard: StdMutex::new(None), + deadline: MODULE_INIT_DEADLINE, + fault_on_spawn_failure: true, + submitted: AtomicBool::new(false), + started: AtomicBool::new(false), + }) + }); } } } self.notify_physical(); drop(guard); - if result.is_err() { self.request_terminal(true); return result; } - if self.is_faulted() || self.admission_closed() { return Err(Error::ConnectionClosed); } + if result.is_err() { + self.request_terminal(true); + return result; + } + if self.is_faulted() || self.admission_closed() { + return Err(Error::ConnectionClosed); + } result } @@ -851,7 +986,10 @@ impl ModuleTransport { self.callbacks.wait_drained(); let code = self.shutdown_sync(deadline.saturating_sub(started.elapsed())); if code == TB_OK { - self.context.native_terminal_state.fetch_or(crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, Ordering::AcqRel); + self.context.native_terminal_state.fetch_or( + crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + Ordering::AcqRel, + ); } self.finish_stop(); self.context @@ -887,62 +1025,103 @@ impl ModuleTransport { let remaining = deadline.saturating_sub(started.elapsed()); self.commit_terminal(false); let (_existing, job, mut acquired, ordinary_handoff) = { - let mut state = self.context.physical.lock().expect("physical state lock"); - if state.cleanup == CleanupState::Idle { state.cleanup = CleanupState::Requested; } - let existing = state.job.is_some(); - let mut acquired = Some(lifecycle); - let job = if existing || state.cleanup != CleanupState::Requested { None } else { - let job = Arc::new(PhysicalJob { transport: self.clone(), - initializer: StdMutex::new(None), guard: StdMutex::new(acquired.take()), - deadline: remaining, fault_on_spawn_failure: false }); - state.job = Some(job.clone()); - Some(job) - }; - let ordinary_handoff = !self.is_faulted() && state.cleanup == CleanupState::Requested - && (job.is_some() || (existing && state.init == InitState::Scheduled)); - if ordinary_handoff { self.stop_task_spawned.store(true, Ordering::Release); } - (existing, job, acquired, ordinary_handoff) + let mut state = self.context.physical.lock().expect("physical state lock"); + if state.cleanup == CleanupState::Idle { + state.cleanup = CleanupState::Requested; + } + let existing = state.job.is_some(); + let mut acquired = Some(lifecycle); + let job = if existing || state.cleanup != CleanupState::Requested { + None + } else { + let job = Arc::new(PhysicalJob { + transport: self.clone(), + initializer: StdMutex::new(None), + guard: StdMutex::new(acquired.take()), + deadline: remaining, + fault_on_spawn_failure: false, + submitted: AtomicBool::new(false), + started: AtomicBool::new(false), + }); + state.job = Some(job.clone()); + Some(job) + }; + let ordinary_handoff = !self.is_faulted() + && state.cleanup == CleanupState::Requested + && (job.is_some() || (existing && state.init == InitState::Scheduled)); + if ordinary_handoff { + self.stop_task_spawned.store(true, Ordering::Release); + } + (existing, job, acquired, ordinary_handoff) }; // A Scheduled initializer must acquire the guard itself to prove no C entry. drop(acquired.take()); - if let Some(job) = job { Self::submit_job(job); } - let classify = |error| if ordinary_handoff { StopError::Started(error) } else { StopError::NotStarted(error) }; + if let Some(job) = job { + Self::submit_job(job); + } + let classify = |error| { + if ordinary_handoff { + StopError::Started(error) + } else { + StopError::NotStarted(error) + } + }; let wait = async { loop { let changed = self.context.physical_notify.notified(); let outcome = { - let state = self.context.physical.lock().expect("physical state lock"); - match state.cleanup { - CleanupState::Complete | CleanupState::NoCallbackRequired => Some(Ok(state.code.unwrap_or(TB_CLOSED))), - CleanupState::Uncertain => Some(match state.code { - Some(code) => Ok(code), - None => Err(classify(Error::transport("module cleanup is uncertain"))), - }), - _ => None, - } + let state = self.context.physical.lock().expect("physical state lock"); + match state.cleanup { + CleanupState::Complete | CleanupState::NoCallbackRequired => { + Some(Ok(state.code.unwrap_or(TB_CLOSED))) + } + CleanupState::Uncertain => Some(match state.code { + Some(code) => Ok(code), + None => Err(classify(Error::transport("module cleanup is uncertain"))), + }), + _ => None, + } }; - if let Some(outcome) = outcome { return outcome; } + if let Some(outcome) = outcome { + return outcome; + } changed.await; } }; - tokio::time::timeout(remaining, wait).await.map_err(|_| classify(Error::failed("module shutdown exceeded its deadline")))? - + tokio::time::timeout(remaining, wait) + .await + .map_err(|_| classify(Error::failed("module shutdown exceeded its deadline")))? } } impl PhysicalJob { fn run(self: Arc) { + self.started.store(true, Ordering::Release); let transport = &self.transport; #[cfg(test)] { - let hook = transport.before_worker_lock.lock().expect("worker test hook lock").take(); - if let Some(hook) = hook { hook(); } + let hook = transport + .before_worker_lock + .lock() + .expect("worker test hook lock") + .take(); + if let Some(hook) = hook { + hook(); + } } - let initializer = self.initializer.lock().expect("physical initializer lock").take(); + let initializer = self + .initializer + .lock() + .expect("physical initializer lock") + .take(); if let Some(initializer) = initializer { let guard = transport.lifecycle.clone().blocking_lock_owned(); let entered = { - let mut state = transport.context.physical.lock().expect("physical state lock"); + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); if transport.admission_closed() || transport.is_faulted() { state.init = InitState::CanceledBeforeEntry; state.cleanup = CleanupState::NoCallbackRequired; @@ -950,7 +1129,10 @@ impl PhysicalJob { state.code = Some(TB_CLOSED); state.job = None; false - } else { state.init = InitState::Entered; true } + } else { + state.init = InitState::Entered; + true + } }; if !entered { drop(initializer); @@ -959,43 +1141,88 @@ impl PhysicalJob { transport.notify_physical(); return; } - let output = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| initializer.run())); + let output = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| initializer.run())); let result = match output { - Ok(Ok(module)) => transport.initialize(module.0).map_err(|_| InitFailure::Failed("module returned an invalid vtable".into())), + Ok(Ok(module)) => transport + .initialize(module.0) + .map_err(|_| InitFailure::Failed("module returned an invalid vtable".into())), Ok(Err(reason)) => Err(InitFailure::Failed(reason)), Err(_) => Err(InitFailure::Failed("module initialization panicked".into())), }; transport.clear_config(); #[cfg(test)] { - let hook = transport.before_init_publication.lock().expect("publication test hook lock").take(); - if let Some(hook) = hook { hook(); } + let hook = transport + .before_init_publication + .lock() + .expect("publication test hook lock") + .take(); + if let Some(hook) = hook { + hook(); + } + } + if result.is_err() { + transport.commit_terminal(true); } - if result.is_err() { transport.commit_terminal(true); } let cleanup = { - let mut state = transport.context.physical.lock().expect("physical state lock"); - state.init = if result.is_ok() { InitState::PublishedValid } else { InitState::PublishedRejected }; - state.result = Some(result.clone()); - if result.is_err() { state.cleanup = CleanupState::Uncertain; } - if state.cleanup == CleanupState::Idle { state.job = None; } + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); + state.init = if result.is_ok() { + InitState::PublishedValid + } else { + InitState::PublishedRejected + }; + state.result.get_or_insert(result.clone()); + if result.is_err() { + state.cleanup = CleanupState::Uncertain; + } + if state.cleanup == CleanupState::Idle { + state.job = None; + } state.cleanup == CleanupState::Requested }; transport.notify_physical(); drop(guard); - if !cleanup { return; } + if !cleanup { + return; + } } else { // Eager entry publishes before releasing its exclusion. Never hold // lifecycle while waiting for an initializer which needs that lock. - let mut state = transport.context.physical.lock().expect("physical state lock"); - while matches!(state.init, InitState::Scheduled | InitState::Entered) && state.result.is_none() { - state = transport.context.physical_changed.wait(state).expect("physical state lock"); + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); + while matches!(state.init, InitState::Scheduled | InitState::Entered) + && state.result.is_none() + { + state = transport + .context + .physical_changed + .wait(state) + .expect("physical state lock"); } } - let guard = self.guard.lock().expect("physical guard lock").take() + let guard = self + .guard + .lock() + .expect("physical guard lock") + .take() .unwrap_or_else(|| transport.lifecycle.clone().blocking_lock_owned()); { - let mut state = transport.context.physical.lock().expect("physical state lock"); - if state.cleanup != CleanupState::Requested { return; } + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); + if state.cleanup != CleanupState::Requested { + return; + } if state.init == InitState::Deferred || state.init == InitState::CanceledBeforeEntry { state.init = InitState::CanceledBeforeEntry; state.cleanup = CleanupState::NoCallbackRequired; @@ -1003,13 +1230,22 @@ impl PhysicalJob { state.code = Some(TB_CLOSED); state.job = None; drop(state); - transport.initializer.lock().expect("module initializer lock").take(); + transport + .initializer + .lock() + .expect("module initializer lock") + .take(); transport.clear_config(); transport.finish_stop(); transport.notify_physical(); return; } - if transport.module.lock().expect("module vtable lock").is_none() { + if transport + .module + .lock() + .expect("module vtable lock") + .is_none() + { state.cleanup = CleanupState::Uncertain; drop(state); transport.notify_physical(); @@ -1017,11 +1253,23 @@ impl PhysicalJob { } state.cleanup = CleanupState::Executing; } - let code = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| transport.stop_sync(self.deadline))); - let mut state = transport.context.physical.lock().expect("physical state lock"); + let code = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + transport.stop_sync(self.deadline) + })); + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); state.code = code.as_ref().ok().copied(); - state.cleanup = if matches!(code, Ok(TB_OK)) { CleanupState::Complete } else { CleanupState::Uncertain }; - if state.cleanup == CleanupState::Complete { state.job = None; } + state.cleanup = if matches!(code, Ok(TB_OK)) { + CleanupState::Complete + } else { + CleanupState::Uncertain + }; + if state.cleanup == CleanupState::Complete { + state.job = None; + } drop(state); transport.notify_physical(); drop(guard); @@ -1030,8 +1278,10 @@ impl PhysicalJob { unsafe extern "C" fn host_native_terminal_state(ctx: *mut c_void) -> u32 { unsafe { ctx.cast::().as_ref() }.map_or( - crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT | crate::module::abi::TB_NATIVE_TERMINAL_FAULT, - |context| context.native_terminal_state.load(Ordering::Acquire)) + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT + | crate::module::abi::TB_NATIVE_TERMINAL_FAULT, + |context| context.native_terminal_state.load(Ordering::Acquire), + ) } #[derive(Debug)] @@ -1221,13 +1471,22 @@ unsafe extern "C" fn host_fault(ctx: *mut c_void, _: *const u8, _: usize) { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { if let Some(context) = unsafe { ctx.cast::().as_ref() } { context.faulted.store(true, Ordering::Release); - context.native_terminal_state.fetch_or(crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT | crate::module::abi::TB_NATIVE_TERMINAL_FAULT, Ordering::AcqRel); - context.native_terminal_state.fetch_and(!crate::module::abi::TB_NATIVE_TERMINAL_LIVE, Ordering::AcqRel); + context.native_terminal_state.fetch_or( + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT + | crate::module::abi::TB_NATIVE_TERMINAL_FAULT, + Ordering::AcqRel, + ); + context.native_terminal_state.fetch_and( + !crate::module::abi::TB_NATIVE_TERMINAL_LIVE, + Ordering::AcqRel, + ); context.init_notify.notify_waiters(); context.ready_notify.notify_waiters(); context.inbound.lock().expect("host inbound lock").take(); let owner = context.owner.lock().expect("physical owner lock").upgrade(); - if let Some(owner) = owner { owner.request_terminal(true); } + if let Some(owner) = owner { + owner.request_terminal(true); + } } })); } diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index e64bc734..14d6e9f0 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1311,7 +1311,10 @@ async fn unanswered_method_calls_are_not_physical_callback_leases() { #[tokio::test] async fn abandoning_initiating_waiter_faults_and_retains_late_initializer_cleanup() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; SHUTDOWN_BLOCKED.store(false, Ordering::Release); SHUTDOWN_CODE.store(TB_OK, Ordering::Release); SHUTDOWN_CALLS.store(0, Ordering::Release); @@ -1321,7 +1324,10 @@ async fn abandoning_initiating_waiter_faults_and_retains_late_initializer_cleanu transport.defer_initializer(host, move |host| { entered_tx.send(()).unwrap(); release_rx.recv().unwrap(); - assert_eq!(unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }, b"secret"); + assert_eq!( + unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }, + b"secret" + ); let mut module = TbModuleVtable::default(); assert_eq!(unsafe { initialize_ok(&host, &mut module) }, TB_OK); Ok(module) @@ -1336,13 +1342,20 @@ async fn abandoning_initiating_waiter_faults_and_retains_late_initializer_cleanu // Always release foreign work even on the unfixed implementation. let faulted = transport.is_faulted(); release_tx.send(()).unwrap(); - assert!(faulted, "initiating waiter abandonment must publish fault immediately"); + assert!( + faulted, + "initiating waiter abandonment must publish fault immediately" + ); tokio::time::timeout(Duration::from_secs(2), async { loop { - if transport.is_stopped() { break; } + if transport.is_stopped() { + break; + } tokio::task::yield_now().await; } - }).await.unwrap(); + }) + .await + .unwrap(); assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1); assert!(transport.ensure_initialized().await.is_err()); } @@ -1353,19 +1366,32 @@ async fn scheduled_initializer_fault_cancels_before_actual_entry() { let entries = Arc::new(AtomicUsize::new(0)); transport.defer_initializer(host, { let entries = entries.clone(); - move |_| { entries.fetch_add(1, Ordering::AcqRel); Err("unexpected C entry".into()) } + move |_| { + entries.fetch_add(1, Ordering::AcqRel); + Err("unexpected C entry".into()) + } }); let guard = transport.lifecycle.clone().lock_owned().await; assert!(transport.schedule_initializer().unwrap()); transport.request_terminal(true); assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); - assert_eq!(transport.context.physical.lock().unwrap().init, InitState::Scheduled); + assert_eq!( + transport.context.physical.lock().unwrap().init, + InitState::Scheduled + ); drop(guard); tokio::time::timeout(Duration::from_secs(2), async { - while !transport.is_stopped() { tokio::task::yield_now().await; } - }).await.unwrap(); + while !transport.is_stopped() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); assert_eq!(entries.load(Ordering::Acquire), 0); - assert_eq!(transport.context.physical.lock().unwrap().cleanup, CleanupState::NoCallbackRequired); + assert_eq!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::NoCallbackRequired + ); assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); } @@ -1375,7 +1401,10 @@ async fn scheduled_initializer_is_adopted_by_original_ordinary_stop() { let entries = Arc::new(AtomicUsize::new(0)); transport.defer_initializer(host, { let entries = entries.clone(); - move |_| { entries.fetch_add(1, Ordering::AcqRel); Err("unexpected C entry".into()) } + move |_| { + entries.fetch_add(1, Ordering::AcqRel); + Err("unexpected C entry".into()) + } }); let (worker_started, worker_started_rx) = tokio::sync::oneshot::channel(); let (worker_release, worker_release_rx) = std::sync::mpsc::channel(); @@ -1384,15 +1413,20 @@ async fn scheduled_initializer_is_adopted_by_original_ordinary_stop() { worker_release_rx.recv().unwrap(); })); let waiter = tokio::spawn({ - let transport = transport.clone(); async move { transport.ensure_initialized().await } + let transport = transport.clone(); + async move { transport.ensure_initialized().await } }); worker_started_rx.await.unwrap(); let mut stopping = Box::pin(transport.clone().stop(Duration::from_secs(2))); tokio::select! { result = &mut stopping => panic!("stop completed before owner: {result:?}"), - _ = async { while !transport.stop_task_spawned() { tokio::task::yield_now().await; } } => {} } + _ = async { while !transport.stop_task_spawned() { tokio::task::yield_now().await; } } => {} } waiter.abort(); assert!(waiter.await.unwrap_err().is_cancelled()); - assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 2, "ordinary cancellation cannot invent Fault"); + assert_eq!( + unsafe { host_native_terminal_state(host.host_ctx) }, + 2, + "ordinary cancellation cannot invent Fault" + ); worker_release.send(()).unwrap(); assert_eq!(stopping.await.unwrap(), TB_CLOSED); assert_eq!(entries.load(Ordering::Acquire), 0); @@ -1402,21 +1436,39 @@ async fn scheduled_initializer_is_adopted_by_original_ordinary_stop() { #[tokio::test] async fn rejected_spawn_retains_actual_initializer_owner_and_never_resubmits() { struct Probe(Arc); - impl Drop for Probe { fn drop(&mut self) { self.0.store(true, Ordering::Release); } } + impl Drop for Probe { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } + } let dropped = Arc::new(AtomicBool::new(false)); let (transport, host) = ModuleTransport::new("spawn-rejection".into(), vec![]); transport.reject_worker_spawn.store(true, Ordering::Release); let probe = Probe(dropped.clone()); - transport.defer_initializer(host, move |_| { drop(probe); Err("must never execute".into()) }); + transport.defer_initializer(host, move |_| { + drop(probe); + Err("must never execute".into()) + }); assert!(transport.ensure_initialized().await.is_err()); assert!(transport.is_faulted()); let weak = Arc::downgrade(&transport); - assert_eq!(transport.context.physical.lock().unwrap().cleanup, CleanupState::Uncertain); - transport.reject_worker_spawn.store(false, Ordering::Release); + assert_eq!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Uncertain + ); + transport + .reject_worker_spawn + .store(false, Ordering::Release); assert!(transport.ensure_initialized().await.is_err()); drop(transport); - assert!(weak.upgrade().is_some(), "context anchor retains actual job through host teardown"); - assert!(!dropped.load(Ordering::Acquire), "closure rejection must not drop controller"); + assert!( + weak.upgrade().is_some(), + "context anchor retains actual job through host teardown" + ); + assert!( + !dropped.load(Ordering::Acquire), + "closure rejection must not drop controller" + ); assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); } @@ -1441,7 +1493,9 @@ unsafe extern "C" fn native_entered(ctx: *mut c_void, host: *const TbHostVtable) let mut state = barrier.state.lock().unwrap(); state.0 = Some(unsafe { *host }); barrier.entered.notify_one(); - while !state.1 { state = barrier.changed.wait(state).unwrap(); } + while !state.1 { + state = barrier.changed.wait(state).unwrap(); + } } unsafe extern "C" fn native_joined(ctx: *mut c_void) { let barrier = unsafe { &*ctx.cast::() }; @@ -1451,17 +1505,38 @@ unsafe extern "C" fn native_joined(ctx: *mut c_void) { barrier.joined.notify_one(); drop(state); let gate = barrier.finish_gate.lock().unwrap().take(); - if let Some(gate) = gate { gate.recv().unwrap(); } + if let Some(gate) = gate { + gate.recv().unwrap(); + } } -pub(crate) fn native_fixture() -> (Arc, TbHostVtable, crate::module::abi::TbModuleInit, Arc) { - let path = std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").expect("controlled native fixture path"); +pub(crate) fn native_fixture() -> ( + Arc, + TbHostVtable, + crate::module::abi::TbModuleInit, + Arc, +) { + let path = std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE") + .expect("controlled native fixture path"); let artifact = crate::module::loader::load(std::path::Path::new(&path), false).unwrap(); let barrier = Arc::new(NativeBarrier { - state: StdMutex::new((None, false, false)), changed: Condvar::new(), - entered: Notify::new(), joined: Notify::new(), finish_gate: StdMutex::new(None), + state: StdMutex::new((None, false, false)), + changed: Condvar::new(), + entered: Notify::new(), + joined: Notify::new(), + finish_gate: StdMutex::new(None), }); - let controls = NativeControls { ctx: Arc::as_ptr(&barrier).cast_mut().cast(), entered: native_entered, joined: native_joined }; - let config = unsafe { std::slice::from_raw_parts(std::ptr::from_ref(&controls).cast::(), size_of::()) }.to_vec(); + let controls = NativeControls { + ctx: Arc::as_ptr(&barrier).cast_mut().cast(), + entered: native_entered, + joined: native_joined, + }; + let config = unsafe { + std::slice::from_raw_parts( + std::ptr::from_ref(&controls).cast::(), + size_of::(), + ) + } + .to_vec(); let (transport, host) = ModuleTransport::new("actual-native".into(), config); (transport, host, artifact.init, barrier) } @@ -1472,30 +1547,47 @@ async fn actual_native_late_init_is_joined_after_initiator_abandonment() { let (transport, host, init, barrier) = native_fixture(); transport.defer_initialize(init, host); let waiter = tokio::spawn({ - let transport = transport.clone(); async move { transport.ensure_initialized().await } + let transport = transport.clone(); + async move { transport.ensure_initialized().await } }); barrier.entered.notified().await; let mut observation = Box::pin(transport.ensure_initialized()); std::future::poll_fn(|cx| { assert!(std::future::Future::poll(observation.as_mut(), cx).is_pending()); std::task::Poll::Ready(()) - }).await; + }) + .await; drop(observation); - assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 1, "observer abandonment cannot cancel initiating owner"); + assert_eq!( + unsafe { host_native_terminal_state(host.host_ctx) }, + 1, + "observer abandonment cannot cancel initiating owner" + ); waiter.abort(); assert!(waiter.await.unwrap_err().is_cancelled()); let view = barrier.state.lock().unwrap().0.unwrap(); - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 6); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 6 + ); assert!(!barrier.state.lock().unwrap().2); { let mut state = barrier.state.lock().unwrap(); state.1 = true; barrier.changed.notify_all(); } - tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); let _physical_exit = transport.lifecycle.lock().await; - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 14); - assert_eq!(transport.context.physical.lock().unwrap().cleanup, CleanupState::Complete); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 14 + ); + assert_eq!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Complete + ); assert!(transport.ensure_initialized().await.is_err()); } @@ -1503,16 +1595,30 @@ async fn actual_native_late_init_is_joined_after_initiator_abandonment() { #[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] async fn actual_native_eager_fault_denies_while_config_remains_borrowed_then_joins() { let (transport, host, init, barrier) = native_fixture(); - let host = DeferredInitializer { initialize: Box::new(|_| unreachable!()), host }; + let host = DeferredInitializer { + initialize: Box::new(|_| unreachable!()), + host, + }; let eager = std::thread::spawn({ let transport = transport.clone(); - move || { let owner = host; transport.eager_initialize(init, owner.host) } + move || { + let owner = host; + transport.eager_initialize(init, owner.host) + } }); barrier.entered.notified().await; let view = barrier.state.lock().unwrap().0.unwrap(); - unsafe { (view.fault)(view.host_ctx, std::ptr::null(), 0); } - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 6); - assert_eq!(transport.context.config.lock().unwrap().len(), size_of::()); + unsafe { + (view.fault)(view.host_ctx, std::ptr::null(), 0); + } + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 6 + ); + assert_eq!( + transport.context.config.lock().unwrap().len(), + size_of::() + ); assert!(!barrier.state.lock().unwrap().2); { let mut state = barrier.state.lock().unwrap(); @@ -1520,23 +1626,33 @@ async fn actual_native_eager_fault_denies_while_config_remains_borrowed_then_joi barrier.changed.notify_all(); } assert!(eager.join().unwrap().is_err()); - tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); let _physical_exit = transport.lifecycle.lock().await; - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 14); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 14 + ); assert!(transport.context.config.lock().unwrap().is_empty()); } - async fn wait_physical_cleanup(transport: &ModuleTransport) { tokio::time::timeout(Duration::from_secs(2), async { loop { let changed = transport.context.physical_notify.notified(); - let terminal = matches!(transport.context.physical.lock().unwrap().cleanup, - CleanupState::Complete | CleanupState::Uncertain | CleanupState::NoCallbackRequired); - if terminal { return; } + let terminal = matches!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Complete | CleanupState::Uncertain | CleanupState::NoCallbackRequired + ); + if terminal { + return; + } changed.await; } - }).await.unwrap(); + }) + .await + .unwrap(); } #[tokio::test] @@ -1546,13 +1662,22 @@ async fn observing_fault_owned_job_does_not_create_ordinary_stop_marker() { transport.defer_initialize(initialize_ok, host); assert!(transport.ensure_initialized().await.is_err()); assert!(!transport.stop_task_spawned()); - assert!(matches!(transport.clone().stop(Duration::from_secs(1)).await, Err(StopError::NotStarted(_)))); - assert!(!transport.stop_task_spawned(), "fault-owned uncertainty cannot become an ordinary handoff"); + assert!(matches!( + transport.clone().stop(Duration::from_secs(1)).await, + Err(StopError::NotStarted(_)) + )); + assert!( + !transport.stop_task_spawned(), + "fault-owned uncertainty cannot become an ordinary handoff" + ); } #[tokio::test] async fn ordinary_shutdown_retains_non_ok_return_without_physical_success() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; let (transport, host) = ModuleTransport::new("shutdown-status".into(), vec![]); transport.defer_initialize(initialize_ok, host); transport.ensure_initialized().await.unwrap(); @@ -1560,17 +1685,30 @@ async fn ordinary_shutdown_retains_non_ok_return_without_physical_success() { SHUTDOWN_BLOCKED.store(false, Ordering::Release); let result = transport.clone().stop(Duration::from_secs(1)).await; SHUTDOWN_CODE.store(TB_OK, Ordering::Release); - assert_eq!(result.unwrap(), TB_TIMEOUT, "ordinary public return remains compatible"); + assert_eq!( + result.unwrap(), + TB_TIMEOUT, + "ordinary public return remains compatible" + ); assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 2); - assert_eq!(transport.context.physical.lock().unwrap().cleanup, CleanupState::Uncertain); + assert_eq!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Uncertain + ); assert!(transport.stop_task_spawned()); - assert!(transport.is_stopped(), "legacy stopped is separate from physical-success proof"); + assert!( + transport.is_stopped(), + "legacy stopped is separate from physical-success proof" + ); assert!(!transport.is_faulted()); } #[tokio::test] async fn rejected_module_outputs_never_invoke_untrusted_callbacks() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; SHUTDOWN_CALLS.store(0, Ordering::Release); for rejection in 0..4 { let (transport, host) = ModuleTransport::new(format!("rejected-{rejection}"), vec![]); @@ -1599,15 +1737,29 @@ async fn rejected_ordinary_worker_submission_retains_original_lifecycle_guard() transport.defer_initialize(initialize_ok, host); transport.ensure_initialized().await.unwrap(); transport.reject_worker_spawn.store(true, Ordering::Release); - assert!(matches!(transport.clone().stop(Duration::from_secs(1)).await, Err(StopError::Started(_)))); + assert!(matches!( + transport.clone().stop(Duration::from_secs(1)).await, + Err(StopError::Started(_)) + )); assert!(transport.stop_task_spawned()); - assert!(transport.lifecycle.try_lock().is_err(), "rejected closure cannot drop actual acquired guard"); + assert!( + transport.lifecycle.try_lock().is_err(), + "rejected closure cannot drop actual acquired guard" + ); let weak = Arc::downgrade(&transport); drop(transport); - let retained = weak.upgrade().expect("actual owner retained beyond host handle"); - assert_eq!(retained.context.physical.lock().unwrap().cleanup, CleanupState::Uncertain); + let retained = weak + .upgrade() + .expect("actual owner retained beyond host handle"); + assert_eq!( + retained.context.physical.lock().unwrap().cleanup, + CleanupState::Uncertain + ); assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 2); - assert!(!retained.is_faulted(), "ordinary spawner failure is uncertainty, not module fault"); + assert!( + !retained.is_faulted(), + "ordinary spawner failure is uncertainty, not module fault" + ); } #[tokio::test] @@ -1616,7 +1768,8 @@ async fn actual_native_deadline_fault_retains_late_cleanup_without_wall_clock_sl let (transport, host, init, barrier) = native_fixture(); transport.defer_initialize(init, host); let waiter = tokio::spawn({ - let transport = transport.clone(); async move { transport.ensure_initialized().await } + let transport = transport.clone(); + async move { transport.ensure_initialized().await } }); barrier.entered.notified().await; tokio::time::pause(); @@ -1631,7 +1784,9 @@ async fn actual_native_deadline_fault_retains_late_cleanup_without_wall_clock_sl } // Resume the real timer only for the outer stuck-test safety deadline. tokio::time::resume(); - tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); wait_physical_cleanup(&transport).await; assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); } @@ -1648,25 +1803,42 @@ async fn actual_native_fault_between_return_and_publication_uses_same_owner() { })); transport.defer_initialize(init, host); let waiter = tokio::spawn({ - let transport = transport.clone(); async move { transport.ensure_initialized().await } + let transport = transport.clone(); + async move { transport.ensure_initialized().await } }); barrier.entered.notified().await; { - let mut state = barrier.state.lock().unwrap(); state.1 = true; barrier.changed.notify_all(); + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); } publication_rx.await.unwrap(); - let job = transport.context.physical.lock().unwrap().job.clone().unwrap(); - unsafe { (host.fault)(host.host_ctx, std::ptr::null(), 0); } + let job = transport + .context + .physical + .lock() + .unwrap() + .job + .clone() + .unwrap(); + unsafe { + (host.fault)(host.host_ctx, std::ptr::null(), 0); + } { let state = transport.context.physical.lock().unwrap(); assert!(Arc::ptr_eq(&job, state.job.as_ref().unwrap())); assert_eq!(state.cleanup, CleanupState::Requested); - assert!(state.result.is_none(), "cleanup must not claim an empty publication"); + assert!( + state.result.is_none(), + "cleanup must not claim an empty publication" + ); } assert!(!barrier.state.lock().unwrap().2); release_tx.send(()).unwrap(); assert!(waiter.await.unwrap().is_err()); - tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); wait_physical_cleanup(&transport).await; assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); } @@ -1676,31 +1848,47 @@ async fn actual_native_fault_between_return_and_publication_uses_same_owner() { async fn actual_native_reinitializer_fault_denies_before_c_returns_and_joins_once() { let (transport, host, init, barrier) = native_fixture(); { - let mut state = barrier.state.lock().unwrap(); state.1 = true; + let mut state = barrier.state.lock().unwrap(); + state.1 = true; } transport.defer_initialize(init, host); transport.ensure_initialized().await.unwrap(); barrier.entered.notified().await; { - let mut state = barrier.state.lock().unwrap(); state.1 = false; + let mut state = barrier.state.lock().unwrap(); + state.1 = false; } let reinit = tokio::spawn({ let transport = transport.clone(); async move { transport.reinitialize(crate::Secret::new(vec![])).await } }); barrier.entered.notified().await; - unsafe { (host.fault)(host.host_ctx, std::ptr::null(), 0); } + unsafe { + (host.fault)(host.host_ctx, std::ptr::null(), 0); + } assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); assert!(!barrier.state.lock().unwrap().2); - assert!(transport.lifecycle.try_lock().is_err(), "actual C reinit owns physical exclusion"); + assert!( + transport.lifecycle.try_lock().is_err(), + "actual C reinit owns physical exclusion" + ); { - let mut state = barrier.state.lock().unwrap(); state.1 = true; barrier.changed.notify_all(); + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); } reinit.await.unwrap().unwrap(); - tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); wait_physical_cleanup(&transport).await; assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); - assert!(transport.reinitialize(crate::Secret::new(vec![])).await.is_err()); + assert!( + transport + .reinitialize(crate::Secret::new(vec![])) + .await + .is_err() + ); } #[tokio::test] @@ -1708,20 +1896,42 @@ async fn actual_native_reinitializer_fault_denies_before_c_returns_and_joins_onc async fn actual_native_late_eager_publication_cannot_refund_failed_cleanup_spawn() { let (transport, host, init, barrier) = native_fixture(); transport.reject_worker_spawn.store(true, Ordering::Release); - let host_owner = DeferredInitializer { initialize: Box::new(|_| unreachable!()), host }; + let host_owner = DeferredInitializer { + initialize: Box::new(|_| unreachable!()), + host, + }; let eager = std::thread::spawn({ let transport = transport.clone(); - move || { let owner = host_owner; transport.eager_initialize(init, owner.host) } + move || { + let owner = host_owner; + transport.eager_initialize(init, owner.host) + } }); barrier.entered.notified().await; - unsafe { (host.fault)(host.host_ctx, std::ptr::null(), 0); } - let job = transport.context.physical.lock().unwrap().job.clone().unwrap(); - assert_eq!(transport.context.physical.lock().unwrap().cleanup, CleanupState::Uncertain); + unsafe { + (host.fault)(host.host_ctx, std::ptr::null(), 0); + } + let job = transport + .context + .physical + .lock() + .unwrap() + .job + .clone() + .unwrap(); + assert_eq!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Uncertain + ); { - let mut state = barrier.state.lock().unwrap(); state.1 = true; barrier.changed.notify_all(); + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); } assert!(eager.join().unwrap().is_err()); - transport.reject_worker_spawn.store(false, Ordering::Release); + transport + .reject_worker_spawn + .store(false, Ordering::Release); transport.request_terminal(true); { let state = transport.context.physical.lock().unwrap(); @@ -1729,9 +1939,15 @@ async fn actual_native_late_eager_publication_cannot_refund_failed_cleanup_spawn assert_eq!(state.cleanup, CleanupState::Uncertain); assert!(Arc::ptr_eq(&job, state.job.as_ref().unwrap())); } - assert!(transport.module.lock().unwrap().is_some(), "valid late output is cleanup access only"); + assert!( + transport.module.lock().unwrap().is_some(), + "valid late output is cleanup access only" + ); assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); - assert!(!barrier.state.lock().unwrap().2, "rejected submission must never be silently retried"); + assert!( + !barrier.state.lock().unwrap().2, + "rejected submission must never be silently retried" + ); let weak = Arc::downgrade(&transport); drop(transport); drop(job); @@ -1742,31 +1958,59 @@ async fn actual_native_late_eager_publication_cannot_refund_failed_cleanup_spawn std::mem::forget(barrier); } - -unsafe extern "C" fn initialize_short_output(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { - unsafe { initialize_ok(host, out); (*out).size = TB_MODULE_VTABLE_BASE_SIZE - 1; } +unsafe extern "C" fn initialize_short_output( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + unsafe { + initialize_ok(host, out); + (*out).size = TB_MODULE_VTABLE_BASE_SIZE - 1; + } TB_OK } -unsafe extern "C" fn initialize_null_context_output(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { - unsafe { initialize_ok(host, out); (*out).module_ctx = std::ptr::null_mut(); } +unsafe extern "C" fn initialize_null_context_output( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + unsafe { + initialize_ok(host, out); + (*out).module_ctx = std::ptr::null_mut(); + } TB_OK } -unsafe extern "C" fn initialize_missing_declared_reinit(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { - unsafe { initialize_ok(host, out); (*out).reinitialize = None; } +unsafe extern "C" fn initialize_missing_declared_reinit( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + unsafe { + initialize_ok(host, out); + (*out).reinitialize = None; + } TB_OK } -unsafe extern "C" fn initialize_non_ok_misleading_output(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { - unsafe { initialize_ok(host, out); } +unsafe extern "C" fn initialize_non_ok_misleading_output( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + unsafe { + initialize_ok(host, out); + } TB_PANICKED } #[tokio::test] async fn eager_rejected_outputs_keep_uncertainty_and_never_call_returned_callbacks() { - let _lock = VTABLE_TEST_LOCK.get_or_init(|| tokio::sync::Mutex::new(())).lock().await; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; SHUTDOWN_CALLS.store(0, Ordering::Release); - for init in [initialize_short_output as crate::module::abi::TbModuleInit, - initialize_null_context_output, initialize_missing_declared_reinit, - initialize_non_ok_misleading_output] { + for init in [ + initialize_short_output as crate::module::abi::TbModuleInit, + initialize_null_context_output, + initialize_missing_declared_reinit, + initialize_non_ok_misleading_output, + ] { let (transport, host) = ModuleTransport::new("eager-invalid".into(), b"borrowed".to_vec()); assert!(transport.eager_initialize(init, host).is_err()); let state = transport.context.physical.lock().unwrap(); @@ -1789,25 +2033,38 @@ fn actual_native_initializer_owner_outlives_runtime_and_host_handle_drop() { let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.block_on(async { let transport = transport.clone(); - tokio::spawn(async move { let _ = transport.ensure_initialized().await; }); + tokio::spawn(async move { + let _ = transport.ensure_initialized().await; + }); barrier.entered.notified().await; }); let weak = Arc::downgrade(&transport); drop(runtime); assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); drop(transport); - let retained = weak.upgrade().expect("real initializer owner survives runtime and host drop"); + let retained = weak + .upgrade() + .expect("real initializer owner survives runtime and host drop"); assert!(!barrier.state.lock().unwrap().2); { - let mut state = barrier.state.lock().unwrap(); state.1 = true; barrier.changed.notify_all(); + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); } // The runtime is gone. Wait on the retained physical predicate/Condvar, // never on a Tokio task, deadline or notification as completion proof. let mut state = retained.context.physical.lock().unwrap(); while state.cleanup != CleanupState::Complete { - let (next, timeout) = retained.context.physical_changed.wait_timeout(state, Duration::from_secs(2)).unwrap(); + let (next, timeout) = retained + .context + .physical_changed + .wait_timeout(state, Duration::from_secs(2)) + .unwrap(); state = next; - assert!(!timeout.timed_out(), "actual retained native cleanup must finish"); + assert!( + !timeout.timed_out(), + "actual retained native cleanup must finish" + ); } drop(state); assert!(barrier.state.lock().unwrap().2); From 2034303ca5edeebcf81cafd68cdbb0b3c7eb8cbb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:16:43 +0300 Subject: [PATCH 39/58] fix(module): publish terminal flags on rejected stop and fault paths The transport now routes fault and rejected-stop transitions through publish_terminal_flags so the faulted and terminal state stay consistent instead of setting the faulted flag directly. A test-only accessor exposes whether the lifecycle guard is held, and a host test covers a rejected stop submission retaining its guard through host drop. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_tests.rs | 39 +++++++++++++++++++++++++ crates/tinybus/src/module/transport.rs | 10 +++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 16c79265..16aa58e4 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -3212,3 +3212,42 @@ async fn actual_native_fault_during_ordinary_stop_keeps_fault_state_and_single_j ); assert!(barrier.state.lock().unwrap().2); } + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_rejected_stop_submission_retains_guard_through_host_drop() { + let (transport, view, barrier, artifact, path) = controlled_native_transport().await; + let host = ModuleHost::new(Broker::new()); + attach_controlled_native(&host, transport.clone(), artifact, &path).unwrap(); + transport.reject_worker_spawn.store(true, Ordering::Release); + assert!(matches!( + transport.clone().stop(Duration::from_secs(1)).await, + Err(StopError::Started(_)) + )); + let weak = Arc::downgrade(&transport); + drop(host); + drop(transport); + let retained = weak + .upgrade() + .expect("external anchor retains physical owner through real Host Drop"); + assert!(retained.lifecycle_is_owned_for_test()); + assert!(retained.stop_task_spawned()); + assert!(!retained.is_faulted()); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2 + ); + assert!( + !barrier.state.lock().unwrap().2, + "rejected submission is not physical completion" + ); + retained.reject_worker_spawn.store(false, Ordering::Release); + retained.request_terminal(false); + assert!( + !barrier.state.lock().unwrap().2, + "Host Drop cannot refund and resubmit an uncertain owner" + ); + // One deliberately retained native uncertainty instance, with its controller + // lifetime retained explicitly; no process-wide leak or backing-Drop claim. + std::mem::forget(barrier); +} diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 67e9429e..141acda9 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -856,6 +856,7 @@ impl ModuleTransport { if self.admission_closed() { return Err(Error::ConnectionClosed); } + self.publish_terminal_flags(true); drop(_pending); self.request_terminal(true); self.context.ready_notify.notify_waiters(); @@ -929,7 +930,7 @@ impl ModuleTransport { pending.clear(); return; } - self.context.faulted.store(true, Ordering::Release); + self.publish_terminal_flags(true); std::mem::take(&mut *pending) }; // Each of these calls incremented `inflight` when it was queued. The @@ -1000,6 +1001,11 @@ impl ModuleTransport { code } + #[cfg(test)] + pub(crate) fn lifecycle_is_owned_for_test(&self) -> bool { + self.lifecycle.try_lock().is_err() + } + pub(crate) fn stop_task_spawned(&self) -> bool { self.stop_task_spawned.load(Ordering::Acquire) } @@ -1311,7 +1317,7 @@ impl Transport for ModuleTransport { return Err(Error::ConnectionClosed); } self.context.init_failed.store(true, Ordering::Release); - self.context.faulted.store(true, Ordering::Release); + self.publish_terminal_flags(true); } self.context.ready_notify.notify_waiters(); if message.header.kind == MessageKind::MethodCall { From 150afdc21ff125759dde2c5f7daa06fff71399f8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:40:59 +0300 Subject: [PATCH 40/58] test(module): cover native callback cancellation after ordinary stop Add ignored integration tests that drive the controlled-init fixture through ordinary shutdown while the loader is suspended, checking that cancellation preserves the non-fault cause. The example fixture now also accepts its Controls bytes as a JSON array so the public loader path can be exercised without product-side seams. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../examples/module_controlled_init.rs | 26 +++++- .../src/module/host_callbacks_tests.rs | 86 +++++++++++++++++++ crates/tinybus/src/module/host_tests.rs | 2 +- crates/tinybus/src/module/transport_tests.rs | 14 +++ 4 files changed, 123 insertions(+), 5 deletions(-) diff --git a/crates/tinybus/examples/module_controlled_init.rs b/crates/tinybus/examples/module_controlled_init.rs index 7c0a4498..a4ee12ba 100644 --- a/crates/tinybus/examples/module_controlled_init.rs +++ b/crates/tinybus/examples/module_controlled_init.rs @@ -67,12 +67,30 @@ pub unsafe extern "C" fn tinybus_module_init_v1( out: *mut TbModuleVtable, ) -> i32 { let host = unsafe { &*host }; - if host.config.len != size_of::() { - return TB_BAD_ARGUMENT; - } - let controls = unsafe { host.config.ptr.cast::().read_unaligned() }; let copied_config = unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }.to_vec(); + // The public loader serializes JSON. This declared test fixture accepts the + // same finite Controls bytes in a JSON array, without product-side seams. + let encoded; + let bytes = if copied_config.len() == size_of::() { + &copied_config + } else { + let Ok(value) = serde_json::from_slice::(&copied_config) else { + return TB_BAD_ARGUMENT; + }; + let Some(value) = value.get("test_controls") else { + return TB_BAD_ARGUMENT; + }; + let Ok(value) = serde_json::from_value::>(value.clone()) else { + return TB_BAD_ARGUMENT; + }; + encoded = value; + &encoded + }; + if bytes.len() != size_of::() { + return TB_BAD_ARGUMENT; + } + let controls = unsafe { bytes.as_ptr().cast::().read_unaligned() }; // The host-side barrier blocks actual opaque C entry, retaining its config // borrow through fault publication and initiating waiter abandonment. unsafe { diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 4ddd7b2f..af4e5acd 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -556,3 +556,89 @@ fn configuration_fingerprints_preserve_json_equality_without_normalizing_strings fingerprint(&serde_json::json!(0.01)) ); } + +// Manually polling the public future lets ordinary shutdown commit while the +// loader is suspended; no scheduler race decides which cancellation is tested. +async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: bool) { + use std::future::Future; + use super::super::super::transport::tests::{native_fixture, native_fixture_json}; + let (unused, _, _, barrier) = native_fixture(); + let config = native_fixture_json(&unused); + drop(unused); + let artifact = std::path::PathBuf::from( + std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); + let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); + let (_dir, artifact) = super::super::tests::staged_module(&artifact, &hash); + let host = ModuleHost::new(Broker::new()); + let binding = host.prepare_host_callbacks(vec![endpoint()]).await.unwrap(); + let (entered, _) = tokio::sync::mpsc::channel(1); + binding.serve_at(endpoint().path, BlockingCallback { + entered, + release: Arc::new(tokio::sync::Semaphore::new(0)), + published: Arc::new(AtomicUsize::new(0)), + guard: binding.guard(), + }).await.unwrap(); + let mut load = Box::pin(host.load_file_with_host_callbacks(&artifact, config, &binding)); + std::future::poll_fn(|cx| { + assert!(load.as_mut().poll(cx).is_pending(), "loader awaits real C init"); + std::task::Poll::Ready(()) + }).await; + tokio::time::timeout(Duration::from_secs(2), barrier.entered.notified()).await.unwrap(); + let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + let view = barrier.state.lock().unwrap().0.unwrap(); + barrier.state.lock().unwrap().1 = true; + barrier.changed.notify_all(); + transport.ensure_initialized().await.unwrap(); + // Advance the public future to its real withheld-Ready observation. + std::future::poll_fn(|cx| { + assert!(load.as_mut().poll(cx).is_pending(), "fixture withholds Ready"); + std::task::Poll::Ready(()) + }).await; + let (finish, receive) = std::sync::mpsc::channel(); + *barrier.finish_gate.lock().unwrap() = Some(receive); + let stopping = transport.clone(); + let stop = tokio::spawn(async move { stopping.stop(Duration::from_secs(2)).await }); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 2); + let load_result = if reject_gate { + transport.release_initialization(false); + drop(load); + None + } else if drop_load { + drop(load); + None + } else { + // Real not-ready branch awaits close, whose native return is held. + std::future::poll_fn(|cx| { + match load.as_mut().poll(cx) { + std::task::Poll::Pending => std::task::Poll::Ready(None), + std::task::Poll::Ready(result) => std::task::Poll::Ready(Some(result)), + } + }).await + }; + let held_flags = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; + let held_fault = transport.is_faulted(); + finish.send(()).unwrap(); // Always release opaque C before RED assertions. + assert!(stop.await.unwrap().is_ok()); + if let Some(result) = load_result { assert!(result.is_err()); } + assert_eq!(held_flags, 2, "ordinary cancellation cannot invent Fault"); + assert!(!held_fault); + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 10); + assert!(barrier.state.lock().unwrap().2); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_callback_notready_after_ordinary_stop_preserves_cause() { + native_callback_load_after_ordinary_stop(false, false).await; +} +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_callback_load_drop_after_ordinary_stop_preserves_cause() { + native_callback_load_after_ordinary_stop(true, false).await; +} +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_callback_gate_rejection_after_ordinary_stop_preserves_cause() { + native_callback_load_after_ordinary_stop(false, true).await; +} diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 16aa58e4..153e9f4b 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -2188,7 +2188,7 @@ async fn one_refused_module_does_not_stop_the_others_in_the_directory_from_loadi /// tracing subscriber inside the library, so initializing the same DLL path /// twice shares state even across independent hosts. Windows staging uses the /// release-cache ACL repair so the real admission gate remains in force. -fn staged_module(artifact: &Path, hash: &str) -> (tempfile::TempDir, PathBuf) { +pub(super) fn staged_module(artifact: &Path, hash: &str) -> (tempfile::TempDir, PathBuf) { // Staged beside the artifact rather than in `/tmp`: the loader refuses a // directory another user could write to, and `/tmp` is exactly that. The // artifact's own directory is user-owned (CI builds into diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 14d6e9f0..a70a5d43 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -2072,3 +2072,17 @@ fn actual_native_initializer_owner_outlives_runtime_and_host_handle_drop() { drop(retained); assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); } + +pub(crate) fn native_fixture_json(transport: &ModuleTransport) -> serde_json::Value { + serde_json::json!({"test_controls": *transport.context.config.lock().unwrap()}) +} + +#[tokio::test] +async fn rejected_observer_after_ordinary_stop_preserves_nonfault_cause() { + let (transport, view) = ModuleTransport::new("ordinary-cancellation".into(), vec![]); + let _ = transport.clone().stop(Duration::from_secs(1)).await; + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 2); + transport.release_initialization(false); + assert!(!transport.is_faulted()); + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 2); +} From 05470af9448b475922d75828d14822d80cc35cbe Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:41:32 +0300 Subject: [PATCH 41/58] test(tinybus): await load future in native callback stop test The load future is now held in an Option so the test can await it in the non-drop branch instead of discarding it, ensuring the withheld-Ready path is actually driven to completion. The remaining branches take the future before dropping it, and the result assertion is split to cover both outcomes. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/module/host_callbacks_tests.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index af4e5acd..05ca86f0 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -578,9 +578,9 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: published: Arc::new(AtomicUsize::new(0)), guard: binding.guard(), }).await.unwrap(); - let mut load = Box::pin(host.load_file_with_host_callbacks(&artifact, config, &binding)); + let mut load = Some(Box::pin(host.load_file_with_host_callbacks(&artifact, config, &binding))); std::future::poll_fn(|cx| { - assert!(load.as_mut().poll(cx).is_pending(), "loader awaits real C init"); + assert!(load.as_mut().unwrap().as_mut().poll(cx).is_pending(), "loader awaits real C init"); std::task::Poll::Ready(()) }).await; tokio::time::timeout(Duration::from_secs(2), barrier.entered.notified()).await.unwrap(); @@ -591,7 +591,7 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: transport.ensure_initialized().await.unwrap(); // Advance the public future to its real withheld-Ready observation. std::future::poll_fn(|cx| { - assert!(load.as_mut().poll(cx).is_pending(), "fixture withholds Ready"); + assert!(load.as_mut().unwrap().as_mut().poll(cx).is_pending(), "fixture withholds Ready"); std::task::Poll::Ready(()) }).await; let (finish, receive) = std::sync::mpsc::channel(); @@ -602,15 +602,15 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 2); let load_result = if reject_gate { transport.release_initialization(false); - drop(load); + drop(load.take()); None } else if drop_load { - drop(load); + drop(load.take()); None } else { // Real not-ready branch awaits close, whose native return is held. std::future::poll_fn(|cx| { - match load.as_mut().poll(cx) { + match load.as_mut().unwrap().as_mut().poll(cx) { std::task::Poll::Pending => std::task::Poll::Ready(None), std::task::Poll::Ready(result) => std::task::Poll::Ready(Some(result)), } @@ -620,7 +620,12 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: let held_fault = transport.is_faulted(); finish.send(()).unwrap(); // Always release opaque C before RED assertions. assert!(stop.await.unwrap().is_ok()); - if let Some(result) = load_result { assert!(result.is_err()); } + if let Some(result) = load_result { + assert!(result.is_err()); + drop(load.take()); + } else if let Some(load) = load { + assert!(load.await.is_err()); + } assert_eq!(held_flags, 2, "ordinary cancellation cannot invent Fault"); assert!(!held_fault); assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 10); From 987c7c66c0f812cd92b89ebcb6aa7032a65d78d3 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:43:03 +0300 Subject: [PATCH 42/58] fix(module): distinguish cancellation from failure during admission Admission rollback now records whether termination came from a real initialization failure or from cancellation of a dropped waiter, so a canceled admission no longer fabricates a fault over an already committed ordinary stop. The rollback path threads this cause through to the transport, which classifies the terminal state accordingly while still preserving any genuine sticky fault. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_callbacks.rs | 38 ++++++++++++------- .../src/module/host_callbacks_tests.rs | 1 + crates/tinybus/src/module/transport.rs | 28 ++++++++++++-- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 932829bf..8fc999c7 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -1,6 +1,7 @@ //! Local, exact-instance confidential callbacks; never artifact attestation. use super::*; +use crate::module::transport::AdmissionTermination; use crate::message::Header; use crate::transport::memory::MemoryTransport; use crate::{CallContext, Connection, Interface, InterfaceName, MemberName, ObjectPath}; @@ -490,6 +491,7 @@ impl ModuleHost { host: self, transport: None, committed: false, + cause: AdmissionTermination::Failed, }; let path = path.as_ref(); if let Some(parent) = path.parent() { @@ -576,19 +578,26 @@ impl ModuleHost { transport.release_initialization(true); transport }; + // All setup operations above are genuine failures. From this point a + // dropped waiter is cancellation; actual init error/deadline is explicit. + rollback.cause = AdmissionTermination::Canceled; let initialized = transport.ensure_initialized().await; - let ready = if initialized.is_ok() { - tokio::time::timeout(Duration::from_secs(5), transport.wait_ready()) - .await - .is_ok() - && transport.is_ready() - } else { - false + let failure = match initialized { + Err(Error::ConnectionClosed) => Some(AdmissionTermination::Canceled), + Err(_) => Some(AdmissionTermination::Failed), + Ok(()) => match tokio::time::timeout( + Duration::from_secs(5), transport.wait_ready() + ).await { + Err(_) => Some(AdmissionTermination::Failed), + Ok(()) if !transport.is_ready() => Some(AdmissionTermination::Canceled), + Ok(()) => None, + }, }; - if !ready { - transport.commit_terminal(true); + if let Some(cause) = failure { + rollback.cause = cause; + let fault = transport.commit_admission_termination(cause); binding.revoke(); - transport.request_terminal(true); + transport.request_terminal(fault); let _ = transport.close().await; self.inner .loaded @@ -633,15 +642,16 @@ struct LoadRollback<'a> { host: &'a ModuleHost, transport: Option>, committed: bool, + cause: AdmissionTermination, } impl Drop for LoadRollback<'_> { fn drop(&mut self) { if self.committed { return; } - if let Some(transport) = &self.transport { - transport.commit_terminal(true); - } + let fault = self.transport.as_ref().map(|transport| { + transport.commit_admission_termination(self.cause) + }); self.binding.revoke(); self.binding.transport.close_outbound(); self.binding @@ -673,7 +683,7 @@ impl Drop for LoadRollback<'_> { } // A timed-out initializer may still run. Stop is best effort; // revocation/detach removes authority without claiming cessation. - transport.request_terminal(true); + transport.request_terminal(fault.expect("rollback transport cause")); self.host .inner .loaded diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 05ca86f0..dc0c75db 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -501,6 +501,7 @@ async fn rollback_after_attach_releases_the_actual_initialization_observer() { host: &host, transport: Some(transport.clone()), committed: false, + cause: AdmissionTermination::Canceled, }); assert!(!transport.init_started()); assert!(host.list().is_empty()); diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 141acda9..ec838460 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -50,6 +50,14 @@ struct HostContext { } /// The broker-facing side of one loaded module. +/// Admission errors remain faults; cancellation preserves an already committed +/// ordinary stop. Neither classification can clear a real, sticky Fault. +#[derive(Clone, Copy)] +pub(crate) enum AdmissionTermination { + Failed, + Canceled, +} + pub(crate) struct ModuleTransport { self_ref: Weak, module: StdMutex>, @@ -351,10 +359,11 @@ impl ModuleTransport { } } - // Admission rejection is a genuine fault, unlike ordinary shutdown. Never - // fabricate READY or init_started to release a rejected observer. + // Releasing rejected observers is cancellation, which may follow an actual + // ordinary stop. Never fabricate READY or init_started to release them. fn terminate_waiters(&self) { - self.request_terminal(true); + let fault = self.commit_admission_termination(AdmissionTermination::Canceled); + self.request_terminal(fault); self.context.init_notify.notify_waiters(); self.context.ready_notify.notify_waiters(); self.initialization_gate_changed.notify_waiters(); @@ -593,6 +602,19 @@ impl ModuleTransport { self.request_terminal(true); } + pub(crate) fn commit_admission_termination(&self, cause: AdmissionTermination) -> bool { + let fault = match cause { + AdmissionTermination::Failed => true, + AdmissionTermination::Canceled => { + let flags = self.context.native_terminal_state.load(Ordering::Acquire); + flags & crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT == 0 + || flags & crate::module::abi::TB_NATIVE_TERMINAL_FAULT != 0 + } + }; + self.commit_terminal(fault); + fault + } + pub(crate) fn commit_terminal(&self, fault: bool) { self.publish_terminal_flags(fault); self.close_admission(); From 991e39a48c12eecce73ad3dcf5d35ec0d15c5acd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:48:45 +0300 Subject: [PATCH 43/58] chore: files changed crates/tinybus/examples/module_controlled_init.rs,crates/tinybus/src/module/hos Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../examples/module_controlled_init.rs | 6 ++++++ crates/tinybus/src/module/host_callbacks.rs | 21 +++++++++++++++++++ crates/tinybus/src/module/transport.rs | 2 +- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/examples/module_controlled_init.rs b/crates/tinybus/examples/module_controlled_init.rs index a4ee12ba..3602cbb1 100644 --- a/crates/tinybus/examples/module_controlled_init.rs +++ b/crates/tinybus/examples/module_controlled_init.rs @@ -72,12 +72,14 @@ pub unsafe extern "C" fn tinybus_module_init_v1( // The public loader serializes JSON. This declared test fixture accepts the // same finite Controls bytes in a JSON array, without product-side seams. let encoded; + let mut ready = false; let bytes = if copied_config.len() == size_of::() { &copied_config } else { let Ok(value) = serde_json::from_slice::(&copied_config) else { return TB_BAD_ARGUMENT; }; + ready = value.get("test_ready").and_then(serde_json::Value::as_bool).unwrap_or(false); let Some(value) = value.get("test_controls") else { return TB_BAD_ARGUMENT; }; @@ -124,5 +126,9 @@ pub unsafe extern "C" fn tinybus_module_init_v1( reinitialize: Some(reinitialize), }; } + if ready { + // Actual opaque initialization emits Ready; tests never fabricate it. + unsafe { (host.ready)(host.host_ctx); } + } TB_OK } diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 8fc999c7..aa2e2cf0 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -142,6 +142,11 @@ pub struct HostCallbackBinding { connection: Connection, state: Arc, transport: Arc, + // Bounded test observers only; neither hook supplies production authority. + #[cfg(test)] + after_attach: Mutex>>, + #[cfg(test)] + before_load_decision: Mutex>>, } /// Outgoing operations on the binding's private connection. @@ -433,6 +438,10 @@ impl ModuleHost { broker: self.inner.broker.clone(), connection, transport, + #[cfg(test)] + after_attach: Mutex::new(None), + #[cfg(test)] + before_load_decision: Mutex::new(None), state: Arc::new(CallbackState { destination, endpoints, @@ -546,6 +555,11 @@ impl ModuleHost { }, )?; rollback.transport = Some(transport.clone()); + #[cfg(test)] + { + let observer = binding.after_attach.lock().expect("test attach observer").take(); + if let Some(observer) = observer { observer(); } + } let mut loaded = self.inner.loaded.lock().expect("module list lock"); let module = loaded .iter_mut() @@ -595,6 +609,13 @@ impl ModuleHost { }; if let Some(cause) = failure { rollback.cause = cause; + } + #[cfg(test)] + { + let observer = binding.before_load_decision.lock().expect("test decision observer").take(); + if let Some(observer) = observer { observer(); } + } + if let Some(cause) = failure { let fault = transport.commit_admission_termination(cause); binding.revoke(); transport.request_terminal(fault); diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index ec838460..8247b188 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -49,7 +49,6 @@ struct HostContext { inflight: AtomicUsize, } -/// The broker-facing side of one loaded module. /// Admission errors remain faults; cancellation preserves an already committed /// ordinary stop. Neither classification can clear a real, sticky Fault. #[derive(Clone, Copy)] @@ -58,6 +57,7 @@ pub(crate) enum AdmissionTermination { Canceled, } +/// The broker-facing side of one loaded module. pub(crate) struct ModuleTransport { self_ref: Weak, module: StdMutex>, From 38338b556f87ca3deffa88dbdf46f96cf88e2ea4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:50:44 +0300 Subject: [PATCH 44/58] test(module): cover native callback decision race during stop The test-only before_load_decision hook now signals entry and waits for an explicit release instead of running a fire-and-forget closure, so tests can pause the load at the decision point. New ignored tests exercise the ready commit, readiness deadline, and native fault paths against an ordinary stop. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_callbacks.rs | 7 +- .../src/module/host_callbacks_tests.rs | 98 +++++++++++++++++++ 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index aa2e2cf0..63683d50 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -146,7 +146,7 @@ pub struct HostCallbackBinding { #[cfg(test)] after_attach: Mutex>>, #[cfg(test)] - before_load_decision: Mutex>>, + before_load_decision: Mutex, tokio::sync::oneshot::Receiver<()>)>>, } /// Outgoing operations on the binding's private connection. @@ -613,7 +613,10 @@ impl ModuleHost { #[cfg(test)] { let observer = binding.before_load_decision.lock().expect("test decision observer").take(); - if let Some(observer) = observer { observer(); } + if let Some((entered, release)) = observer { + entered.notify_one(); + release.await.expect("test decision observer released"); + } } if let Some(cause) = failure { let fault = transport.commit_admission_termination(cause); diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index dc0c75db..eda5c1c0 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -648,3 +648,101 @@ async fn actual_native_callback_load_drop_after_ordinary_stop_preserves_cause() async fn actual_native_callback_gate_rejection_after_ordinary_stop_preserves_cause() { native_callback_load_after_ordinary_stop(false, true).await; } + +async fn served_native_binding(host: &ModuleHost) -> HostCallbackBinding { + let binding = host.prepare_host_callbacks(vec![endpoint()]).await.unwrap(); + let (entered, _) = tokio::sync::mpsc::channel(1); + binding.serve_at(endpoint().path, BlockingCallback { + entered, + release: Arc::new(tokio::sync::Semaphore::new(0)), + published: Arc::new(AtomicUsize::new(0)), + guard: binding.guard(), + }).await.unwrap(); + binding +} + +async fn native_callback_decision_race(ready: bool, native_fault: bool) { + use super::super::super::transport::tests::{native_fixture, native_fixture_json}; + let (unused, _, _, barrier) = native_fixture(); + let mut config = native_fixture_json(&unused); + config["test_ready"] = serde_json::json!(ready); + drop(unused); + let artifact = std::path::PathBuf::from( + std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); + let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); + let (_dir, artifact) = super::super::tests::staged_module(&artifact, &hash); + let host = Arc::new(ModuleHost::new(Broker::new())); + let binding = Arc::new(served_native_binding(&host).await); + let decision = Arc::new(Notify::new()); + let (resume, release) = tokio::sync::oneshot::channel(); + *binding.before_load_decision.lock().unwrap() = Some((decision.clone(), release)); + let loading_host = host.clone(); + let loading_binding = binding.clone(); + let load = tokio::spawn(async move { + loading_host.load_file_with_host_callbacks(artifact, config, &loading_binding).await + }); + // Keep the paused runtime runnable until actual C entry/publication. This + // prevents virtual time from auto-advancing the initializer's deadline. + while barrier.state.lock().unwrap().0.is_none() { tokio::task::yield_now().await; } + let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + let view = barrier.state.lock().unwrap().0.unwrap(); + barrier.state.lock().unwrap().1 = true; + barrier.changed.notify_all(); + // Poll until native initialization is genuinely published, without an idle + // paused timer deciding the deadline before the native owner can run. + while !transport.init_started() { tokio::task::yield_now().await; } + // init_started precedes actual publication; the observer result is the proof. + let initialized = transport.ensure_initialized(); + tokio::pin!(initialized); + std::future::poll_fn(|cx| { + match std::future::Future::poll(initialized.as_mut(), cx) { + std::task::Poll::Ready(result) => std::task::Poll::Ready(result), + std::task::Poll::Pending => { cx.waker().wake_by_ref(); std::task::Poll::Pending } + } + }).await.unwrap(); + if !ready { + // Real wait_ready deadline, not a fake readiness flag. Let the public + // future install its timer before advancing past the actual bound. + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(6)).await; + } + decision.notified().await; + assert_eq!(transport.is_ready(), ready); + let (finish, receive) = std::sync::mpsc::channel(); + *barrier.finish_gate.lock().unwrap() = Some(receive); + let stopping = transport.clone(); + let stop = tokio::spawn(async move { stopping.stop(Duration::from_secs(2)).await }); + while !barrier.state.lock().unwrap().2 { tokio::task::yield_now().await; } + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 2); + if native_fault { + unsafe { (view.fault)(view.host_ctx, std::ptr::null(), 0); } + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 6); + } + resume.send(()).unwrap(); + // Rollback revokes before awaiting physical close or returning failure. + while !binding.state.lifetime.lock().unwrap().revoked { tokio::task::yield_now().await; } + let held = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; + finish.send(()).unwrap(); + assert!(stop.await.unwrap().is_ok()); + assert!(load.await.unwrap().is_err()); + assert_eq!(held, if ready && !native_fault { 2 } else { 6 }); + assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + if ready && !native_fault { 10 } else { 14 }); + assert_eq!(transport.is_faulted(), !ready || native_fault); +} + +#[tokio::test(start_paused = true)] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_ready_commit_cancellation_preserves_ordinary_cause() { + native_callback_decision_race(true, false).await; +} +#[tokio::test(start_paused = true)] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_readiness_deadline_still_faults_after_ordinary_stop() { + native_callback_decision_race(false, false).await; +} +#[tokio::test(start_paused = true)] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_fault_at_ready_commit_dominates_ordinary_stop() { + native_callback_decision_race(true, true).await; +} From 33164732218ac6dbe428e479e0e08d4d56fee5ef Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:52:20 +0300 Subject: [PATCH 45/58] test(module): avoid hanging on rollback race in host callback test The rollback race test could spin forever waiting for the revoked flag when the load task had already finished, so the wait now also exits once the load future completes. The stop future is likewise driven to completion before being awaited, and its failure message now includes the actual result to make diagnosis easier. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_callbacks_tests.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index eda5c1c0..e80056f1 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -720,10 +720,15 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { } resume.send(()).unwrap(); // Rollback revokes before awaiting physical close or returning failure. - while !binding.state.lifetime.lock().unwrap().revoked { tokio::task::yield_now().await; } + while !binding.state.lifetime.lock().unwrap().revoked && !load.is_finished() { + tokio::task::yield_now().await; + } let held = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; finish.send(()).unwrap(); - assert!(stop.await.unwrap().is_ok()); + // Keep virtual time runnable through the actual opaque return. + while !stop.is_finished() { tokio::task::yield_now().await; } + let stopped = stop.await.unwrap(); + assert!(stopped.is_ok(), "actual stop: {stopped:?}"); assert!(load.await.unwrap().is_err()); assert_eq!(held, if ready && !native_fault { 2 } else { 6 }); assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, From 22b7b54fdde6f1445e0e130a686c4f1cc18f0077 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:52:56 +0300 Subject: [PATCH 46/58] fix(module): reject host calls once transport admission is closed The host callback readiness check now also consults the transport's admission state, so a module whose admission has closed can no longer be treated as ready. This required widening the visibility of admission_closed to the crate. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_callbacks.rs | 1 + crates/tinybus/src/module/transport.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 63683d50..f6772009 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -648,6 +648,7 @@ impl ModuleHost { // turn the returned successful snapshot into Faulted or Stopped. if lifetime.revoked || !lifetime.active + || transport.admission_closed() || !transport.is_ready() || !matches!(info.state, ModuleState::Ready) { diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 8247b188..37e451ee 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -406,7 +406,7 @@ impl ModuleTransport { self.initialization_gate_changed.notify_waiters(); } - fn admission_closed(&self) -> bool { + pub(crate) fn admission_closed(&self) -> bool { self.context.admission_closed.load(Ordering::Acquire) } From a8a8f9ffb935409da5bc3ead20dcad44f7299292 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 12:53:52 +0300 Subject: [PATCH 47/58] test(module): cover native callback setup failure races Add tests that drive the native callback admission path through setup error and unwind while an ordinary stop is in flight, asserting that a genuine setup failure still faults the transport and that the C initializer never entered. The remaining changes are rustfmt reflow and a plan note documenting the Failed versus Canceled cause split. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../examples/module_controlled_init.rs | 9 +- crates/tinybus/src/module/host_callbacks.rs | 39 ++-- .../src/module/host_callbacks_tests.rs | 219 ++++++++++++++---- crates/tinybus/src/module/transport_tests.rs | 10 +- docs/plans/native-terminal-authority.md | 19 ++ 5 files changed, 231 insertions(+), 65 deletions(-) diff --git a/crates/tinybus/examples/module_controlled_init.rs b/crates/tinybus/examples/module_controlled_init.rs index 3602cbb1..014ae391 100644 --- a/crates/tinybus/examples/module_controlled_init.rs +++ b/crates/tinybus/examples/module_controlled_init.rs @@ -79,7 +79,10 @@ pub unsafe extern "C" fn tinybus_module_init_v1( let Ok(value) = serde_json::from_slice::(&copied_config) else { return TB_BAD_ARGUMENT; }; - ready = value.get("test_ready").and_then(serde_json::Value::as_bool).unwrap_or(false); + ready = value + .get("test_ready") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); let Some(value) = value.get("test_controls") else { return TB_BAD_ARGUMENT; }; @@ -128,7 +131,9 @@ pub unsafe extern "C" fn tinybus_module_init_v1( } if ready { // Actual opaque initialization emits Ready; tests never fabricate it. - unsafe { (host.ready)(host.host_ctx); } + unsafe { + (host.ready)(host.host_ctx); + } } TB_OK } diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index f6772009..676fca99 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -1,8 +1,8 @@ //! Local, exact-instance confidential callbacks; never artifact attestation. use super::*; -use crate::module::transport::AdmissionTermination; use crate::message::Header; +use crate::module::transport::AdmissionTermination; use crate::transport::memory::MemoryTransport; use crate::{CallContext, Connection, Interface, InterfaceName, MemberName, ObjectPath}; use async_trait::async_trait; @@ -557,8 +557,14 @@ impl ModuleHost { rollback.transport = Some(transport.clone()); #[cfg(test)] { - let observer = binding.after_attach.lock().expect("test attach observer").take(); - if let Some(observer) = observer { observer(); } + let observer = binding + .after_attach + .lock() + .expect("test attach observer") + .take(); + if let Some(observer) = observer { + observer(); + } } let mut loaded = self.inner.loaded.lock().expect("module list lock"); let module = loaded @@ -599,20 +605,24 @@ impl ModuleHost { let failure = match initialized { Err(Error::ConnectionClosed) => Some(AdmissionTermination::Canceled), Err(_) => Some(AdmissionTermination::Failed), - Ok(()) => match tokio::time::timeout( - Duration::from_secs(5), transport.wait_ready() - ).await { - Err(_) => Some(AdmissionTermination::Failed), - Ok(()) if !transport.is_ready() => Some(AdmissionTermination::Canceled), - Ok(()) => None, - }, + Ok(()) => { + match tokio::time::timeout(Duration::from_secs(5), transport.wait_ready()).await { + Err(_) => Some(AdmissionTermination::Failed), + Ok(()) if !transport.is_ready() => Some(AdmissionTermination::Canceled), + Ok(()) => None, + } + } }; if let Some(cause) = failure { rollback.cause = cause; } #[cfg(test)] { - let observer = binding.before_load_decision.lock().expect("test decision observer").take(); + let observer = binding + .before_load_decision + .lock() + .expect("test decision observer") + .take(); if let Some((entered, release)) = observer { entered.notify_one(); release.await.expect("test decision observer released"); @@ -674,9 +684,10 @@ impl Drop for LoadRollback<'_> { if self.committed { return; } - let fault = self.transport.as_ref().map(|transport| { - transport.commit_admission_termination(self.cause) - }); + let fault = self + .transport + .as_ref() + .map(|transport| transport.commit_admission_termination(self.cause)); self.binding.revoke(); self.binding.transport.close_outbound(); self.binding diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index e80056f1..4e0626fa 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -561,30 +561,44 @@ fn configuration_fingerprints_preserve_json_equality_without_normalizing_strings // Manually polling the public future lets ordinary shutdown commit while the // loader is suspended; no scheduler race decides which cancellation is tested. async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: bool) { - use std::future::Future; use super::super::super::transport::tests::{native_fixture, native_fixture_json}; + use std::future::Future; let (unused, _, _, barrier) = native_fixture(); let config = native_fixture_json(&unused); drop(unused); - let artifact = std::path::PathBuf::from( - std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); + let artifact = + std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); let (_dir, artifact) = super::super::tests::staged_module(&artifact, &hash); let host = ModuleHost::new(Broker::new()); let binding = host.prepare_host_callbacks(vec![endpoint()]).await.unwrap(); let (entered, _) = tokio::sync::mpsc::channel(1); - binding.serve_at(endpoint().path, BlockingCallback { - entered, - release: Arc::new(tokio::sync::Semaphore::new(0)), - published: Arc::new(AtomicUsize::new(0)), - guard: binding.guard(), - }).await.unwrap(); - let mut load = Some(Box::pin(host.load_file_with_host_callbacks(&artifact, config, &binding))); + binding + .serve_at( + endpoint().path, + BlockingCallback { + entered, + release: Arc::new(tokio::sync::Semaphore::new(0)), + published: Arc::new(AtomicUsize::new(0)), + guard: binding.guard(), + }, + ) + .await + .unwrap(); + let mut load = Some(Box::pin( + host.load_file_with_host_callbacks(&artifact, config, &binding), + )); std::future::poll_fn(|cx| { - assert!(load.as_mut().unwrap().as_mut().poll(cx).is_pending(), "loader awaits real C init"); + assert!( + load.as_mut().unwrap().as_mut().poll(cx).is_pending(), + "loader awaits real C init" + ); std::task::Poll::Ready(()) - }).await; - tokio::time::timeout(Duration::from_secs(2), barrier.entered.notified()).await.unwrap(); + }) + .await; + tokio::time::timeout(Duration::from_secs(2), barrier.entered.notified()) + .await + .unwrap(); let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); let view = barrier.state.lock().unwrap().0.unwrap(); barrier.state.lock().unwrap().1 = true; @@ -592,15 +606,24 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: transport.ensure_initialized().await.unwrap(); // Advance the public future to its real withheld-Ready observation. std::future::poll_fn(|cx| { - assert!(load.as_mut().unwrap().as_mut().poll(cx).is_pending(), "fixture withholds Ready"); + assert!( + load.as_mut().unwrap().as_mut().poll(cx).is_pending(), + "fixture withholds Ready" + ); std::task::Poll::Ready(()) - }).await; + }) + .await; let (finish, receive) = std::sync::mpsc::channel(); *barrier.finish_gate.lock().unwrap() = Some(receive); let stopping = transport.clone(); let stop = tokio::spawn(async move { stopping.stop(Duration::from_secs(2)).await }); - tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await.unwrap(); - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 2); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2 + ); let load_result = if reject_gate { transport.release_initialization(false); drop(load.take()); @@ -610,12 +633,11 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: None } else { // Real not-ready branch awaits close, whose native return is held. - std::future::poll_fn(|cx| { - match load.as_mut().unwrap().as_mut().poll(cx) { - std::task::Poll::Pending => std::task::Poll::Ready(None), - std::task::Poll::Ready(result) => std::task::Poll::Ready(Some(result)), - } - }).await + std::future::poll_fn(|cx| match load.as_mut().unwrap().as_mut().poll(cx) { + std::task::Poll::Pending => std::task::Poll::Ready(None), + std::task::Poll::Ready(result) => std::task::Poll::Ready(Some(result)), + }) + .await }; let held_flags = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; let held_fault = transport.is_faulted(); @@ -629,7 +651,10 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: } assert_eq!(held_flags, 2, "ordinary cancellation cannot invent Fault"); assert!(!held_fault); - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 10); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 10 + ); assert!(barrier.state.lock().unwrap().2); } @@ -652,12 +677,18 @@ async fn actual_native_callback_gate_rejection_after_ordinary_stop_preserves_cau async fn served_native_binding(host: &ModuleHost) -> HostCallbackBinding { let binding = host.prepare_host_callbacks(vec![endpoint()]).await.unwrap(); let (entered, _) = tokio::sync::mpsc::channel(1); - binding.serve_at(endpoint().path, BlockingCallback { - entered, - release: Arc::new(tokio::sync::Semaphore::new(0)), - published: Arc::new(AtomicUsize::new(0)), - guard: binding.guard(), - }).await.unwrap(); + binding + .serve_at( + endpoint().path, + BlockingCallback { + entered, + release: Arc::new(tokio::sync::Semaphore::new(0)), + published: Arc::new(AtomicUsize::new(0)), + guard: binding.guard(), + }, + ) + .await + .unwrap(); binding } @@ -667,8 +698,8 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { let mut config = native_fixture_json(&unused); config["test_ready"] = serde_json::json!(ready); drop(unused); - let artifact = std::path::PathBuf::from( - std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); + let artifact = + std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); let (_dir, artifact) = super::super::tests::staged_module(&artifact, &hash); let host = Arc::new(ModuleHost::new(Broker::new())); @@ -679,27 +710,38 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { let loading_host = host.clone(); let loading_binding = binding.clone(); let load = tokio::spawn(async move { - loading_host.load_file_with_host_callbacks(artifact, config, &loading_binding).await + loading_host + .load_file_with_host_callbacks(artifact, config, &loading_binding) + .await }); // Keep the paused runtime runnable until actual C entry/publication. This // prevents virtual time from auto-advancing the initializer's deadline. - while barrier.state.lock().unwrap().0.is_none() { tokio::task::yield_now().await; } + while barrier.state.lock().unwrap().0.is_none() { + tokio::task::yield_now().await; + } let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); let view = barrier.state.lock().unwrap().0.unwrap(); barrier.state.lock().unwrap().1 = true; barrier.changed.notify_all(); // Poll until native initialization is genuinely published, without an idle // paused timer deciding the deadline before the native owner can run. - while !transport.init_started() { tokio::task::yield_now().await; } + while !transport.init_started() { + tokio::task::yield_now().await; + } // init_started precedes actual publication; the observer result is the proof. let initialized = transport.ensure_initialized(); tokio::pin!(initialized); - std::future::poll_fn(|cx| { - match std::future::Future::poll(initialized.as_mut(), cx) { + std::future::poll_fn( + |cx| match std::future::Future::poll(initialized.as_mut(), cx) { std::task::Poll::Ready(result) => std::task::Poll::Ready(result), - std::task::Poll::Pending => { cx.waker().wake_by_ref(); std::task::Poll::Pending } - } - }).await.unwrap(); + std::task::Poll::Pending => { + cx.waker().wake_by_ref(); + std::task::Poll::Pending + } + }, + ) + .await + .unwrap(); if !ready { // Real wait_ready deadline, not a fake readiness flag. Let the public // future install its timer before advancing past the actual bound. @@ -712,11 +754,21 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { *barrier.finish_gate.lock().unwrap() = Some(receive); let stopping = transport.clone(); let stop = tokio::spawn(async move { stopping.stop(Duration::from_secs(2)).await }); - while !barrier.state.lock().unwrap().2 { tokio::task::yield_now().await; } - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 2); + while !barrier.state.lock().unwrap().2 { + tokio::task::yield_now().await; + } + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2 + ); if native_fault { - unsafe { (view.fault)(view.host_ctx, std::ptr::null(), 0); } - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 6); + unsafe { + (view.fault)(view.host_ctx, std::ptr::null(), 0); + } + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 6 + ); } resume.send(()).unwrap(); // Rollback revokes before awaiting physical close or returning failure. @@ -726,13 +778,17 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { let held = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; finish.send(()).unwrap(); // Keep virtual time runnable through the actual opaque return. - while !stop.is_finished() { tokio::task::yield_now().await; } + while !stop.is_finished() { + tokio::task::yield_now().await; + } let stopped = stop.await.unwrap(); assert!(stopped.is_ok(), "actual stop: {stopped:?}"); assert!(load.await.unwrap().is_err()); assert_eq!(held, if ready && !native_fault { 2 } else { 6 }); - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, - if ready && !native_fault { 10 } else { 14 }); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + if ready && !native_fault { 10 } else { 14 } + ); assert_eq!(transport.is_faulted(), !ready || native_fault); } @@ -751,3 +807,72 @@ async fn actual_native_readiness_deadline_still_faults_after_ordinary_stop() { async fn actual_native_fault_at_ready_commit_dominates_ordinary_stop() { native_callback_decision_race(true, true).await; } + +async fn native_callback_setup_failure_race(panic_setup: bool) { + use super::super::super::transport::tests::{native_fixture, native_fixture_json}; + let (unused, _, _, barrier) = native_fixture(); + let config = native_fixture_json(&unused); + drop(unused); + let artifact = + std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); + let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); + let (_dir, artifact) = super::super::tests::staged_module(&artifact, &hash); + let host = Arc::new(ModuleHost::new(Broker::new())); + let binding = Arc::new(served_native_binding(&host).await); + let attached = Arc::new(Notify::new()); + let seen = attached.clone(); + let (resume, release) = std::sync::mpsc::channel(); + *binding.after_attach.lock().unwrap() = Some(Box::new(move || { + seen.notify_one(); + release.recv().unwrap(); + assert!( + !panic_setup, + "actual Rust unwind during host callback setup" + ); + })); + let loading_host = host.clone(); + let loading_binding = binding.clone(); + let load = tokio::spawn(async move { + loading_host + .load_file_with_host_callbacks(artifact, config, &loading_binding) + .await + }); + tokio::time::timeout(Duration::from_secs(2), attached.notified()) + .await + .unwrap(); + let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + // Actual ordinary lifecycle ownership, before the callback initialization + // gate opens. This instance proves C never entered, rather than any join. + let _ = transport.clone().stop(Duration::from_secs(2)).await; + assert!(!transport.is_faulted()); + if !panic_setup { + // Public revocation makes real setup return its concurrently-used error. + binding.revoke(); + } + resume.send(()).unwrap(); + let result = load.await; + if panic_setup { + assert!(result.unwrap_err().is_panic()); + } else { + assert!(result.unwrap().is_err()); + } + assert!( + transport.is_faulted(), + "genuine setup failure upgrades ordinary stop" + ); + assert!( + barrier.state.lock().unwrap().0.is_none(), + "C initializer never entered" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_callback_setup_error_still_faults_after_ordinary_stop() { + native_callback_setup_failure_race(false).await; +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_callback_setup_unwind_still_faults_after_ordinary_stop() { + native_callback_setup_failure_race(true).await; +} diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index a70a5d43..cf1f966e 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -2081,8 +2081,14 @@ pub(crate) fn native_fixture_json(transport: &ModuleTransport) -> serde_json::Va async fn rejected_observer_after_ordinary_stop_preserves_nonfault_cause() { let (transport, view) = ModuleTransport::new("ordinary-cancellation".into(), vec![]); let _ = transport.clone().stop(Duration::from_secs(1)).await; - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 2); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2 + ); transport.release_initialization(false); assert!(!transport.is_faulted()); - assert_eq!(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, 2); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2 + ); } diff --git a/docs/plans/native-terminal-authority.md b/docs/plans/native-terminal-authority.md index 23b88719..ce9d7f1d 100644 --- a/docs/plans/native-terminal-authority.md +++ b/docs/plans/native-terminal-authority.md @@ -195,6 +195,25 @@ before scheduling and never invokes C recursively, joins or waits. Rollback and Host Drop commit all flags before revocation, release router/loaded locks and route through retained cleanup. No lifecycle-bypassing synchronous Drop cleanup. +Callback-load admission carries an explicit Failed versus Canceled cause. +Actual setup Err/unwind, initialization error and actual readiness deadline are +Failed even when ordinary stop races them. After successful setup, waiter Drop, +admission-gate rejection and terminal-caused not-ready/final-commit cancellation +preserve already committed ordinary StopIntent without adding Fault. Live +abandonment still faults; real Fault remains sticky. Apply the selected cause +before revocation and to retained cleanup, including nested observer release. +The final load liveness check rejects actual admission closure even while the +legacy Ready bit remains set and opaque shutdown has not returned. + +Bounded cfg(test) admission/decision observers and fixture JSON Controls encode +test barriers only, never production authority. The declared native fixture emits +actual Ready during opaque init when requested. Tests hold actual shutdown return +after the independent worker joins, prove ordinary cancellation flags2 and eventual +TB_OK flags10, and distinguish genuine deadline/actual host_fault flags6/14. +A host setup unwind test proves Rust rollback classification, not module catch-time +panic containment or Task4 hooks. Setup rejection before gate release proves C never +entered and makes no physical-join claim. + Normal stop remains deadline-bound while obtaining the actual owned lifecycle guard. Before acquisition it owns no ordinary intent, job or stop_task_spawned marker; cancellation/deadline is NotStarted and remains live absent real fault. From c1f59e4d6500813d55d17ec7a02561a34a9d3929 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:09:15 +0300 Subject: [PATCH 48/58] docs(plans): expand task 4 with accepted module-local lifecycle protocol Rework the native terminal authority plan so task 4 records the accepted module-local lifecycle protocol instead of the earlier proposal text. The document now specifies the public API, request and finish ownership, startup resource transfer, shutdown classification, and the fixed admission cap, and adds the binding panic-payload ordering and catch-time observer corrections that the writer must follow. Auto-committed-on: dragonfly Co-authored-by: Medulla --- docs/plans/native-terminal-authority.md | 353 +++++++++++++++++++++--- 1 file changed, 314 insertions(+), 39 deletions(-) diff --git a/docs/plans/native-terminal-authority.md b/docs/plans/native-terminal-authority.md index ce9d7f1d..59fd4a9a 100644 --- a/docs/plans/native-terminal-authority.md +++ b/docs/plans/native-terminal-authority.md @@ -57,7 +57,7 @@ New crates/tinybus-module/src/terminal.rs and terminal_tests.rs; lib.rs reexport enum TerminalCompletion { PhysicallyStopped, Uncertain } struct NativeInstanceLiveness { inner: Arc } struct RuntimeHooks { - request: Box, + request: Arc, finish: Box TerminalCompletion + Send>, } @@ -247,44 +247,319 @@ actual reinit fault/query denial before return, ordinary callback lease retentio Host Drop/rollback tombstones, instance isolation and one real join. Keep every original #57 callback drain, physical lock, retry and original marker assertion. -## Task 4: module-local hooks through runtime and fault paths - -Files: new terminal.rs/terminal_tests.rs/lib_terminal_tests.rs in tinybus-module; -lib.rs, lib_setup_factory_tests.rs, lib_lifecycle_tests.rs. -Declare sibling test modules, no inline tests. - -Red tests unavailable query rejects before factory; two instances isolated; -retained view after runtime shutdown; host fault precedes local request; -request/finish once; poisoned hook storage; finish panic stays uncertain; -terminal reinit never reruns setup; retained service refs cannot suppress cleanup. - -Implement API above. New private start_terminal_reconfigurable_module_factory_mode -and doc-hidden linked/unlinked entrypoints serve macro arms. Existing entrypoints -retain old behavior. start_module_runtime takes optional Arc; -transport, RuntimeState and setup monitor share it, existing callers pass None. -shutdown requests Stop before callback disposal, releases reinitializer mutex, -then runs finish synchronously on retained physical host cleanup worker before -runtime.shutdown_timeout. Indefinite join outlives deadline without TB_OK. -Uncertain/panic finish returns content-free failure, never physical completion. - -Task 4 remains separate, unimplemented scope. Its startup guard owns returned -hooks until runtime/output publication; factory Err/panic before return owns -partial disposal itself. Request callback is an independent Arc and finish a -separate take-once slot. Publish intent, claim bounded request outside locks, -signal request completion even on panic; physical finish waits request-done -off runtime, callback-gate and reinitializer locks. Pre-registration fault -persists and invokes bounded request once at registration. Ordinary -terminal-caused setup cancellation must remain non-Fault, while genuine -Err/panic still upgrades Fault even racing ordinary stop. - -The accepted host prerequisite does not establish catch-time fault publication, -module-local worker/resource finish, fixed retained job bounds or confidential -payload/backing last-Drop. Task 4's own accepted specification must prove those -contracts at the actual CatchUnwind boundary before payload destruction; no -enqueue/send-success inference, extra panic task or global panic hook join. -A caught registry-free callback boundary must publish fault/request before -returning its error. New physical-success evidence remains distinct from legacy -Stopped and receiver backing destruction. +## Task 4: accepted module-local lifecycle protocol + +Baseline for this task: a8a8f9ffb935409da5bc3ead20dcad44f7299292. +The corrections below supersede conflicting earlier proposal text. + +# Task 4 proposed concrete module ownership + +Evidence-only proposal for root preflight. No source/build/pin/ref/commit mutation. +Uses accepted fe080 ABI tail and Task 2/3 host ownership, not another ABI. +Source observations: lib.rs RuntimeState/start_module_runtime/factory-mode/shutdown; +connection.rs::handle_call invokes caught panic handler after dispatch unwinds and +before error enqueue; registry locks are released before user invocation. Existing +transport send faults only after successful send; this cannot remain primary fault +notification for opted-in modules. Existing entrypoints retain their behavior. + +## Proposed public module-local API + +New terminal.rs, terminal_tests.rs and lib_terminal_tests.rs; lib.rs reexports: + + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub enum TerminalReason { Stop, HostClosed, Fault, SetupFailed } + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub enum TerminalCompletion { PhysicallyStopped, Uncertain } + #[derive(Clone)] + pub struct NativeInstanceLiveness { inner: Arc } + pub struct RuntimeHooks { + request: Arc, + finish: Box TerminalCompletion + Send>, + } + impl RuntimeHooks { + pub fn new(request: R, finish: F) -> Self + where R: Fn(TerminalReason) + Send + Sync + 'static, + F: FnOnce() -> TerminalCompletion + Send + 'static; + } + impl NativeInstanceLiveness { + pub fn is_live(&self) -> bool; + pub fn request_fault(&self); + } + +All fields private; no raw pointer, constructor, serde or Connection enrollment. +New terminal_setup_factory macro/linked arm takes factory(view) -> +tinybus::Result<(F, RuntimeHooks)>, F: Fn(Connection, C)->Fut + Send+Sync+'static, +Fut: Future>+Send+'static. Same Arc serves setup/reinit. +New doc-hidden start_terminal_reconfigurable_module_with_factory and linked +equivalent own C argument validation; no change to old setup/setup_factory APIs. + +Normalize/copy HostCalls only through trusted unsafe module-init contract. Check +native_terminal_state exists and returns exactly TB_NATIVE_TERMINAL_LIVE before +factory evaluation; unavailable/terminal returns TB_CLOSED with factory untouched. +Do not infer lifetime from broker_routing or sender JSON. Strong callback lifetime +is already part of accepted unsafe ABI contract; actual HostContext persists. +Every is_live samples local terminal, actual callback, local terminal again. Unknown +flags fail closed, mark observed HostClosed locally; valid Fault upgrades Fault. +StopIntent observes Stop. Observation does not claim physical fence/join/backing Drop. + +Factory constructs controller/resources but launches worker only after returned +hooks are registered, during setup. Before return, factory owns partial Err/panic +cleanup. A factory calling view.request_fault before returning persists terminal +intent; after hooks register replay one bounded request and deny setup. Factory +Err/panic with no hooks cannot give SDK a join handle it never received. + +## Request protocol, cause precedence and non-recursion + +TerminalState holds copied HostCalls; monotonic terminal/fault atomics; mutex +ControlState with request Arc Option, separate finish Option, RequestState +(Unregistered/Idle/Running/Done/Panicked), FinishState +(Idle/Anchored/Running/Complete/Uncertain), startup-publication state; Condvar. +All locks recover poison to terminal/uncertain rather than second panic. Atomic +intent is committed before taking these locks or invoking arbitrary code. + +Cause precedence: real Fault dominates Stop/HostClosed. SetupFailed is diagnostic +fault cause, also sets fault flag; never downgrade fault after once request. Stop +and HostClosed both normal terminal, no HostCalls::fault. Preserve first real fault +cause; reason passed to already-invoked request is immutable, persisted reason may +upgrade later. Request contract is identical atomic admission closure for every +cause, so fault escalation never needs a second request invocation. + +request_fault calls HostCalls::fault FIRST, then records local fault and runs +request_if_registered. Host callback schedules only, never awaits/reenters module +hooks. Host intent closes independent worker phases even if module request cannot +yet run. request_if_registered clones request Arc and marks Running under lock, +releases lock, catches callback unwind, reacquires lock to publish Done/Panicked +and notify. If another caller sees Running it returns without waiting; only +physical finish waits request-done. Request panic commits fault/Uncertain and +notifies BEFORE invoking host.fault for escalation; no retry or recursive request. +Pre-registration request stays pending; registration installs both callbacks under +one lock then invokes request_if_registered outside it. No missed early fault. + +Request MUST be bounded atomic admission-close/cancel signal: no I/O/join/reentrant +SDK API, no waiting physical phase gate or Registry/reinitializer mutex. The SDK +cannot forcibly bound trusted callback code; state docs/tests establish contract. +Already granted Publish may finish; actual next-phase query denies immediately. +Physical gate fence plus actual worker join occurs only in finish. Arbitrary worker +catch invokes request_fault after unwinding out of physical gate, not panic hook. + +## One retained physical finish owner including no RuntimeState paths + +Use private module-global anchor collection of Arc, one entry per +opted-in instance while controller ownership needs retention. Insert BEFORE +factory evaluation/transfer can create SDK-owned hooks and before any spawn. +Each instance ControlState has ONE optional Arc; job owns terminal +state plus startup/runtime cleanup resources. Collection owns state independently +of rejected spawn closure, setup future, service refs and output vtable. No public +fake constructor; no extra HostContext or per-view/per-query allocation. This is +explicit module-local controller/anchor bookkeeping, not an ABI lifetime object. +The collection is inspectable/testable and clears completed entries; uncertain +entries deliberately remain. No global boundedness claim; finite total admission +would require separately approved cap. Do not substitute an untracked Box leak +or self-cycle for this external retained owner. + +finish scheduling under ControlState: install job then mark Anchored exactly once; +clone job into Builder::spawn. Spawn Err drops only clone, retains original anchor, +marks Uncertain, retains resources and closes intent. No implicit retry/claim reset. +Spawner injectable privately. Actual worker waits request-done AND startup ownership +publication before taking finish. Neither request caller nor runtime worker waits. +Startup locks/ControlState/phase/reinitializer locks all released before finish. +Take FnOnce once under lock, then run outside lock/catch_unwind. Physical completion +requires TerminalCompletion::PhysicallyStopped; failure/panic stays Uncertain and +retains job/state/resources conservatively. Cleanup thread never calls host shutdown. + +This additional module cleanup thread exists once per terminal instance, not once +per error; request itself never spawns. C shutdown observes/schedules same job then +waits completion on host's physical blocking worker. A fault during stop cannot +claim second finish. Module startup failure has identical finish owner even when +no valid vtable was ever published and host cannot call shutdown. + +## Startup ownership guard: exact transfer and failure paths + +Create StartupGuard outside catch_unwind after argument/config/capability validation +and TerminalState/anchor creation. Factory Err/panic before returning hooks marks +terminal, removes empty anchor only when SDK owns no controller/resources; factory +partial disposal remains its responsibility. On successful factory return register +hooks immediately under guard, before tracing subscriber/runtime creation. + +Guard owns shared StartupResources (runtime Option, reinitializer slot Option, +connection/inbound resources as created). Every resource is attached to this owned +structure immediately; no fallible gap leaves runtime/closure on unwinding stack +with accidental blocking Drop. Only cleanup worker takes them after publication. + +Refactor opted-in runtime builder path, preserving legacy calls: + +1. tracing set_global_default failure after hook transfer: guard records SetupFailed, + host fault, bounded request, schedules anchored finish; returns TB_CLOSED. +2. runtime Builder failure: same, runtime absent. No direct finish on init callback. +3. panic in reinitializer construction, task spawn, RuntimeState construction or + after ownership transfer: outer catch retains guard/StartupResources, marks fault, + schedules same finish and returns TB_PANICKED. Runtime Drop occurs on cleanup + thread after finish, not unwinding ABI/runtime worker. +4. RuntimeState Box created: guard still owns resources until output publication. + write_module_vtable only uses prevalidated writable capacity. Once complete, + mark PublishedRuntime and disarm guard, keeping anchor until actual finish. +5. Panic before output publication: no TB_OK/usable output promised; guard publishes + StartupFailed/resources to job, no host cleanup assumption. After successful + publication, RuntimeState/shutdown own normal routing and shared finish job. + +Guard Drop never joins/blocks: it marks terminal and transfers to already anchored +resources/job, signals publication; catch path performs bounded request/schedule +outside unwind. Guard is fallback when unwinding, not an arbitrary-callback panic +hook. Startup publication wait prevents fault-triggered finish from overtaking +registration/runtime transfer. No await while holding startup locks. + +## Runtime shutdown and setup result classification + +RuntimeState adds optional terminal state; old entrypoints None unchanged. Opted-in +shutdown first samples actual host flags to select Fault/SetupFailed precedence; +if still Live and actual shutdown entered, commit local Stop. Request once. +Take reinitializer under its mutex, release mutex, drop captured callback outside +lock. Close inbound. Publish cleanup resources; schedule same finish owner. +Wait request/finish completion on host physical shutdown worker, holding no module +runtime/reinitializer/startup/phase locks. Actual Host lifecycle remains serialized +around opaque C callback, as accepted Task 2/3. Runtime.shutdown_timeout runs ONLY +after successful join result, on physical cleanup thread or host worker after +resources ownership is taken once. Uncertain finish returns TB_PANICKED, never TB_OK; +resources remain retained if releasing could discard actual uncertain ownership. + +Setup monitor: actual Ok(Err) or JoinError::is_panic upgrades SetupFailed/Fault even +if ordinary stop races. JoinError::is_cancelled while persisted Stop/HostClosed +shares normal cleanup WITHOUT host.fault; successful setup's intentional pending +task canceled by ordinary runtime shutdown is covered. Unexpected cancellation +while Live is SetupFailed. Fault host flags always dominate local normal cause. +Record cancellation decision before runtime teardown; monitor cannot be sole +controller owner because shutdown can abort it. Setup monitor schedules, never +awaits physical finish; failed callback disposal on physical worker releases its +mutex before joining and never blocks runtime needed by in-flight setup/reinit. +Setup/reinit reject terminal before start and before activation. Real setup +Err/panic must be reported at source catch/result boundary before publication of +normal shutdown success; do not rely solely on a monitor canceled during teardown. + +Caught method handler installed with __set_panic_handler invokes request_fault +before content-free error construction/enqueue. Existing handle_call clones/drops +handler lock; dispatch already unwound Registry locks. Remove opted-in reliance +on successful ModuleTransport::send panic-error handling; old behavior unchanged. +Global panic hook remains location-only, no terminal hook/join/reentrant SDK code. + +## Host init output interaction + +No TB_OK from startup failure, and no host invocation of misleading rejected output. +Host PublishedRejected/NoCallbackRequired concerns usable host callback access; +it DOES NOT certify no module startup controller/backing remains. The module's +independent anchor owns failed-startup finish. If finish pending/Uncertain, host +native query stays terminal without PhysicallyStopped. Successful startup finish +with no valid returned vtable still cannot manufacture a host physical-success bit; +only host accepted-vtable shutdown success sets that ABI bit. No extra ABI needed. +Malformed foreign module output/launch without returned hooks remains unjoinable +uncertainty; this safe API never promises to clean resources it never received. + +## Scoped red/green tests and actual compiled fixtures + +Sibling tests pin unavailable query before factory, retained copied view after +runtime shutdown, independent instance/factory isolation, unknown flags deny; +fault before registration -> exactly one request after registration/no setup; +request races finish with barriers -> finish starts only after request done; +request panic -> host scheduling once/no recursion/no second request; +poison recovery -> terminal uncertain/no double panic; +spawn failure -> real controller/resource Drop stays retained after no-vtable +startup error, Host Drop and runtime teardown; finish FnOnce never repeats. + +Inject tracing-registration conflict, runtime-build error, panic after hook transfer, +panic after runtime creation/before output publication; observe actual finish/join +off init/runtime workers and correct uncertainty on spawn failure. Test normal +initial setup cancel and successful pending-task cancel; race real Err/panic with +stop -> Fault preserved; block outbox/host send -> caught handler still closes +native authority before error enqueue. Reinit terminal never launches/replaces. + +Actual compiled module_terminal fixture registers std worker/controller, holds +Publish lease and opaque reinit/init, exposes init-bound test C release controls. +Host fault during opaque hold denies next worker query before cleanup can enter; +old lease release enables actual fence/join. Deadline/drop retains join, hook +request bounded while gate held, actual worker panic after unwind, startup failure +with no valid vtable still finishes through module anchor, post-HostDrop query +tombstone safe. Real old-prefix host rejects only opted-in factory; legacy works. +Use barriers/events, no sleeps; compiled controls never become production authority. + +No tests infer confidential last-Drop/refund from callback/future/join counters. +No codec/crypto/retirement/global load cap addition. Root preflight must approve +module-global anchor bookkeeping and StartupResources ownership transfer before +Task 4 writer implements; they are explicit proposed internal ownership, not +pretended current APIs or an additional C lifetime contract. + +# Task 4 module-local lifecycle preflight + +Reviewed accepted-task4-proposed.md against unchanged tinybus-module/src/lib.rs and connection.rs; Task 1 additive ABI now exists. Read-only: only this ignored report written. No source/build/ref/pin mutations. No extra C ABI or allocator/lifetime handoff is needed. + +## Important findings and accepted concrete resolutions + +**New global collection needs finite admission.** Proposal:93–104 adds a collection retaining failed-startup/Uncertain instances indefinitely. Existing RuntimeState leakage does not account for this: tracing/build failures return before its allocation (lib.rs:570–579,608–615 versus 684–691). A growing Vec plus a global-bound disclaimer is insufficient. + +Root accepts a fixed **256** private terminal-instance permit/anchor array reserved before factory evaluation/SDK hook ownership. Exhaustion returns TB_CLOSED without invoking factory. Each state owns its SlotLease; external slot owns Arc, state owns one Arc, Job holds Weak, and a submitted worker captures strong State+Job. No state/job strong self-cycle. Successful actual finish plus all SDK-owned resource teardown permits anchor/job removal under bookkeeping locks, with destructors outside locks. The slot permit is released only at actual State last-Drop, so retained old liveness views keep their slot occupied and cannot cause unbounded retired-state accumulation through slot reuse. Uncertain/spawn failure retains slot/state/job/resources permanently; no reset/retry/refund. Factory failure before SDK resource transfer can remove its empty anchor; an escaped retained view still keeps the permit occupied until actual Drop. Fixed metadata does not bound arbitrary trusted closure/resource bytes or all mapped SDK images/host contexts. This is an admission cap for the new opt-in bookkeeping, not a broad legacy loader rewrite. Test capacity exhaustion, retained-view permit retention, actual success retirement and uncertain permanent occupancy. + +**Startup attachment must include all owned captures, not just Runtime.** Proposal:130–133 mentions runtime/slot/connection/inbound resources but factory returns F before runtime construction. Current factory path builds initial_setup and reinitialize_factory on the ABI stack (lib.rs:927–932,970–971); tracing failure can drop those captures before RuntimeState exists. Root accepts immediate attachment of returned Arc, parsed config/initial setup closure, reinitializer factory/slot, transport/connection/inbound resources and actual task handles to preexisting StartupResources before subsequent fallible/unwind boundaries. Successful returned Runtime must be attached via poison-recovering, non-user-code transfer immediately after Builder success at lib.rs:608–615, before channels/reinitializer/task construction at 616–631. RuntimeState and FinishJob share this resource owner rather than transferring exclusive Runtime to a fallible local stack. No Runtime Drop on ABI/runtime worker; startup failure cleanup runs on anchored physical worker. Arbitrary factory partial cleanup/destructors before hook transfer remain trusted factory obligations; the SDK cannot promise to control resources it never received. Tests inject failure after every transfer boundary and observe actual destructor thread and retention, not merely flags. + +**Request failure must not spend physical finish.** Proposal:79–80 says request panic commits Uncertain, but FinishState Uncertain must not prevent the still-uninvoked finish callback from executing. Use orthogonal sticky_uncertainty plus finish invocation state. Request panic/poison publishes fault, marks RequestState Panicked/done and sticky uncertainty, then schedules the same still-eligible physical finish. Never repeat request or consume finish merely because uncertainty was recorded. After actual finish, uncertainty stays visible; no TB_OK/permit refund follows a failed request merely because join succeeded. A failed spawn remains unsubmitted and retained with no automatic retry. Tests cover request panic before job submission and while finish waits request-done. + +## Accepted setup/reinit protocol: settle execution before physical completion + +Current source keeps the successful initial setup task pending forever (lib.rs:646–655), and its monitor owns the exclusive JoinHandle (664–665). Err/panic is reported only after callback disposal and host fault at 669–680. RuntimeState shutdown disposes callback under the reinitializer mutex then destroys runtime (375–400); C reinit holds that mutex while block_on polls setup (404–417,955–960). A new job that waits this mutex or initial setup completion without first arranging cancellation can wait forever for work whose completion needs terminal cleanup. Joining a worker before an initial/reinit setup future is settled also lets that future launch work after the claimed join. + +Use this concrete opted-in-only protocol: + +1. Keep initial setup invocation and its returned future in a real task. Catch both synchronous F(connection, config) construction and asynchronous future polling inline, not only the later monitor. Publish actual Err/panic -> Fault/SetupFailed immediately at that result boundary, before readiness or any task-completion notification. Record cancellation separately, preserving already-committed Stop/HostClosed. Existing non-opted-in behavior is unchanged. +2. Initial task returns after its actual setup result; do not park it forever. The already-persisted connection_slot owns the serving Connection (lib.rs:648–649); StartupResources owns that slot for runtime lifetime. FinishJob/StartupResources owns the task's sole JoinHandle. A monitor may observe shared published outcome, but must not consume the only handle required for real cancellation/completion proof. +3. On terminal request, keep Runtime alive, abort an unfinished initial task, and await its actual JoinHandle on the physical cleanup thread (via retained runtime handle). This proves future cancellation/drop has actually happened. Classify is_cancelled caused by Stop/HostClosed as normal; genuine source-observed Err/panic always upgrades Fault. Do not await that join on a runtime worker, under reinitializer/startup/control/gate locks, or after shutting down its runtime. +4. Opted-in reinit drives its setup invocation under terminal-aware cancellation (a terminal notification/predicate select with inline result/panic recording; poll the result branch first when result and terminal are both ready, so an actual ready Err is not lost). This lets a persisted request close the running reinit future while C still owns its mutex; it then returns a normal terminal code and releases that mutex. A real Err/panic observed while polling still faults before returning. No new setup invocation starts after terminal, and readiness/owner publication rechecks terminal. Cleanup can then take the reinitializer slot, release its mutex and drop captured callbacks outside it. Waiting a reinit that ignores cancellation synchronously retains pending ownership; it is not quiescence. +5. After actual initial-setup settlement and reinitializer disposal, run the one finish callback, keeping Runtime services available for the actual controller worker's fence/join. The request callback must already have performed bounded cancel/wake; finish must not rely on an initial/reinit setup future that has been canceled, nor await host shutdown. This dependency contract belongs in public callback docs. Synchronous never-returning user setup, blocking destructor, nonreturning reinit, or worker dependency that cannot settle remains retained pending/Uncertain with no successful completion. Do not fake a timeout as join. +6. After finish success, teardown SDK-owned runtime/connection/callback/task resources exactly once off locks on the physical worker. Only then publish completion and retire the anchor if there is no sticky uncertainty. Recheck local/actual host fault when answering C shutdown; normal Stop cannot erase genuine Fault/SetupFailed arriving during setup or teardown. Fault/uncertain never earns the host physical-success bit. Successful no-vtable startup cleanup proves only module-local cleanup: host query stays terminal and cannot fabricate accepted-vtable shutdown evidence. + +This avoids the concrete initial-task/reinitializer/runtime dependency cycle without adding a C field. It changes only the opt-in internal setup task lifetime. If implementation chooses another protocol, it must prove the same actual future settlement before finish/physical success and retain a sole actual JoinHandle or equally strong completion owner; a setup monitor canceled by runtime teardown is insufficient. + +## Boundaries already supported by source + +Caught per-connection panic handler runs after registry-free dispatch catches unwind and before error enqueue: connection.rs:1441–1453; object-tree lock ends at 1584–1587 before invocation. Transport writing is a separate writer task (1355–1365). Put host fault first in that handler, before error construction/send. ModuleTransport::send's existing TB_OK-only fault at lib.rs:270–285 cannot be relied on for the opted-in path. Request callback is separate Arc; finish FnOnce is separately taken, callback storage locks released before arbitrary code, request completion notified even on panic. Global panic hook remains location-only. A factory pre-fault is persisted before registration, request replayed once after registration, and setup denied; finish waits the startup-publication predicate to avoid overtaking resource transfer. + +Task 1's actual host query field docs (abi.rs:184–202) cover copied callback/context and post-runtime worker lifetime. Module-local anchors never export Rust traits/Arc or allocations through C and add no new host context. PhysicalStopped means actual supported shutdown/join evidence, never receiver-last-Drop, codec retirement or foreign allocation destruction. + +## Scoped verdict + +**Task 4 IMPLEMENTABLE** with this report's accepted corrections governing the writer. Root has accepted fixed256 actual-last-State-Drop admission, job Weak ownership, immediate StartupResources transfer, the initial-task real JoinHandle protocol, result-first terminal-aware reinit cancellation, actual source Err/panic publication, off-runtime request/initial-task settlement/finish/resource-teardown ordering, and final local/native Fault recheck. This is a concrete module-local protocol with no remaining source contradiction or extra ABI requirement. + +The required orthogonal matrix is: Request Panicked/done -> sticky_uncertainty, while uninvoked Finish remains eligible for its same once job; actual finish return is recorded independently. SpawnFailed retains anchored unsubmitted owner, sticky uncertainty, no retry. Finish panic/Uncertain retains remaining anchored resources and permit. Only actual successful finish AND all SDK-owned teardown with no sticky uncertainty can remove anchor/job; permit itself remains until State last-Drop. Cause Fault is independent of whether resources were actually joined/released and never disappears from terminal state. A successfully cleaned no-vtable startup instance cannot make the host physical-success bit appear. NoCallbackRequired for no returned callable host vtable must never imply that a registered module controller does not exist. + +Arbitrary synchronous nonreturning setup/Drop, blocked opaque reinit or worker dependency can retain pending ownership indefinitely. Callback docs must require bounded cancel/wake request and finish's independent gate/fence/join responsibilities; there is no forced trusted-code cancellation, no arbitrary captured-byte bound, and no confidential receiver destruction proof. Existing legacy entrypoints remain unchanged. Fresh actual compiled failure/cancellation/join/admission proofs remain implementation acceptance gates, not preflight claims. This verdict does not change the active host Task2/3 writer's scope. + + +## Binding panic-payload order addendum + +Root accepts request panic/poison sticky uncertainty independently from the once finish state. The writer must also prevent primary panic payload Drop before the actual fault fence. `catch_unwind(...).is_err()`, `Err(_)`, or temporary matches can dispose the caught `Box` before later fault notification. Use explicit `Err(payload)` ownership in every new catch. Do not format/downcast payloads or override the global hook. + +For once-only module request/finish and the new setup/reinit catches, use fixed private per-instance first-payload cells inside admitted TerminalState, rather than an unbounded vector or a local value whose destructor runs at return. Exact new-source categories: request (once), finish (once), initial setup (first construction/poll panic), reinit (first panic, terminal then denies further invocation), factory (first/only call), startup wrapper (once), and cleanup/SDK teardown wrapper (once). **At most seven first-payload Box owners per admitted instance**, separately named cells, with no arbitrary payload-byte bound. These cells can be omitted where a boundary is already captured by another category, but must not alias categories that can both fail. Initial/reinit construction and polling share their respective first-fault cell; once terminal they cannot be polled/retried to accumulate new primary payloads. This is not a bound on arbitrary panics in unrelated user tasks or legacy/concurrent method calls. + +On primary capture: retain ownership explicitly; publish actual host/native Fault plus local terminal/sticky uncertainty without invoking any foreign destructor, then install the payload in its still-empty fixed cell under poison-recovered control storage. For request panic, set Request Panicked/done and notify completion outside held locks BEFORE any potential payload disposal; finish remains eligible once. For setup/reinit publish the real fault before task/C result completion; for finish/teardown record uncertain completion and notify before return. Atomic native/local intent must not depend on acquiring payload storage. No replacement/take-and-drop of a populated cell on a retry: terminal/once admission forbids that retry. + +The minimal safe disposal policy is **do not dispose captured primary payloads in this slice**. Sticky-fault state remains externally anchored with its permit/controller/resources; the fixed cells retain payloads for the process lifetime. This is accountable finite owner-count retention under fixed256 admitted states, not an untracked per-panic leak, a zeroization guarantee, or a total foreign-byte bound. The same real physical finish is still attempted once after request completion; sticky uncertainty prevents TB_OK, success-bit publication or permit refund even if worker join succeeds. If implementation instead chooses guarded disposal, its secondary-payload owner must also have an explicitly reserved fixed cell and fault/done notifications must already have happened; arbitrary recursive destructor panic/block/abort cannot be promised recoverable. Do not add a generic panic framework or a resubmission loop. + +Captured payload ownership does not make arbitrary user future/capture destruction safe: synchronous blocking Drop or secondary unwind during cancellation can leave the actual task JoinHandle pending or fail. Keep the accepted runtime/job/permit owner alive and do not claim completion. A destructor may abort the process; no generic abort/foreign allocator recovery is promised. Initial-task actual join and reinit exclusion remain required before physical completion; runtime timeout/future cancellation is not their substitute. + +### Actual method-panic source gap requiring a narrow opt-in catch observer + +Source review found a stronger prerequisite than the prior report's handler-only wording. connection.rs:1554–1557 catches a user future poll with `Err(_) => Poll::Ready(Err(()))`. Its foreign payload can be dropped *inside that catch match*, before `handle_call` invokes the registered panic handler at 1451–1453. Therefore installing fault-first logic only in `__set_panic_handler` is too late for a payload that blocks/panics on Drop. + +Add a narrowly private opted-in catch-time terminal observer to the Connection/CatchUnwind path (Rust-local only, no C ABI). Clone its callback outside handler/registry locks. On `Err(payload)` invoke the observer to commit actual host/local Fault, bounded request completion and sticky uncertainty BEFORE payload is dropped; then produce the content-free method error. Existing legacy observer-absent behavior can remain. If observer-triggered request fails, its own fixed request cell/completion protocol applies. Ordinary method dispatch and global panic hooks must not acquire controller joins/gates. + +Do not pretend all concurrent method panic payloads fit the seven once-boundary cells: method calls have no such count bound. The narrow observer guarantees fault-before-disposal for those payloads; it does not promise to retain/recover every arbitrary method destructor. After observer commits sticky fault, a blocking/secondary-panicking method payload can prevent later error delivery/task settlement, but native authority is already denied and the actual external owner/permit remains retained. No physical success/refund follows that method panic. Guarding and repeatedly retaining arbitrary method payloads would require separately bounded admission and is outside this slice. + +**Addendum verdict:** module-local once-boundary retention is implementable within fixed admitted state and without new ABI. Full Task4 fault-before-payloadDrop proof additionally requires the narrow opted-in Connection catch-time observer above; handler-only implementation does not pass. This is a precise source-backed correction, not a promise about all legacy connection panics. Add a panic_any payload with blocking Drop barrier: verify independent native query denies and request-done is observable before the barrier is released; repeat request/finish/initial/reinit boundary cases and assert the actual once finish attempt/retained permit rather than flags alone. + + +Root accepts the narrow opt-in Connection catch-time prerequisite. Its observer is installed only by the actual terminal factory/runtime path, closes private TerminalState plus actual HostCalls::fault first, and catches request panic through the fixed request-cell/done protocol before the primary method payload can drop. No public fake liveness constructor, payload-selected observer authority or connection-wide growing collection is introduced. New setup construction/poll/reinit catches use the same terminal observer ordering. Preserve existing content-free MethodFailed wire schema and legacy observer-absent behavior. + +Runtime resource qualification is binding: `runtime.shutdown_timeout` (current lib.rs:393) can return with opaque blocking tasks/Drop still alive. Its return is not proof those owners were destroyed. Remove successful anchors/release accounting only with actual teardown evidence. Any surviving SDK task/transport/observer must retain its actual TerminalState clone, keeping the State-owned permit occupied through actual task/clone Drop; if that ownership cannot be established, keep the external Uncertain anchor. Never interpret timeout, async Connection closure or callback/future drain as actual method-task Drop, independent-worker join or receiver last-Drop. The controller's proved worker join is a separate fact from SDK-owned resource destruction. User-created detached tasks not received/tracked by SDK remain the trusted controller's lifecycle obligation; no generic task-destruction guarantee follows. + +Exact source touchpoints/tests for this correction: `tinybus/src/connection.rs::CatchUnwind::poll` (1554–1557) explicit payload capture/observer before Drop, `handle_call` (1441–1453) observer cloning outside lock/content-free error after fault, and `tinybus-module/src/lib.rs` caught setup/runtime wrappers plus new terminal.rs request/finish first-payload cells. Controlled `panic_any` payload records Drop entry and may block there: assert actual native Fault observed before Drop entry, actual Request done notification before waiting finish, and outgoing blocked/failed send cannot postpone the fence. Request/finish primary payload is retained in its fixed cell and never reaches Drop in this slice; method payload may drop only after the observer fence. Check actual original once finish attempt, sticky uncertainty and held permit while task/destructor/native join remains pending. Do not infer task destruction from future-count or message-drain acknowledgements. + +**Final scoped Task4 verdict: IMPLEMENTABLE with all binding corrections in this report, including the accepted narrow opt-in catch-time observer and actual SDK-owner/permit retention qualification.** Implementation acceptance still requires fresh native/source proofs. No source, builds, refs, pins, global hook changes or foreign payload formatting were performed for this review. ## Task 5: actual compiled native proof and compatibility negatives From d9b2ee75b360cd10b2fd3ff925b58b4f660a4f32 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:13:49 +0300 Subject: [PATCH 49/58] test(transport): cover native controller ownership during shutdown Add an ignored integration test that verifies the physical owner retains the native controller while the shutdown callback is still borrowing it, and that the fixture owner is reclaimed once the transport is dropped. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 35 ++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index cf1f966e..08bad175 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1541,6 +1541,41 @@ pub(crate) fn native_fixture() -> ( (transport, host, artifact.init, barrier) } +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_controller_is_owned_until_shutdown_callback_returns() { + let (transport, host, init, barrier) = native_fixture(); + barrier.state.lock().unwrap().1 = true; + transport.defer_initialize(init, host); + transport.ensure_initialized().await.unwrap(); + barrier.entered.notified().await; + let weak = Arc::downgrade(&barrier); + let (release, gate) = std::sync::mpsc::channel(); + *barrier.finish_gate.lock().unwrap() = Some(gate); + let stopping = tokio::spawn({ + let transport = transport.clone(); + async move { transport.stop(Duration::from_secs(2)).await } + }); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); + // Observe ownership safely while the real native callback still borrows + // the controller. Always release C before a regression assertion can panic. + let retained = Arc::strong_count(&barrier) > 1; + assert_ne!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Complete + ); + release.send(()).unwrap(); + assert_eq!(stopping.await.unwrap().unwrap(), TB_OK); + drop(barrier); + let retained_through_transport = weak.upgrade().is_some(); + drop(transport); + assert!(weak.upgrade().is_none(), "successful fixture owner is reclaimed"); + assert!(retained, "physical owner must retain the borrowed native controller"); + assert!(retained_through_transport); +} + #[tokio::test] #[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] async fn actual_native_late_init_is_joined_after_initiator_abandonment() { From 9b88d42c1a621303c94918ca02e10d3c5a8f365c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:17:36 +0300 Subject: [PATCH 50/58] feat(module): retain native fixture controller with physical transport Native test fixtures now hand their controller to the transport so the borrowed controller stays owned for as long as the physical transport exists, including uncertain jobs. This removes the deliberate std::mem::forget leaks in the tests and lets them assert the controller is still alive through a weak reference instead. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_callbacks.rs | 17 ++++++++++++++++ .../src/module/host_callbacks_tests.rs | 3 +++ crates/tinybus/src/module/host_tests.rs | 8 +++++--- crates/tinybus/src/module/transport.rs | 6 ++++++ crates/tinybus/src/module/transport_tests.rs | 20 +++++++++++++------ 5 files changed, 45 insertions(+), 9 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 676fca99..90362476 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -146,6 +146,8 @@ pub struct HostCallbackBinding { #[cfg(test)] after_attach: Mutex>>, #[cfg(test)] + fixture_controller: Mutex>>, + #[cfg(test)] before_load_decision: Mutex, tokio::sync::oneshot::Receiver<()>)>>, } @@ -441,6 +443,8 @@ impl ModuleHost { #[cfg(test)] after_attach: Mutex::new(None), #[cfg(test)] + fixture_controller: Mutex::new(None), + #[cfg(test)] before_load_decision: Mutex::new(None), state: Arc::new(CallbackState { destination, @@ -538,6 +542,19 @@ impl ModuleHost { let fingerprint = callback_config_fingerprint(&bytes); drop(config); let (transport, host) = ModuleTransport::new(admitted.name.clone(), bytes); + #[cfg(test)] + { + // Transfer the fixture config borrow before C can enter, so + // setup unwind and loader cancellation retain the same owner. + *transport + .fixture_controller + .lock() + .expect("fixture controller lock") = binding + .fixture_controller + .lock() + .expect("fixture controller lock") + .take(); + } transport.pause_initialization(); transport.defer_initialize(artifact.init, host); let name = admitted.name.clone(); diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 4e0626fa..778db8ba 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -572,6 +572,7 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: let (_dir, artifact) = super::super::tests::staged_module(&artifact, &hash); let host = ModuleHost::new(Broker::new()); let binding = host.prepare_host_callbacks(vec![endpoint()]).await.unwrap(); + *binding.fixture_controller.lock().unwrap() = Some(barrier.clone()); let (entered, _) = tokio::sync::mpsc::channel(1); binding .serve_at( @@ -704,6 +705,7 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { let (_dir, artifact) = super::super::tests::staged_module(&artifact, &hash); let host = Arc::new(ModuleHost::new(Broker::new())); let binding = Arc::new(served_native_binding(&host).await); + *binding.fixture_controller.lock().unwrap() = Some(barrier.clone()); let decision = Arc::new(Notify::new()); let (resume, release) = tokio::sync::oneshot::channel(); *binding.before_load_decision.lock().unwrap() = Some((decision.clone(), release)); @@ -819,6 +821,7 @@ async fn native_callback_setup_failure_race(panic_setup: bool) { let (_dir, artifact) = super::super::tests::staged_module(&artifact, &hash); let host = Arc::new(ModuleHost::new(Broker::new())); let binding = Arc::new(served_native_binding(&host).await); + *binding.fixture_controller.lock().unwrap() = Some(barrier.clone()); let attached = Arc::new(Notify::new()); let seen = attached.clone(); let (resume, release) = std::sync::mpsc::channel(); diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 153e9f4b..8208e5f2 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -3247,7 +3247,9 @@ async fn actual_native_rejected_stop_submission_retains_guard_through_host_drop( !barrier.state.lock().unwrap().2, "Host Drop cannot refund and resubmit an uncertain owner" ); - // One deliberately retained native uncertainty instance, with its controller - // lifetime retained explicitly; no process-wide leak or backing-Drop claim. - std::mem::forget(barrier); + // This one uncertain physical owner retains its actual controller. + // It makes no process-wide leak or backing-Drop claim. + let controller = Arc::downgrade(&barrier); + drop(barrier); + assert!(controller.upgrade().is_some()); } diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 37e451ee..650826b2 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -81,6 +81,10 @@ pub(crate) struct ModuleTransport { before_worker_lock: StdMutex>>, #[cfg(test)] before_init_publication: StdMutex>>, + // Native fixtures pass a borrowed controller through C config. Retain its + // actual owner with the physical transport, including uncertain jobs. + #[cfg(test)] + pub(crate) fixture_controller: StdMutex>>, } // Reservations cover physical callback execution, including signals. They are @@ -275,6 +279,8 @@ impl ModuleTransport { before_worker_lock: StdMutex::new(None), #[cfg(test)] before_init_publication: StdMutex::new(None), + #[cfg(test)] + fixture_controller: StdMutex::new(None), }); *context.owner.lock().expect("physical owner lock") = Arc::downgrade(&transport); let config = context.config.lock().expect("module config lock"); diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 08bad175..e6628e80 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1538,6 +1538,7 @@ pub(crate) fn native_fixture() -> ( } .to_vec(); let (transport, host) = ModuleTransport::new("actual-native".into(), config); + *transport.fixture_controller.lock().unwrap() = Some(barrier.clone()); (transport, host, artifact.init, barrier) } @@ -1571,8 +1572,14 @@ async fn actual_native_controller_is_owned_until_shutdown_callback_returns() { drop(barrier); let retained_through_transport = weak.upgrade().is_some(); drop(transport); - assert!(weak.upgrade().is_none(), "successful fixture owner is reclaimed"); - assert!(retained, "physical owner must retain the borrowed native controller"); + assert!( + weak.upgrade().is_none(), + "successful fixture owner is reclaimed" + ); + assert!( + retained, + "physical owner must retain the borrowed native controller" + ); assert!(retained_through_transport); } @@ -1987,10 +1994,11 @@ async fn actual_native_late_eager_publication_cannot_refund_failed_cleanup_spawn drop(transport); drop(job); assert!(weak.upgrade().is_some()); - // The deliberately unjoinable native instance retains its externally anchored - // controller too. This is one explicit uncertain test instance, not global - // leak-bound or secret-erasure proof. - std::mem::forget(barrier); + // The existing uncertain physical owner retains this one controller too. + // No separate leaked controller is needed; this is not global leak-bound proof. + let controller = Arc::downgrade(&barrier); + drop(barrier); + assert!(controller.upgrade().is_some()); } unsafe extern "C" fn initialize_short_output( From ccb7dd099b54d798a8bf8551e8f665831b5f20b2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:20:48 +0300 Subject: [PATCH 51/58] refactor(module): flatten nested condition in host state transition The combined `if` with a let-chain was split into two nested `if` blocks so the transition logic no longer depends on let-chain support. Behaviour is unchanged. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/tinybus/src/module/host.rs b/crates/tinybus/src/module/host.rs index b86add27..9e3de525 100644 --- a/crates/tinybus/src/module/host.rs +++ b/crates/tinybus/src/module/host.rs @@ -1673,11 +1673,11 @@ impl ModuleControl for ModuleHostInner { let module = loaded .iter_mut() .find(|module| &module.unique_name == unique_name)?; - if !module.transport.is_faulted() - && let Some(old) = module.transition_from.take() - { - module.info.state = ModuleState::Stopped; - return Some((module.info.name.clone(), old, ModuleState::Stopped)); + if !module.transport.is_faulted() { + if let Some(old) = module.transition_from.take() { + module.info.state = ModuleState::Stopped; + return Some((module.info.name.clone(), old, ModuleState::Stopped)); + } } if !module.transport.is_faulted() || matches!( From 9f151c3b4f1329c610d896e2a5dcde1277acb3fc Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:22:32 +0300 Subject: [PATCH 52/58] test(transport): wait for task reclamation instead of asserting immediately The two tests that check a transport is dropped after shutdown asserted reclamation synchronously, which raced with the task's own Arc drop. They now poll the weak count under a timeout so the assertion observes actual reclamation rather than a flag that may precede task exit. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 21 +++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index e6628e80..8de14720 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -755,7 +755,15 @@ async fn stopping_initialized_module_before_ready_releases_observers_and_queued_ assert!(!transport.init_failed(), "S29"); let weak = Arc::downgrade(&transport); drop(transport); - assert!(weak.upgrade().is_none(), "S30"); + // drain_started and init publication precede the actual task's Arc drop. + // Observe reclamation itself instead of treating either flag as task exit. + tokio::time::timeout(Duration::from_secs(2), async { + while weak.strong_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("S30"); } #[tokio::test] @@ -1572,10 +1580,13 @@ async fn actual_native_controller_is_owned_until_shutdown_callback_returns() { drop(barrier); let retained_through_transport = weak.upgrade().is_some(); drop(transport); - assert!( - weak.upgrade().is_none(), - "successful fixture owner is reclaimed" - ); + tokio::time::timeout(Duration::from_secs(2), async { + while weak.strong_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("successful fixture owner is reclaimed after actual callback return"); assert!( retained, "physical owner must retain the borrowed native controller" From 98419a6981e499b88e96f226e0cd9dd7e87bae8d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:32:15 +0300 Subject: [PATCH 53/58] test(transport): defer cleanup assertion until after shutdown The cleanup state check now records whether cleanup was incomplete before releasing the callback and asserts it only after shutdown completes, so a regression cannot panic while the native callback still borrows the controller. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 8de14720..038f9d56 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1571,12 +1571,11 @@ async fn actual_native_controller_is_owned_until_shutdown_callback_returns() { // Observe ownership safely while the real native callback still borrows // the controller. Always release C before a regression assertion can panic. let retained = Arc::strong_count(&barrier) > 1; - assert_ne!( - transport.context.physical.lock().unwrap().cleanup, - CleanupState::Complete - ); + let cleanup_incomplete = + transport.context.physical.lock().unwrap().cleanup != CleanupState::Complete; release.send(()).unwrap(); assert_eq!(stopping.await.unwrap().unwrap(), TB_OK); + assert!(cleanup_incomplete); drop(barrier); let retained_through_transport = weak.upgrade().is_some(); drop(transport); From 9eee6b4393fad5a828045b19c94cdceaa2df070d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:06:08 +0300 Subject: [PATCH 54/58] test(module): secure release cache in host tests On Windows the temp directories used by these host tests inherit a private DACL, which caused the content gate assertions to fail for reasons unrelated to the behaviour under test. Securing the release cache for each directory first lets the tests exercise the intended content checks. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_tests.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 7ab31ac5..a2ba669f 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -1864,6 +1864,9 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { let _ = ModuleHost::new(Broker::new()).load_search_paths(); let directory = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); + // Files inherit the private DACL, so these assertions exercise content gates. + #[cfg(windows)] + crate::module::cache::secure_release_cache(directory.path(), directory.path()); let module = directory.path().join(if cfg!(windows) { "clock.dll" } else if cfg!(target_os = "macos") { @@ -1923,6 +1926,9 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { #[test] fn an_allowlist_with_a_mismatched_hash_refuses_the_file() { let directory = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); + // Files inherit the private DACL, so these assertions exercise content gates. + #[cfg(windows)] + crate::module::cache::secure_release_cache(directory.path(), directory.path()); let extension = if cfg!(windows) { "dll" } else if cfg!(target_os = "macos") { From 9ea71f45841e56590ab66188fe546c4fcbd5896f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:07:01 +0300 Subject: [PATCH 55/58] test(tinybus): raise host callback setup timeout The native callback setup failure race test now waits up to 30 seconds instead of 2 for the attached notification. Admission hashes the real artifact twice before this hook, and instrumented CI builds can spend several seconds there, so the old bound was too tight for a fixture setup wait that is independent of the lifecycle deadline and the race assertions below. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_callbacks_tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 778db8ba..8c964d73 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -840,7 +840,10 @@ async fn native_callback_setup_failure_race(panic_setup: bool) { .load_file_with_host_callbacks(artifact, config, &loading_binding) .await }); - tokio::time::timeout(Duration::from_secs(2), attached.notified()) + // Admission hashes the real artifact twice before this hook; instrumented + // CI builds can spend several seconds there. This is a fixture setup bound, + // independent of the lifecycle deadline and the race assertions below. + tokio::time::timeout(Duration::from_secs(30), attached.notified()) .await .unwrap(); let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); From c9e7789c5d559fd017a58b89f01c15bb8542cd9b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:13:20 +0300 Subject: [PATCH 56/58] test(transport): cover native lifecycle release after cleanup Add a test-only hook that runs after cleanup publication so a test can observe the transport state at that point. The new test confirms that a completed shutdown has already released its native lifecycle marker. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 15 ++++++++++ crates/tinybus/src/module/transport_tests.rs | 30 ++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 615bf93f..09ebcd8d 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -82,6 +82,8 @@ pub(crate) struct ModuleTransport { before_worker_lock: StdMutex>>, #[cfg(test)] before_init_publication: StdMutex>>, + #[cfg(test)] + after_cleanup_publication: StdMutex>>, // Native fixtures pass a borrowed controller through C config. Retain its // actual owner with the physical transport, including uncertain jobs. #[cfg(test)] @@ -297,6 +299,8 @@ impl ModuleTransport { #[cfg(test)] before_init_publication: StdMutex::new(None), #[cfg(test)] + after_cleanup_publication: StdMutex::new(None), + #[cfg(test)] fixture_controller: StdMutex::new(None), }); *context.owner.lock().expect("physical owner lock") = Arc::downgrade(&transport); @@ -1330,6 +1334,17 @@ impl PhysicalJob { } drop(state); transport.notify_physical(); + #[cfg(test)] + { + let hook = transport + .after_cleanup_publication + .lock() + .expect("cleanup publication test hook lock") + .take(); + if let Some(hook) = hook { + hook(); + } + } drop(pending); drop(guard); } diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 038f9d56..b7c67963 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -2145,3 +2145,33 @@ async fn rejected_observer_after_ordinary_stop_preserves_nonfault_cause() { 2 ); } + +#[tokio::test] +async fn completed_shutdown_has_already_released_its_native_lifecycle_marker() { + let _guard = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + SHUTDOWN_CODE.store(TB_OK, Ordering::Release); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let (transport, host) = ModuleTransport::new("completed-shutdown".into(), vec![]); + transport.eager_initialize(initialize_ok, host).unwrap(); + let published = Arc::new(Notify::new()); + let entered = published.clone(); + let (release, worker) = std::sync::mpsc::channel(); + *transport.after_cleanup_publication.lock().unwrap() = Some(Box::new(move || { + entered.notify_one(); + worker.recv().unwrap(); + })); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + published.notified().await; + let result = stopping.await.unwrap(); + let still_pending = transport.native_lifecycle_pending(); + // Always release the worker before asserting, even against the unfixed code. + release.send(()).unwrap(); + assert_eq!(result.unwrap(), TB_OK); + assert!( + !still_pending, + "completed cleanup still reports native work" + ); +} From f241c79e996e29a9ca9f17293152a55446a9c4dd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:14:06 +0300 Subject: [PATCH 57/58] fix(tinybus): clear native activity before releasing lifecycle guard Move the pending activity drop ahead of the state update so a completed stop cannot leave enable/disable reporting that opaque work is still running. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 09ebcd8d..152fab0a 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -1318,6 +1318,10 @@ impl PhysicalJob { let code = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { transport.stop_sync(self.deadline) })); + // Completion is observable before this worker releases its lifecycle + // guard. Clear native activity first so a completed stop cannot leave + // enable/disable reporting that opaque work is still running. + drop(pending); let mut state = transport .context .physical @@ -1345,7 +1349,6 @@ impl PhysicalJob { hook(); } } - drop(pending); drop(guard); } } From a87fabe02576edf98264cb530594b41950dc67dd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:19:39 +0300 Subject: [PATCH 58/58] test(module): add native terminal state to execution context test host The test host builder now initialises the new native_terminal_state field so the execution context tests keep compiling against the updated host struct. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/execution_context_tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/tinybus-module/src/execution_context_tests.rs b/crates/tinybus-module/src/execution_context_tests.rs index 3d7e0dc9..7b0d052a 100644 --- a/crates/tinybus-module/src/execution_context_tests.rs +++ b/crates/tinybus-module/src/execution_context_tests.rs @@ -27,6 +27,7 @@ fn owner() -> (ExecutionOwner, &'static AtomicUsize) { }, ready, broker_routing: None, + native_terminal_state: None, }); (ExecutionOwner::new(host), count) }