From 154507b6c82b454d744f437f45d987091dcde57e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:36:59 +0300 Subject: [PATCH 001/130] feat(connection): add catch observer hook for method panics Add a hidden `__set_catch_observer` setter that lets the SDK register a bounded callback fired before a panicking method's payload is dropped, so callers can fence state ahead of the drop. A test confirms the observer runs before the primary payload is released. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/connection.rs | 10 ++++++++ crates/tinybus/src/connection_tests.rs | 32 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/crates/tinybus/src/connection.rs b/crates/tinybus/src/connection.rs index d4d4af7d..d11a14ff 100644 --- a/crates/tinybus/src/connection.rs +++ b/crates/tinybus/src/connection.rs @@ -192,6 +192,7 @@ struct Inner { signal_deliveries: broadcast::Sender, local_publisher: crate::LocalPublisherIdentity, panic_handler: std::sync::RwLock Error + Send + Sync>>>, + catch_observer: std::sync::RwLock>>, /// Bulk streams being received. On the connection rather than in the object /// tree because a chunk has to be checked against the header's stamped /// `sender`; interfaces receive only the verified subset in CallContext. @@ -389,6 +390,7 @@ impl Connection { signal_deliveries, local_publisher: crate::LocalPublisherIdentity::new(), panic_handler: std::sync::RwLock::new(None), + catch_observer: std::sync::RwLock::new(None), streams: StreamRegistry::new(), }); tokio::spawn(writer_loop(inner.transport.clone(), outbound)); @@ -1021,6 +1023,14 @@ impl Connection { .expect("panic handler lock") = Some(handler); } + /// Install SDK-local catch-time notification before a method panic payload drops. + /// The observer must be bounded, must not unwind, and must never join workers. + /// This plumbing does not grant native authority or replace the panic policy. + #[doc(hidden)] + pub fn __set_catch_observer(&self, observer: Arc) { + *self.inner.catch_observer.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(observer); + } + /// What this connection will accept from peers sending it bulk streams. pub fn stream_limits(&self) -> StreamLimits { self.inner.streams.limits() diff --git a/crates/tinybus/src/connection_tests.rs b/crates/tinybus/src/connection_tests.rs index 54ca5ed6..7e056005 100644 --- a/crates/tinybus/src/connection_tests.rs +++ b/crates/tinybus/src/connection_tests.rs @@ -891,3 +891,35 @@ async fn weak_references_cannot_resurrect_dispatch_state_after_the_last_owner_dr assert!(format!("{weak:?}").contains("has_strong_owner: false")); Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn opted_in_method_fault_fences_before_primary_payload_drop() { + struct Payload { + fenced: Arc, + observed: Arc, + } + impl Drop for Payload { + fn drop(&mut self) { + self.observed.store(self.fenced.load(Ordering::Acquire), Ordering::Release); + } + } + struct PanicService(Arc, Arc); + #[async_trait] + impl Interface for PanicService { + fn name(&self) -> InterfaceName { "ai.tinyhumans.Test".parse().unwrap() } + fn members(&self) -> Vec { vec!["Panic".parse().unwrap()] } + async fn call(&self, _: &MemberName, _: Value) -> Result { + std::panic::panic_any(Payload { fenced: self.0.clone(), observed: self.1.clone() }); + } + } + let fenced = Arc::new(AtomicBool::new(false)); + let observed = Arc::new(AtomicBool::new(false)); + let (client, service) = pair().await; + service.serve_at(path(), PanicService(fenced.clone(), observed.clone())).await.unwrap(); + let fence = fenced.clone(); + service.__set_catch_observer(Arc::new(move || { fence.store(true, Ordering::Release); })); + service.__set_panic_handler(Arc::new(|| Error::MethodFailed { name: "ai.tinyhumans.tinybus.Error.ModulePanicked".into(), message: "a module method panicked".into() })); + let error = client.call_raw(call("Panic", Value::Null), DEFAULT_TIMEOUT).await.unwrap_err(); + assert_eq!(error.wire_name(), "ai.tinyhumans.tinybus.Error.ModulePanicked"); + assert!(observed.load(Ordering::Acquire), "primary payload dropped before opted-in fault observer"); +} From bb79088d39700f1baaf79c5f3c352b90bd4ad8cb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:38:01 +0300 Subject: [PATCH 002/130] feat(connection): notify catch observer on dispatch panic CatchUnwind now carries an optional observer callback that is invoked when a dispatched call panics, before the panic payload is dropped. This lets opted-in modules close native authority ahead of foreign payload destruction, since no registry or handler lock is held at that point. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/connection.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/tinybus/src/connection.rs b/crates/tinybus/src/connection.rs index d11a14ff..81297d2e 100644 --- a/crates/tinybus/src/connection.rs +++ b/crates/tinybus/src/connection.rs @@ -1453,12 +1453,14 @@ async fn handle_call(inner: Arc, message: Message, brokered: bool) { .read() .expect("panic handler lock") .clone(); + let catch_observer = inner.catch_observer.read() + .unwrap_or_else(std::sync::PoisonError::into_inner).clone(); let result = if header.stream_reply && header.confidential { Err(Error::protocol( "a confidential call cannot request a streamed reply", )) } else if let Some(panic_handler) = panic_handler { - match CatchUnwind::new(dispatch(&inner, &header, message.body, brokered)).await { + match CatchUnwind::new(dispatch(&inner, &header, message.body, brokered), catch_observer).await { Ok(result) => result, Err(()) => Err(panic_handler()), } @@ -1546,12 +1548,14 @@ async fn send_reply(inner: &Inner, mut reply: Message) -> bool { struct CatchUnwind { future: Pin>, + observer: Option>, } impl CatchUnwind { - fn new(future: F) -> Self { + fn new(future: F, observer: Option>) -> Self { Self { future: Box::pin(future), + observer, } } } @@ -1560,11 +1564,20 @@ impl Future for CatchUnwind { type Output = std::result::Result; fn poll(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll { - let future = self.get_mut().future.as_mut(); + let this = self.get_mut(); + let future = this.future.as_mut(); match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| future.poll(context))) { Ok(Poll::Ready(value)) => Poll::Ready(Ok(value)), Ok(Poll::Pending) => Poll::Pending, - Err(_) => Poll::Ready(Err(())), + Err(payload) => { + // No registry or handler lock is held here. Opted-in modules + // close actual native authority before foreign payload Drop. + if let Some(observer) = &this.observer { + observer(); + } + drop(payload); + Poll::Ready(Err(())) + } } } } From 6aba6d97facbfb396a0f0c5f90065e336c108940 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:42:52 +0300 Subject: [PATCH 003/130] test(module): harden host test fixtures and attach race The native callback setup race test now selects on the load task, the attach notification, and a deadline so a load that finishes or stalls before attach fails with a clear message instead of a bare timeout. The host helper tests create their artifacts inside a private subdirectory with a protected ACL, so files no longer inherit the checkout's DACL on Windows. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/module/host_callbacks_tests.rs | 12 ++++++---- crates/tinybus/src/module/host_tests.rs | 24 ++++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 778db8ba..d2b1488a 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -835,14 +835,18 @@ async fn native_callback_setup_failure_race(panic_setup: bool) { })); let loading_host = host.clone(); let loading_binding = binding.clone(); - let load = tokio::spawn(async move { + let mut load = tokio::spawn(async move { loading_host .load_file_with_host_callbacks(artifact, config, &loading_binding) .await }); - tokio::time::timeout(Duration::from_secs(2), attached.notified()) - .await - .unwrap(); + tokio::select! { + _ = attached.notified() => {}, + result = &mut load => panic!("native callback load ended before attach: {result:?}"), + _ = tokio::time::sleep(Duration::from_secs(2)) => { + panic!("native callback load did not reach attach before the setup deadline"); + } + } let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); // Actual ordinary lifecycle ownership, before the callback initialization // gate opens. This instance proves C never entered, rather than any join. diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 8208e5f2..5fd59203 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -1860,8 +1860,11 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { ); let _ = ModuleHost::new(Broker::new()).load_search_paths(); - let directory = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); - let module = directory.path().join(if cfg!(windows) { + let root = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); + let directory = root.path().join("private"); + // Protect the artifact parent before files inherit the checkout's DACL. + super::super::windows_acl::create_private_dir_all(&directory).unwrap(); + let module = directory.join(if cfg!(windows) { "clock.dll" } else if cfg!(target_os = "macos") { "clock.dylib" @@ -1871,7 +1874,7 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { std::fs::write(&module, b"module bytes").unwrap(); let digest = crate::module::hash::file_hex(std::fs::File::open(&module).unwrap()).unwrap(); std::fs::write( - directory.path().join("modules.toml"), + directory.join("modules.toml"), format!( "clock.{} = \"{digest}\"\n", if cfg!(windows) { @@ -1885,7 +1888,7 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { ) .unwrap(); assert!(check_file(&module).is_ok()); - let text = directory.path().join("clock.txt"); + let text = directory.join("clock.txt"); std::fs::write(&text, b"module bytes").unwrap(); assert!( check_file(&text) @@ -1893,7 +1896,7 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { .to_string() .contains("extension is not loadable") ); - std::fs::write(directory.path().join("modules.toml"), "other = \"00\"\n").unwrap(); + std::fs::write(directory.join("modules.toml"), "other = \"00\"\n").unwrap(); assert!( check_file(&module) .unwrap_err() @@ -1901,7 +1904,7 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { .contains("absent") ); std::fs::write( - directory.path().join("modules.toml"), + directory.join("modules.toml"), "clock = \"not-a-hash\"\n", ) .unwrap(); @@ -1919,7 +1922,10 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { #[test] fn an_allowlist_with_a_mismatched_hash_refuses_the_file() { - let directory = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); + let root = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); + let directory = root.path().join("private"); + // Protect the artifact parent before files inherit the checkout's DACL. + super::super::windows_acl::create_private_dir_all(&directory).unwrap(); let extension = if cfg!(windows) { "dll" } else if cfg!(target_os = "macos") { @@ -1927,10 +1933,10 @@ fn an_allowlist_with_a_mismatched_hash_refuses_the_file() { } else { "so" }; - let path = directory.path().join(format!("clock.{extension}")); + let path = directory.join(format!("clock.{extension}")); std::fs::write(&path, b"not a module").unwrap(); std::fs::write( - directory.path().join("modules.toml"), + directory.join("modules.toml"), format!("clock.{extension} = \"{}\"\n", "0".repeat(64)), ) .unwrap(); From 81ae954aa4d6c1b3185880e6a3ac54f32f9f4226 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:43:47 +0300 Subject: [PATCH 004/130] test(tinybus): cover native attach wait timing and failure Extract the attach wait into a helper and add tests proving a valid preload outliving product deadlines is not mistaken for setup failure, and that an early load failure surfaces as a panic. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/module/host_callbacks_tests.rs | 56 ++++++++++++++++--- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index d2b1488a..5cfa8498 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -810,6 +810,54 @@ async fn actual_native_fault_at_ready_commit_dominates_ordinary_stop() { native_callback_decision_race(true, true).await; } +async fn await_native_attach( + attached: &Notify, + load: &mut tokio::task::JoinHandle, +) { + tokio::select! { + _ = attached.notified() => {}, + result = load => panic!("native callback load ended before attach: {result:?}"), + _ = tokio::time::sleep(Duration::from_secs(2)) => { + panic!("native callback load did not reach attach before the setup deadline"); + } + } +} + +#[tokio::test(start_paused = true)] +async fn setup_attachment_wait_allows_preload_to_outlive_product_deadlines() { + let attached = Arc::new(Notify::new()); + let observed = attached.clone(); + let (release, preload) = tokio::sync::oneshot::channel::<()>(); + let mut load = tokio::spawn(async move { + preload.await.unwrap(); + std::future::pending::<()>().await; + }); + let wait = tokio::spawn(async move { + await_native_attach(&observed, &mut load).await; + load.abort(); + assert!(load.await.unwrap_err().is_cancelled()); + }); + tokio::task::yield_now().await; + // The setup assertion starts after attachment, regardless of how long + // admission, hashing, and native loading take to establish that precondition. + tokio::time::advance(Duration::from_secs(3)).await; + tokio::task::yield_now().await; + assert!(!wait.is_finished(), "valid preload was mistaken for setup failure"); + release.send(()).unwrap(); + attached.notify_one(); + wait.await.unwrap(); +} + +#[tokio::test] +async fn setup_attachment_wait_reports_an_early_load_failure() { + let attached = Arc::new(Notify::new()); + let mut load = tokio::spawn(async { Err::<(), _>(Error::ConnectionClosed) }); + let wait = tokio::spawn(async move { + await_native_attach(&attached, &mut load).await; + }); + assert!(wait.await.unwrap_err().is_panic()); +} + async fn native_callback_setup_failure_race(panic_setup: bool) { use super::super::super::transport::tests::{native_fixture, native_fixture_json}; let (unused, _, _, barrier) = native_fixture(); @@ -840,13 +888,7 @@ async fn native_callback_setup_failure_race(panic_setup: bool) { .load_file_with_host_callbacks(artifact, config, &loading_binding) .await }); - tokio::select! { - _ = attached.notified() => {}, - result = &mut load => panic!("native callback load ended before attach: {result:?}"), - _ = tokio::time::sleep(Duration::from_secs(2)) => { - panic!("native callback load did not reach attach before the setup deadline"); - } - } + await_native_attach(&attached, &mut load).await; let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); // Actual ordinary lifecycle ownership, before the callback initialization // gate opens. This instance proves C never entered, rather than any join. From 309b67d7b30ffb442fe1811d05017392117012f5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:44:45 +0300 Subject: [PATCH 005/130] test(tinybus): harden native attach watchdog and cover stuck preload The native callback attach helper now uses a biased select, aborts the load task on timeout, and waits up to 60 seconds so directory admission and hashing cannot be mistaken for a setup failure. A new test asserts that a stuck preload is cancelled, and the early-failure test now checks the panic message contents. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/connection.rs | 20 +++++++-- crates/tinybus/src/connection_tests.rs | 45 ++++++++++++++----- .../src/module/host_callbacks_tests.rs | 40 ++++++++++++++--- crates/tinybus/src/module/host_tests.rs | 6 +-- 4 files changed, 87 insertions(+), 24 deletions(-) diff --git a/crates/tinybus/src/connection.rs b/crates/tinybus/src/connection.rs index 81297d2e..338ee273 100644 --- a/crates/tinybus/src/connection.rs +++ b/crates/tinybus/src/connection.rs @@ -1028,7 +1028,11 @@ impl Connection { /// This plumbing does not grant native authority or replace the panic policy. #[doc(hidden)] pub fn __set_catch_observer(&self, observer: Arc) { - *self.inner.catch_observer.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(observer); + *self + .inner + .catch_observer + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(observer); } /// What this connection will accept from peers sending it bulk streams. @@ -1453,14 +1457,22 @@ async fn handle_call(inner: Arc, message: Message, brokered: bool) { .read() .expect("panic handler lock") .clone(); - let catch_observer = inner.catch_observer.read() - .unwrap_or_else(std::sync::PoisonError::into_inner).clone(); + let catch_observer = inner + .catch_observer + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); let result = if header.stream_reply && header.confidential { Err(Error::protocol( "a confidential call cannot request a streamed reply", )) } else if let Some(panic_handler) = panic_handler { - match CatchUnwind::new(dispatch(&inner, &header, message.body, brokered), catch_observer).await { + match CatchUnwind::new( + dispatch(&inner, &header, message.body, brokered), + catch_observer, + ) + .await + { Ok(result) => result, Err(()) => Err(panic_handler()), } diff --git a/crates/tinybus/src/connection_tests.rs b/crates/tinybus/src/connection_tests.rs index 7e056005..e8cabdc3 100644 --- a/crates/tinybus/src/connection_tests.rs +++ b/crates/tinybus/src/connection_tests.rs @@ -900,26 +900,51 @@ async fn opted_in_method_fault_fences_before_primary_payload_drop() { } impl Drop for Payload { fn drop(&mut self) { - self.observed.store(self.fenced.load(Ordering::Acquire), Ordering::Release); + self.observed + .store(self.fenced.load(Ordering::Acquire), Ordering::Release); } } struct PanicService(Arc, Arc); #[async_trait] impl Interface for PanicService { - fn name(&self) -> InterfaceName { "ai.tinyhumans.Test".parse().unwrap() } - fn members(&self) -> Vec { vec!["Panic".parse().unwrap()] } + fn name(&self) -> InterfaceName { + "ai.tinyhumans.Test".parse().unwrap() + } + fn members(&self) -> Vec { + vec!["Panic".parse().unwrap()] + } async fn call(&self, _: &MemberName, _: Value) -> Result { - std::panic::panic_any(Payload { fenced: self.0.clone(), observed: self.1.clone() }); + std::panic::panic_any(Payload { + fenced: self.0.clone(), + observed: self.1.clone(), + }); } } let fenced = Arc::new(AtomicBool::new(false)); let observed = Arc::new(AtomicBool::new(false)); let (client, service) = pair().await; - service.serve_at(path(), PanicService(fenced.clone(), observed.clone())).await.unwrap(); + service + .serve_at(path(), PanicService(fenced.clone(), observed.clone())) + .await + .unwrap(); let fence = fenced.clone(); - service.__set_catch_observer(Arc::new(move || { fence.store(true, Ordering::Release); })); - service.__set_panic_handler(Arc::new(|| Error::MethodFailed { name: "ai.tinyhumans.tinybus.Error.ModulePanicked".into(), message: "a module method panicked".into() })); - let error = client.call_raw(call("Panic", Value::Null), DEFAULT_TIMEOUT).await.unwrap_err(); - assert_eq!(error.wire_name(), "ai.tinyhumans.tinybus.Error.ModulePanicked"); - assert!(observed.load(Ordering::Acquire), "primary payload dropped before opted-in fault observer"); + service.__set_catch_observer(Arc::new(move || { + fence.store(true, Ordering::Release); + })); + service.__set_panic_handler(Arc::new(|| Error::MethodFailed { + name: "ai.tinyhumans.tinybus.Error.ModulePanicked".into(), + message: "a module method panicked".into(), + })); + let error = client + .call_raw(call("Panic", Value::Null), DEFAULT_TIMEOUT) + .await + .unwrap_err(); + assert_eq!( + error.wire_name(), + "ai.tinyhumans.tinybus.Error.ModulePanicked" + ); + assert!( + observed.load(Ordering::Acquire), + "primary payload dropped before opted-in fault observer" + ); } diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 5cfa8498..28485e04 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -814,11 +814,15 @@ async fn await_native_attach( attached: &Notify, load: &mut tokio::task::JoinHandle, ) { + // This watchdog bounds a broken fixture, not product readiness. Admission + // walks directories and hashes the library before attachment can happen. tokio::select! { + biased; _ = attached.notified() => {}, - result = load => panic!("native callback load ended before attach: {result:?}"), - _ = tokio::time::sleep(Duration::from_secs(2)) => { - panic!("native callback load did not reach attach before the setup deadline"); + result = &mut *load => panic!("native callback load ended before attach: {result:?}"), + _ = tokio::time::sleep(Duration::from_secs(60)) => { + load.abort(); + panic!("native callback fixture never attached"); } } } @@ -842,7 +846,10 @@ async fn setup_attachment_wait_allows_preload_to_outlive_product_deadlines() { // admission, hashing, and native loading take to establish that precondition. tokio::time::advance(Duration::from_secs(3)).await; tokio::task::yield_now().await; - assert!(!wait.is_finished(), "valid preload was mistaken for setup failure"); + assert!( + !wait.is_finished(), + "valid preload was mistaken for setup failure" + ); release.send(()).unwrap(); attached.notify_one(); wait.await.unwrap(); @@ -855,7 +862,30 @@ async fn setup_attachment_wait_reports_an_early_load_failure() { let wait = tokio::spawn(async move { await_native_attach(&attached, &mut load).await; }); - assert!(wait.await.unwrap_err().is_panic()); + let panic = wait.await.unwrap_err().into_panic(); + let message = panic.downcast_ref::().unwrap(); + assert!(message.contains("load ended before attach"), "{message}"); + assert!(message.contains("ConnectionClosed"), "{message}"); +} + +#[tokio::test(start_paused = true)] +async fn setup_attachment_watchdog_cancels_a_stuck_preload() { + let attached = Notify::new(); + let mut load = tokio::spawn(std::future::pending::<()>()); + let cancellation = load.abort_handle(); + let wait = tokio::spawn(async move { + await_native_attach(&attached, &mut load).await; + }); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(61)).await; + let panic = wait.await.unwrap_err().into_panic(); + let message = panic.downcast_ref::<&str>().unwrap(); + assert_eq!(*message, "native callback fixture never attached"); + tokio::task::yield_now().await; + assert!( + cancellation.is_finished(), + "stuck preload must be cancelled" + ); } async fn native_callback_setup_failure_race(panic_setup: bool) { diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index 5fd59203..097cd232 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -1903,11 +1903,7 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { .to_string() .contains("absent") ); - std::fs::write( - directory.join("modules.toml"), - "clock = \"not-a-hash\"\n", - ) - .unwrap(); + std::fs::write(directory.join("modules.toml"), "clock = \"not-a-hash\"\n").unwrap(); assert!( check_file(&module) .unwrap_err() From f7b040fc8b469197d5e9c982d32ad676e48d9915 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 13:58:08 +0300 Subject: [PATCH 006/130] test(module): cover controlled release on early unwind Add a fixture and test asserting that an early unwind wakes native entry and releases the finish and decision channels rather than disconnecting them. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/module/host_callbacks_tests.rs | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 28485e04..bd876357 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -558,6 +558,48 @@ fn configuration_fingerprints_preserve_json_equality_without_normalizing_strings ); } +// Private fixture-only release ownership. Drop never joins or asserts. +struct ControlledRelease { + barrier: Arc, + finish: Option>, + resume: Option>, + transport: Option>, +} +impl ControlledRelease { + fn new(barrier: Arc) -> Self { + Self { barrier, finish: None, resume: None, transport: None } + } +} +impl Drop for ControlledRelease { + fn drop(&mut self) {} +} + +#[test] +fn controlled_release_wakes_entry_and_releases_channels_on_early_unwind() { + use super::super::super::transport::tests::NativeBarrier; + let barrier = Arc::new(NativeBarrier { + state: Mutex::new((None, false, false)), + changed: std::sync::Condvar::new(), + entered: Notify::new(), joined: Notify::new(), + finish_gate: Mutex::new(None), + }); + let (finish, receive) = std::sync::mpsc::channel(); + let (resume, decision) = tokio::sync::oneshot::channel(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut cleanup = ControlledRelease::new(barrier.clone()); + cleanup.finish = Some(finish); + cleanup.resume = Some(resume); + panic!("fixture assertion before controlled release"); + })); + let finish_result = receive.try_recv(); + let mut decision = decision; + let decision_result = decision.try_recv(); + assert!(result.is_err()); + assert!(barrier.state.lock().unwrap().1, "early unwind must wake native entry"); + assert!(finish_result.is_ok(), "finish must be released, not disconnected"); + assert!(decision_result.is_ok(), "decision must be released, not disconnected"); +} + // Manually polling the public future lets ordinary shutdown commit while the // loader is suspended; no scheduler race decides which cancellation is tested. async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: bool) { From 2a3f3f78f4ae0c9f17d3f13c5cedce33ce1209ba Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 14:00:22 +0300 Subject: [PATCH 007/130] chore: files changed crates/tinybus/src/connection.rs,crates/tinybus/src/module/host_callbacks_tests Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/connection.rs | 1 + .../src/module/host_callbacks_tests.rs | 250 +++++++++++++----- crates/tinybus/src/module/transport_tests.rs | 20 +- 3 files changed, 194 insertions(+), 77 deletions(-) diff --git a/crates/tinybus/src/connection.rs b/crates/tinybus/src/connection.rs index 338ee273..65238500 100644 --- a/crates/tinybus/src/connection.rs +++ b/crates/tinybus/src/connection.rs @@ -1024,6 +1024,7 @@ impl Connection { } /// Install SDK-local catch-time notification before a method panic payload drops. + /// Requires `__set_panic_handler`: without a handler no catch fence is constructed. /// The observer must be bounded, must not unwind, and must never join workers. /// This plumbing does not grant native authority or replace the panic policy. #[doc(hidden)] diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index bd876357..da149c67 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -567,11 +567,46 @@ struct ControlledRelease { } impl ControlledRelease { fn new(barrier: Arc) -> Self { - Self { barrier, finish: None, resume: None, transport: None } + Self { + barrier, + finish: None, + resume: None, + transport: None, + } + } +} +impl ControlledRelease { + fn release(&mut self) { + self.barrier + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .1 = true; + self.barrier.changed.notify_all(); + if let Some(resume) = self.resume.take() { + let _ = resume.send(()); + } + if let Some(finish) = self.finish.take() { + let _ = finish.send(()); + } } } impl Drop for ControlledRelease { - fn drop(&mut self) {} + fn drop(&mut self) { + self.release(); + // A failing test cannot join from Drop. The existing transport physical + // anchor remains authoritative; this task retains the real owner while + // requesting stop after all artificial gates have opened. + if let Some(transport) = self.transport.take() { + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + let barrier = self.barrier.clone(); + runtime.spawn(async move { + let _ = transport.stop(Duration::from_secs(2)).await; + drop(barrier); + }); + } + } + } } #[test] @@ -580,7 +615,8 @@ fn controlled_release_wakes_entry_and_releases_channels_on_early_unwind() { let barrier = Arc::new(NativeBarrier { state: Mutex::new((None, false, false)), changed: std::sync::Condvar::new(), - entered: Notify::new(), joined: Notify::new(), + entered: Notify::new(), + joined: Notify::new(), finish_gate: Mutex::new(None), }); let (finish, receive) = std::sync::mpsc::channel(); @@ -595,9 +631,18 @@ fn controlled_release_wakes_entry_and_releases_channels_on_early_unwind() { let mut decision = decision; let decision_result = decision.try_recv(); assert!(result.is_err()); - assert!(barrier.state.lock().unwrap().1, "early unwind must wake native entry"); - assert!(finish_result.is_ok(), "finish must be released, not disconnected"); - assert!(decision_result.is_ok(), "decision must be released, not disconnected"); + assert!( + barrier.state.lock().unwrap().1, + "early unwind must wake native entry" + ); + assert!( + finish_result.is_ok(), + "finish must be released, not disconnected" + ); + assert!( + decision_result.is_ok(), + "decision must be released, not disconnected" + ); } // Manually polling the public future lets ordinary shutdown commit while the @@ -628,6 +673,7 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: ) .await .unwrap(); + let mut cleanup = ControlledRelease::new(barrier.clone()); let mut load = Some(Box::pin( host.load_file_with_host_callbacks(&artifact, config, &binding), )); @@ -643,6 +689,7 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: .await .unwrap(); let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + cleanup.transport = Some(transport.clone()); let view = barrier.state.lock().unwrap().0.unwrap(); barrier.state.lock().unwrap().1 = true; barrier.changed.notify_all(); @@ -658,15 +705,13 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: .await; let (finish, receive) = std::sync::mpsc::channel(); *barrier.finish_gate.lock().unwrap() = Some(receive); + cleanup.finish = Some(finish); let stopping = transport.clone(); let stop = tokio::spawn(async move { stopping.stop(Duration::from_secs(2)).await }); tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) .await .unwrap(); - assert_eq!( - unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, - 2 - ); + let stopped_flags = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; let load_result = if reject_gate { transport.release_initialization(false); drop(load.take()); @@ -684,14 +729,22 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: }; let held_flags = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; let held_fault = transport.is_faulted(); - finish.send(()).unwrap(); // Always release opaque C before RED assertions. - assert!(stop.await.unwrap().is_ok()); - if let Some(result) = load_result { - assert!(result.is_err()); + cleanup.release(); + let stopped = stop.await; + let loaded = if let Some(result) = load_result { drop(load.take()); + Some(result) } else if let Some(load) = load { - assert!(load.await.is_err()); + Some(load.await) + } else { + None + }; + cleanup.transport = None; + assert!(stopped.unwrap().is_ok()); + if let Some(result) = loaded { + assert!(result.is_err()); } + assert_eq!(stopped_flags, 2); assert_eq!(held_flags, 2, "ordinary cancellation cannot invent Fault"); assert!(!held_fault); assert_eq!( @@ -751,6 +804,8 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { let decision = Arc::new(Notify::new()); let (resume, release) = tokio::sync::oneshot::channel(); *binding.before_load_decision.lock().unwrap() = Some((decision.clone(), release)); + let mut cleanup = ControlledRelease::new(barrier.clone()); + cleanup.resume = Some(resume); let loading_host = host.clone(); let loading_binding = binding.clone(); let load = tokio::spawn(async move { @@ -764,6 +819,7 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { tokio::task::yield_now().await; } let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + cleanup.transport = Some(transport.clone()); let view = barrier.state.lock().unwrap().0.unwrap(); barrier.state.lock().unwrap().1 = true; barrier.changed.notify_all(); @@ -793,41 +849,45 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { tokio::time::advance(Duration::from_secs(6)).await; } decision.notified().await; - assert_eq!(transport.is_ready(), ready); + let observed_ready = transport.is_ready(); let (finish, receive) = std::sync::mpsc::channel(); *barrier.finish_gate.lock().unwrap() = Some(receive); + cleanup.finish = Some(finish); let stopping = transport.clone(); let stop = tokio::spawn(async move { stopping.stop(Duration::from_secs(2)).await }); while !barrier.state.lock().unwrap().2 { tokio::task::yield_now().await; } - assert_eq!( - unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, - 2 - ); + let stopped_flags = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; + let mut fault_flags = None; if native_fault { unsafe { (view.fault)(view.host_ctx, std::ptr::null(), 0); } - assert_eq!( - unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, - 6 - ); + fault_flags = Some(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }); } - resume.send(()).unwrap(); + let resumed = cleanup.resume.take().unwrap().send(()); // Rollback revokes before awaiting physical close or returning failure. while !binding.state.lifetime.lock().unwrap().revoked && !load.is_finished() { tokio::task::yield_now().await; } let held = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; - finish.send(()).unwrap(); + cleanup.release(); // Keep virtual time runnable through the actual opaque return. while !stop.is_finished() { tokio::task::yield_now().await; } - let stopped = stop.await.unwrap(); - assert!(stopped.is_ok(), "actual stop: {stopped:?}"); - assert!(load.await.unwrap().is_err()); + let stopped = stop.await; + let loaded = load.await; + cleanup.transport = None; + assert!(resumed.is_ok()); + assert!(stopped.unwrap().is_ok()); + assert!(loaded.unwrap().is_err()); + assert_eq!(observed_ready, ready); + assert_eq!(stopped_flags, 2); + if let Some(flags) = fault_flags { + assert_eq!(flags, 6); + } assert_eq!(held, if ready && !native_fault { 2 } else { 6 }); assert_eq!( unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, @@ -852,19 +912,40 @@ async fn actual_native_fault_at_ready_commit_dominates_ordinary_stop() { native_callback_decision_race(true, true).await; } -async fn await_native_attach( +#[derive(Debug, PartialEq, Eq)] +enum AttachOutcome { + Attached, + LoadSucceeded, + LoadRefused, + LoadPanicked, + LoadCanceled, + WatchdogExpired, +} + +async fn await_native_attach( attached: &Notify, - load: &mut tokio::task::JoinHandle, -) { - // This watchdog bounds a broken fixture, not product readiness. Admission - // walks directories and hashes the library before attachment can happen. + load: &mut tokio::task::JoinHandle>, + completed: &mut Option, tokio::task::JoinError>>, +) -> AttachOutcome { + // A fixture precondition watchdog, not a product security/readiness TTL. + // Abort requests cooperative cancellation; it cannot interrupt synchronous + // admission or prove physical native quiescence. The caller owns cleanup. tokio::select! { biased; - _ = attached.notified() => {}, - result = &mut *load => panic!("native callback load ended before attach: {result:?}"), + _ = attached.notified() => AttachOutcome::Attached, + result = &mut *load => { + let category = match &result { + Ok(Ok(_)) => AttachOutcome::LoadSucceeded, + Ok(Err(_)) => AttachOutcome::LoadRefused, + Err(error) if error.is_cancelled() => AttachOutcome::LoadCanceled, + Err(_) => AttachOutcome::LoadPanicked, + }; + *completed = Some(result); + category + }, _ = tokio::time::sleep(Duration::from_secs(60)) => { load.abort(); - panic!("native callback fixture never attached"); + AttachOutcome::WatchdogExpired } } } @@ -875,59 +956,64 @@ async fn setup_attachment_wait_allows_preload_to_outlive_product_deadlines() { let observed = attached.clone(); let (release, preload) = tokio::sync::oneshot::channel::<()>(); let mut load = tokio::spawn(async move { - preload.await.unwrap(); - std::future::pending::<()>().await; + let _ = preload.await; + std::future::pending::>().await }); let wait = tokio::spawn(async move { - await_native_attach(&observed, &mut load).await; + let mut completed = None; + let outcome = await_native_attach(&observed, &mut load, &mut completed).await; load.abort(); - assert!(load.await.unwrap_err().is_cancelled()); + let result = match completed { + Some(result) => result, + None => load.await, + }; + (outcome, result) }); tokio::task::yield_now().await; - // The setup assertion starts after attachment, regardless of how long - // admission, hashing, and native loading take to establish that precondition. tokio::time::advance(Duration::from_secs(3)).await; tokio::task::yield_now().await; - assert!( - !wait.is_finished(), - "valid preload was mistaken for setup failure" - ); - release.send(()).unwrap(); + let waiting = !wait.is_finished(); + let released = release.send(()); attached.notify_one(); - wait.await.unwrap(); + let joined = wait.await; + assert!(waiting, "valid preload was mistaken for setup failure"); + assert!(released.is_ok()); + let (outcome, result) = joined.unwrap(); + assert_eq!(outcome, AttachOutcome::Attached); + assert!(result.unwrap_err().is_cancelled()); } #[tokio::test] async fn setup_attachment_wait_reports_an_early_load_failure() { - let attached = Arc::new(Notify::new()); + let attached = Notify::new(); let mut load = tokio::spawn(async { Err::<(), _>(Error::ConnectionClosed) }); - let wait = tokio::spawn(async move { - await_native_attach(&attached, &mut load).await; - }); - let panic = wait.await.unwrap_err().into_panic(); - let message = panic.downcast_ref::().unwrap(); - assert!(message.contains("load ended before attach"), "{message}"); - assert!(message.contains("ConnectionClosed"), "{message}"); + let mut completed = None; + let outcome = await_native_attach(&attached, &mut load, &mut completed).await; + let result = completed.unwrap(); + assert_eq!(outcome, AttachOutcome::LoadRefused); + assert!(matches!(result, Ok(Err(Error::ConnectionClosed)))); } #[tokio::test(start_paused = true)] async fn setup_attachment_watchdog_cancels_a_stuck_preload() { let attached = Notify::new(); - let mut load = tokio::spawn(std::future::pending::<()>()); - let cancellation = load.abort_handle(); + let mut load = tokio::spawn(std::future::pending::>()); let wait = tokio::spawn(async move { - await_native_attach(&attached, &mut load).await; + let mut completed = None; + let outcome = await_native_attach(&attached, &mut load, &mut completed).await; + // Actual task completion is consumed. This synthetic task is cooperative; + // this is not evidence of synchronous/native cancellation or C return. + let result = match completed { + Some(result) => result, + None => load.await, + }; + (outcome, result) }); tokio::task::yield_now().await; tokio::time::advance(Duration::from_secs(61)).await; - let panic = wait.await.unwrap_err().into_panic(); - let message = panic.downcast_ref::<&str>().unwrap(); - assert_eq!(*message, "native callback fixture never attached"); - tokio::task::yield_now().await; - assert!( - cancellation.is_finished(), - "stuck preload must be cancelled" - ); + let (outcome, result) = wait.await.unwrap(); + assert_eq!(outcome, AttachOutcome::WatchdogExpired); + assert!(result.unwrap_err().is_cancelled()); } async fn native_callback_setup_failure_race(panic_setup: bool) { @@ -945,9 +1031,11 @@ async fn native_callback_setup_failure_race(panic_setup: bool) { let attached = Arc::new(Notify::new()); let seen = attached.clone(); let (resume, release) = std::sync::mpsc::channel(); + let mut cleanup = ControlledRelease::new(barrier.clone()); + cleanup.finish = Some(resume); *binding.after_attach.lock().unwrap() = Some(Box::new(move || { seen.notify_one(); - release.recv().unwrap(); + let _ = release.recv(); assert!( !panic_setup, "actual Rust unwind during host callback setup" @@ -960,18 +1048,34 @@ async fn native_callback_setup_failure_race(panic_setup: bool) { .load_file_with_host_callbacks(artifact, config, &loading_binding) .await }); - await_native_attach(&attached, &mut load).await; + let mut completed = None; + let attachment = await_native_attach(&attached, &mut load, &mut completed).await; + if attachment != AttachOutcome::Attached { + cleanup.release(); + // Before attachment C is paused. Preserve the original finite cause + // even if synchronous admission delays cooperative abort completion. + let completion = match completed { + Some(result) => result, + None => load.await, + }; + drop(completion); + panic!("native fixture attachment: {attachment:?}"); + } let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + cleanup.transport = Some(transport.clone()); // Actual ordinary lifecycle ownership, before the callback initialization // gate opens. This instance proves C never entered, rather than any join. - let _ = transport.clone().stop(Duration::from_secs(2)).await; - assert!(!transport.is_faulted()); + let stopped = transport.clone().stop(Duration::from_secs(2)).await; + let held_fault = transport.is_faulted(); if !panic_setup { // Public revocation makes real setup return its concurrently-used error. binding.revoke(); } - resume.send(()).unwrap(); + cleanup.release(); let result = load.await; + cleanup.transport = None; + assert!(stopped.is_ok()); + assert!(!held_fault); if panic_setup { assert!(result.unwrap_err().is_panic()); } else { diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 038f9d56..cb9b0963 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1498,11 +1498,17 @@ unsafe impl Send for NativeBarrier {} unsafe impl Sync for NativeBarrier {} unsafe extern "C" fn native_entered(ctx: *mut c_void, host: *const TbHostVtable) { let barrier = unsafe { &*ctx.cast::() }; - let mut state = barrier.state.lock().unwrap(); + let mut state = barrier + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); state.0 = Some(unsafe { *host }); barrier.entered.notify_one(); while !state.1 { - state = barrier.changed.wait(state).unwrap(); + state = barrier + .changed + .wait(state) + .unwrap_or_else(std::sync::PoisonError::into_inner); } } unsafe extern "C" fn native_joined(ctx: *mut c_void) { @@ -1512,9 +1518,15 @@ unsafe extern "C" fn native_joined(ctx: *mut c_void) { state.2 = true; barrier.joined.notify_one(); drop(state); - let gate = barrier.finish_gate.lock().unwrap().take(); + let gate = barrier + .finish_gate + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + // This artificial gate runs after the fixture's actual worker join. A failed + // Rust assertion may disconnect it; never unwind through the C callback. if let Some(gate) = gate { - gate.recv().unwrap(); + let _ = gate.recv(); } } pub(crate) fn native_fixture() -> ( From 6ec8a2d0ad94ed4f12f4bd357378604a25affd59 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 14:02:00 +0300 Subject: [PATCH 008/130] test(tinybus): cover native early unwind and finish gate Add an ignored test that exercises the controlled-release guard during an ordinary Rust unwind while native entry is blocked, asserting that stop completes and the barrier is reclaimed. Rework the native fixture attachment failure path to report the original cause without turning it into a readiness failure, and make the artificial finish gate tolerate a poisoned lock and a disconnected channel so a failed assertion cannot unwind through the C callback. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/module/host_callbacks_tests.rs | 70 ++++++++++++++++--- crates/tinybus/src/module/transport_tests.rs | 15 +++- 2 files changed, 76 insertions(+), 9 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index da149c67..d386f270 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -645,6 +645,49 @@ fn controlled_release_wakes_entry_and_releases_channels_on_early_unwind() { ); } +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_early_unwind_releases_entry_and_reclaims_after_callback_return() { + use super::super::super::transport::tests::native_fixture; + let (transport, host, init, barrier) = native_fixture(); + let weak = Arc::downgrade(&barrier); + let mut cleanup = ControlledRelease::new(barrier.clone()); + cleanup.transport = Some(transport.clone()); + let (finish, receive) = std::sync::mpsc::channel(); + cleanup.finish = Some(finish); + *barrier.finish_gate.lock().unwrap() = Some(receive); + transport.defer_initialize(init, host); + let initializing = tokio::spawn({ + let transport = transport.clone(); + async move { transport.ensure_initialized().await } + }); + let entered = tokio::time::timeout(Duration::from_secs(2), barrier.entered.notified()).await; + // Deliberate ordinary Rust unwind exercises the guard while actual C entry + // is blocked on its condition variable. No panic crosses a C boundary. + let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || { + let _cleanup = cleanup; + panic!("early controlled fixture failure"); + })); + let initialized = initializing.await; + let stopped = transport.clone().stop(Duration::from_secs(2)).await; + let callback_entered_after_worker_join = barrier.state.lock().unwrap().2; + drop(barrier); + drop(transport); + let reclaimed = tokio::time::timeout(Duration::from_secs(2), async { + while weak.strong_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await; + // Stop completion, rather than callback entry or abort, proves actual return. + assert!(entered.is_ok()); + assert!(unwind.is_err()); + assert!(initialized.is_ok()); + assert!(stopped.is_ok()); + assert!(callback_entered_after_worker_join); + assert!(reclaimed.is_ok()); +} + // Manually polling the public future lets ordinary shutdown commit while the // loader is suspended; no scheduler race decides which cancellation is tested. async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: bool) { @@ -1052,14 +1095,25 @@ async fn native_callback_setup_failure_race(panic_setup: bool) { let attachment = await_native_attach(&attached, &mut load, &mut completed).await; if attachment != AttachOutcome::Attached { cleanup.release(); - // Before attachment C is paused. Preserve the original finite cause - // even if synchronous admission delays cooperative abort completion. - let completion = match completed { - Some(result) => result, - None => load.await, - }; - drop(completion); - panic!("native fixture attachment: {attachment:?}"); + // Report the original finite cause before cleanup: synchronous admission + // cannot be interrupted by abort. Do not turn it into a readiness failure. + eprintln!("native fixture attachment: {attachment:?}"); + if let Some(completion) = completed { + drop(completion); + } else if tokio::time::timeout(Duration::from_secs(2), &mut load) + .await + .is_err() + { + // Pending ownership is explicit, not a quiescence claim. The task + // consumes eventual completion and keeps the real staging, binding, + // host and controller alive; transport's physical anchor also remains. + tokio::spawn(async move { + let _owners = (_dir, host, binding, barrier); + let _ = load.await; + }); + panic!("native fixture attachment: {attachment:?}; cleanup pending"); + } + panic!("native fixture attachment: {attachment:?}; loader completion consumed"); } let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); cleanup.transport = Some(transport.clone()); diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index cb9b0963..62c30817 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -1513,7 +1513,10 @@ unsafe extern "C" fn native_entered(ctx: *mut c_void, host: *const TbHostVtable) } unsafe extern "C" fn native_joined(ctx: *mut c_void) { let barrier = unsafe { &*ctx.cast::() }; - let mut state = barrier.state.lock().unwrap(); + let mut state = barrier + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); assert!(!state.2, "one actual native shutdown/join"); state.2 = true; barrier.joined.notify_one(); @@ -1525,10 +1528,20 @@ unsafe extern "C" fn native_joined(ctx: *mut c_void) { .take(); // This artificial gate runs after the fixture's actual worker join. A failed // Rust assertion may disconnect it; never unwind through the C callback. + finish_native_control(gate); +} +fn finish_native_control(gate: Option>) { if let Some(gate) = gate { let _ = gate.recv(); } } +#[test] +fn disconnected_artificial_finish_gate_does_not_unwind() { + let (release, receive) = std::sync::mpsc::channel(); + drop(release); + let result = std::panic::catch_unwind(|| finish_native_control(Some(receive))); + assert!(result.is_ok()); +} pub(crate) fn native_fixture() -> ( Arc, TbHostVtable, From 1250dde19bb829256a1a3c63ff2533b13779c8f7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 14:03:53 +0300 Subject: [PATCH 009/130] test(tinybus): assert loader poll outcomes instead of panicking in poll_fn The native callback tests now capture the loader's poll result and assert on it after cleanup, so a premature completion or an expired entry watchdog releases the fixture and consumes the completion before failing rather than panicking inside the poll closure. Initialization results are also asserted after the transport is released, keeping the failure path leak-free. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/module/host_callbacks_tests.rs | 70 ++++++++++--------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index d386f270..391f76e2 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -720,32 +720,33 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: let mut load = Some(Box::pin( host.load_file_with_host_callbacks(&artifact, config, &binding), )); - std::future::poll_fn(|cx| { - assert!( - load.as_mut().unwrap().as_mut().poll(cx).is_pending(), - "loader awaits real C init" - ); - std::task::Poll::Ready(()) - }) - .await; - tokio::time::timeout(Duration::from_secs(2), barrier.entered.notified()) - .await - .unwrap(); + let first = + std::future::poll_fn(|cx| std::task::Poll::Ready(load.as_mut().unwrap().as_mut().poll(cx))) + .await; + if let std::task::Poll::Ready(completion) = first { + cleanup.release(); + drop(completion); + drop(load.take()); // already completed, no pending owner is discarded + panic!("native fixture completed before C entry"); + } + let entered = tokio::time::timeout(Duration::from_secs(2), barrier.entered.notified()).await; + if entered.is_err() { + cleanup.release(); + let completion = load.take().unwrap().await; + drop(completion); + panic!("native fixture entry watchdog expired; loader completion consumed"); + } let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); cleanup.transport = Some(transport.clone()); let view = barrier.state.lock().unwrap().0.unwrap(); barrier.state.lock().unwrap().1 = true; barrier.changed.notify_all(); - transport.ensure_initialized().await.unwrap(); + let initialized = transport.ensure_initialized().await; // Advance the public future to its real withheld-Ready observation. - std::future::poll_fn(|cx| { - assert!( - load.as_mut().unwrap().as_mut().poll(cx).is_pending(), - "fixture withholds Ready" - ); - std::task::Poll::Ready(()) - }) - .await; + let withheld = + std::future::poll_fn(|cx| std::task::Poll::Ready(load.as_mut().unwrap().as_mut().poll(cx))) + .await; + let was_withheld = withheld.is_pending(); let (finish, receive) = std::sync::mpsc::channel(); *barrier.finish_gate.lock().unwrap() = Some(receive); cleanup.finish = Some(finish); @@ -755,7 +756,9 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: .await .unwrap(); let stopped_flags = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; - let load_result = if reject_gate { + let load_result = if let std::task::Poll::Ready(result) = withheld { + Some(result) + } else if reject_gate { transport.release_initialization(false); drop(load.take()); None @@ -783,6 +786,8 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: None }; cleanup.transport = None; + assert!(initialized.is_ok()); + assert!(was_withheld, "fixture withholds Ready"); assert!(stopped.unwrap().is_ok()); if let Some(result) = loaded { assert!(result.is_err()); @@ -874,17 +879,17 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { // init_started precedes actual publication; the observer result is the proof. let initialized = transport.ensure_initialized(); tokio::pin!(initialized); - std::future::poll_fn( - |cx| match std::future::Future::poll(initialized.as_mut(), cx) { - std::task::Poll::Ready(result) => std::task::Poll::Ready(result), - std::task::Poll::Pending => { - cx.waker().wake_by_ref(); - std::task::Poll::Pending - } - }, - ) - .await - .unwrap(); + let initialization = + std::future::poll_fn( + |cx| match std::future::Future::poll(initialized.as_mut(), cx) { + std::task::Poll::Ready(result) => std::task::Poll::Ready(result), + std::task::Poll::Pending => { + cx.waker().wake_by_ref(); + std::task::Poll::Pending + } + }, + ) + .await; if !ready { // Real wait_ready deadline, not a fake readiness flag. Let the public // future install its timer before advancing past the actual bound. @@ -923,6 +928,7 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { let stopped = stop.await; let loaded = load.await; cleanup.transport = None; + assert!(initialization.is_ok()); assert!(resumed.is_ok()); assert!(stopped.unwrap().is_ok()); assert!(loaded.unwrap().is_err()); From 13d87e7ad5c1446fc0fe2279aece55f1d3d1b8f2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 14:08:10 +0300 Subject: [PATCH 010/130] test(module): cover release owner surviving canceled waiter Rework the controlled release guard to commit and schedule synchronously via request_terminal instead of spawning on the current Tokio runtime, so the physical owner is retained even when the runtime is dropped. Add an ignored native test that cancels a stop waiter and drops the runtime to verify the owner is reclaimed only after the callback returns, and relax the early-unwind assertion to accept a not-started connection-closed stop. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/module/host_callbacks_tests.rs | 72 ++++++++++++++++--- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 391f76e2..4bafc0ed 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -594,17 +594,13 @@ impl ControlledRelease { impl Drop for ControlledRelease { fn drop(&mut self) { self.release(); - // A failing test cannot join from Drop. The existing transport physical - // anchor remains authoritative; this task retains the real owner while - // requesting stop after all artificial gates have opened. + // Commit and schedule synchronously, before any Tokio cleanup observer + // could be canceled. request_terminal anchors PhysicalJob in the actual + // context and submits its OS thread; neither depends on this runtime. + // Only fixture-local locks can be poisoned by test assertions; release + // above recovers those. No SDK internal lock is held by this guard. if let Some(transport) = self.transport.take() { - if let Ok(runtime) = tokio::runtime::Handle::try_current() { - let barrier = self.barrier.clone(); - runtime.spawn(async move { - let _ = transport.stop(Duration::from_secs(2)).await; - drop(barrier); - }); - } + transport.request_terminal(false); } } } @@ -683,11 +679,65 @@ async fn actual_native_early_unwind_releases_entry_and_reclaims_after_callback_r assert!(entered.is_ok()); assert!(unwind.is_err()); assert!(initialized.is_ok()); - assert!(stopped.is_ok()); + assert!(matches!( + stopped, + Ok(_) + | Err(super::super::super::transport::StopError::NotStarted( + Error::ConnectionClosed + )) + )); assert!(callback_entered_after_worker_join); assert!(reclaimed.is_ok()); } +#[test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +fn actual_native_release_owner_survives_canceled_waiter_and_runtime_drop() { + use super::super::super::transport::tests::native_fixture; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + let (transport, host, init, barrier) = native_fixture(); + barrier.state.lock().unwrap().1 = true; + transport.defer_initialize(init, host); + let initialized = runtime.block_on(transport.ensure_initialized()); + let weak = Arc::downgrade(&barrier); + let (finish, receive) = std::sync::mpsc::channel(); + *barrier.finish_gate.lock().unwrap() = Some(receive); + let mut cleanup = ControlledRelease::new(barrier.clone()); + cleanup.transport = Some(transport.clone()); + // This test's external finish control stays held to inspect physical owner + // retention. It is released before any assertions, including failure paths. + drop(cleanup); + let waiter = runtime.spawn({ + let transport = transport.clone(); + async move { transport.stop(Duration::from_secs(2)).await } + }); + waiter.abort(); // intentionally cancel without relying on a first poll + let canceled = runtime.block_on(waiter); + let joined_worker = runtime.block_on(async { + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await + }); + drop(runtime); + drop(barrier); + drop(transport); + let retained_during_callback = weak.strong_count() != 0; + let released = finish.send(()); + // The physical thread is independent of the dropped Tokio runtime. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while weak.strong_count() != 0 && std::time::Instant::now() < deadline { + std::thread::yield_now(); + } + let reclaimed_after_return = weak.strong_count() == 0; + assert!(initialized.is_ok()); + assert!(canceled.unwrap_err().is_cancelled()); + assert!(joined_worker.is_ok()); + assert!(retained_during_callback); + assert!(released.is_ok()); + assert!(reclaimed_after_return); +} + // Manually polling the public future lets ordinary shutdown commit while the // loader is suspended; no scheduler race decides which cancellation is tested. async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: bool) { From 6062ba851209971f61cacd57cf66306f02e1146d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 14:10:49 +0300 Subject: [PATCH 011/130] test(module): distinguish load failure kinds in attach tests The attach test helper now maps module refusals, closed connections, and other load errors to separate outcomes instead of lumping every error into LoadRefused. A new test covers a genuine admission refusal before the C entry point, asserting the module is not attached or published as Ready. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/module/host_callbacks_tests.rs | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 4bafc0ed..00d1a6e2 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -1016,6 +1016,8 @@ enum AttachOutcome { Attached, LoadSucceeded, LoadRefused, + LoadConnectionClosed, + LoadFailed, LoadPanicked, LoadCanceled, WatchdogExpired, @@ -1035,7 +1037,9 @@ async fn await_native_attach( result = &mut *load => { let category = match &result { Ok(Ok(_)) => AttachOutcome::LoadSucceeded, - Ok(Err(_)) => AttachOutcome::LoadRefused, + Ok(Err(Error::ModuleRefused { .. })) => AttachOutcome::LoadRefused, + Ok(Err(Error::ConnectionClosed)) => AttachOutcome::LoadConnectionClosed, + Ok(Err(_)) => AttachOutcome::LoadFailed, Err(error) if error.is_cancelled() => AttachOutcome::LoadCanceled, Err(_) => AttachOutcome::LoadPanicked, }; @@ -1089,10 +1093,36 @@ async fn setup_attachment_wait_reports_an_early_load_failure() { let mut completed = None; let outcome = await_native_attach(&attached, &mut load, &mut completed).await; let result = completed.unwrap(); - assert_eq!(outcome, AttachOutcome::LoadRefused); + assert_eq!(outcome, AttachOutcome::LoadConnectionClosed); assert!(matches!(result, Ok(Err(Error::ConnectionClosed)))); } +#[tokio::test] +async fn setup_attachment_wait_preserves_actual_admission_refusal_before_c_entry() { + let host = Arc::new(ModuleHost::new(Broker::new())); + let binding = Arc::new(served_native_binding(&host).await); + let attached = Notify::new(); + let directory = tempfile::tempdir().unwrap(); + let missing = directory.path().join("absent-module.so"); + let mut load = tokio::spawn({ + let host = host.clone(); + let binding = binding.clone(); + async move { + host.load_file_with_host_callbacks(missing, serde_json::json!({}), &binding) + .await + } + }); + let mut completed = None; + let outcome = await_native_attach(&attached, &mut load, &mut completed).await; + let completion = completed.unwrap(); + assert_eq!(outcome, AttachOutcome::LoadRefused); + assert!(matches!(completion, Ok(Err(Error::ModuleRefused { .. })))); + assert!( + host.list().is_empty(), + "refusal cannot attach or publish Ready" + ); +} + #[tokio::test(start_paused = true)] async fn setup_attachment_watchdog_cancels_a_stuck_preload() { let attached = Notify::new(); From 30f9a205aa89b2d18c77148667669aba4ce484c8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 14:26:21 +0300 Subject: [PATCH 012/130] test(transport): hold the vtable fixture lock until physical cleanup The closed-delivery fixture now releases the shared vtable lock only after the transport's physical cleanup has actually run, and a new test asserts that a fault reply does not let the next fixture start early. This keeps the global vtable state serialized through its final writes instead of freeing it as soon as the error reply is observed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 79 +++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 62c30817..12ca2b3b 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -341,7 +341,7 @@ async fn a_failed_initializer_is_reported_to_callers() { #[tokio::test(flavor = "current_thread")] async fn a_pending_call_is_failed_when_the_module_closes_its_delivery_queue() { - let _lock = VTABLE_TEST_LOCK + let lock = VTABLE_TEST_LOCK .get_or_init(|| tokio::sync::Mutex::new(())) .lock() .await; @@ -357,7 +357,84 @@ async fn a_pending_call_is_failed_when_the_module_closes_its_delivery_queue() { .unwrap(); assert_eq!(reply.header.kind, MessageKind::Error); assert!(transport.is_faulted()); + finish_closed_delivery_fixture(&transport, lock).await; +} + + +// Keep the shared vtable fixture serialized through its final global writes. +async fn finish_closed_delivery_fixture( + _transport: &ModuleTransport, + lock: tokio::sync::MutexGuard<'_, ()>, +) { DELIVERY_CODE.store(TB_OK, Ordering::Release); + drop(lock); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_fault_reply_does_not_release_the_fixture_before_physical_cleanup() { + use std::future::Future; + + struct WorkerRelease(Option>); + impl Drop for WorkerRelease { + fn drop(&mut self) { + if let Some(release) = self.0.take() { + let _ = release.send(()); + } + } + } + + let lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("serialized-fault-cleanup".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.unwrap(); + let (started, started_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); + let worker_release = WorkerRelease(Some(release)); + *transport.before_worker_lock.lock().unwrap() = Some(Box::new(move || { + let _ = started.send(()); + let _ = release_rx.recv(); + })); + DELIVERY_CODE.store(TB_CLOSED, Ordering::Release); + transport.send(call()).await.unwrap(); + unsafe { (host.ready)(host.host_ctx) }; + // The actual independently owned fault worker cannot acquire lifecycle yet. + let reply = tokio::time::timeout(Duration::from_secs(2), transport.recv()).await; + let started = tokio::time::timeout(Duration::from_secs(2), started_rx).await; + let faulted_before_cleanup = transport.is_faulted(); + let callback_pending = !SHUTDOWN_ENTERED.load(Ordering::Acquire) + && !matches!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Complete | CleanupState::Uncertain | CleanupState::NoCallbackRequired + ); + let mut completion = Box::pin(finish_closed_delivery_fixture(&transport, lock)); + let completed_early = std::future::poll_fn(|cx| { + std::task::Poll::Ready(completion.as_mut().poll(cx).is_ready()) + }) + .await; + let released_early = VTABLE_TEST_LOCK.get().unwrap().try_lock().is_ok(); + + // Release and observe the real callback before any regression assertion. + // The transport retains the actual context; cancelling an observer is not cleanup. + drop(worker_release); + wait_physical_cleanup(&transport).await; + if !completed_early { + completion.await; + } + let next_fixture_can_start = VTABLE_TEST_LOCK.get().unwrap().try_lock().is_ok(); + assert!(matches!(reply, Ok(Ok(Some(ref reply))) if reply.header.kind == MessageKind::Error)); + assert!(matches!(started, Ok(Ok(())))); + assert!(faulted_before_cleanup && callback_pending); + assert!(SHUTDOWN_ENTERED.load(Ordering::Acquire)); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1); + assert!(!completed_early, "fault reply is not physical fixture completion"); + assert!(!released_early, "next fixture must wait for actual cleanup"); + assert!(next_fixture_can_start); } #[tokio::test] From de2d5300f7ffd5a29940d547469ec86b74e2657a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 14:28:21 +0300 Subject: [PATCH 013/130] test(transport): await physical cleanup before releasing fixture lock The shared vtable fixture now waits for physical cleanup before its final global writes, and the fault-reply test awaits its turn on the fixture lock instead of treating legitimate contention after release as a failure. This keeps the serialized fixture assertions from racing with other queued tests. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 35 ++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index 12ca2b3b..e621930c 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -360,12 +360,12 @@ async fn a_pending_call_is_failed_when_the_module_closes_its_delivery_queue() { finish_closed_delivery_fixture(&transport, lock).await; } - // Keep the shared vtable fixture serialized through its final global writes. async fn finish_closed_delivery_fixture( - _transport: &ModuleTransport, + transport: &ModuleTransport, lock: tokio::sync::MutexGuard<'_, ()>, ) { + wait_physical_cleanup(transport).await; DELIVERY_CODE.store(TB_OK, Ordering::Release); drop(lock); } @@ -387,7 +387,6 @@ async fn a_fault_reply_does_not_release_the_fixture_before_physical_cleanup() { .get_or_init(|| tokio::sync::Mutex::new(())) .lock() .await; - let _release = ShutdownRelease; SHUTDOWN_ENTERED.store(false, Ordering::Release); SHUTDOWN_CALLS.store(0, Ordering::Release); let (transport, host) = ModuleTransport::new("serialized-fault-cleanup".into(), vec![]); @@ -413,28 +412,38 @@ async fn a_fault_reply_does_not_release_the_fixture_before_physical_cleanup() { CleanupState::Complete | CleanupState::Uncertain | CleanupState::NoCallbackRequired ); let mut completion = Box::pin(finish_closed_delivery_fixture(&transport, lock)); - let completed_early = std::future::poll_fn(|cx| { - std::task::Poll::Ready(completion.as_mut().poll(cx).is_ready()) - }) - .await; + let completed_early = + std::future::poll_fn(|cx| std::task::Poll::Ready(completion.as_mut().poll(cx).is_ready())) + .await; let released_early = VTABLE_TEST_LOCK.get().unwrap().try_lock().is_ok(); // Release and observe the real callback before any regression assertion. // The transport retains the actual context; cancelling an observer is not cleanup. drop(worker_release); wait_physical_cleanup(&transport).await; + let callback_entered = SHUTDOWN_ENTERED.load(Ordering::Acquire); + let callback_calls = SHUTDOWN_CALLS.load(Ordering::Acquire); if !completed_early { completion.await; } - let next_fixture_can_start = VTABLE_TEST_LOCK.get().unwrap().try_lock().is_ok(); + // Another queued fixture can acquire first; await our turn rather than + // treating legitimate contention after release as a failure. + let next_fixture = tokio::time::timeout( + Duration::from_secs(2), + VTABLE_TEST_LOCK.get().unwrap().lock(), + ) + .await; assert!(matches!(reply, Ok(Ok(Some(ref reply))) if reply.header.kind == MessageKind::Error)); assert!(matches!(started, Ok(Ok(())))); assert!(faulted_before_cleanup && callback_pending); - assert!(SHUTDOWN_ENTERED.load(Ordering::Acquire)); - assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1); - assert!(!completed_early, "fault reply is not physical fixture completion"); + assert!(callback_entered); + assert_eq!(callback_calls, 1); + assert!( + !completed_early, + "fault reply is not physical fixture completion" + ); assert!(!released_early, "next fixture must wait for actual cleanup"); - assert!(next_fixture_can_start); + assert!(next_fixture.is_ok()); } #[tokio::test] @@ -845,11 +854,11 @@ async fn stopping_initialized_module_before_ready_releases_observers_and_queued_ #[tokio::test] async fn ordinary_stop_wakes_backpressure_without_turning_closure_into_a_fault() { - let _release = ShutdownRelease; let _lock = VTABLE_TEST_LOCK .get_or_init(|| tokio::sync::Mutex::new(())) .lock() .await; + let _release = ShutdownRelease; SHUTDOWN_BLOCKED.store(false, Ordering::Release); DELIVERY_CODE.store(TB_BACKPRESSURE, Ordering::Release); DELIVERIES.store(0, Ordering::Release); From 39036ce7db6ced65607ea2fba5a1303375378ea3 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 14:38:00 +0300 Subject: [PATCH 014/130] test(transport): defer assertions until after terminal cleanup The pending-call test now captures the receive result and fault state before requesting terminal cleanup, then asserts afterwards. This keeps the fixture's global writes serialized so the test no longer races with cleanup. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/transport_tests.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index e621930c..dbcbfa26 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -350,14 +350,15 @@ async fn a_pending_call_is_failed_when_the_module_closes_its_delivery_queue() { transport.defer_initialize(initialize_ok, host); transport.send(call()).await.unwrap(); unsafe { (host.ready)(host.host_ctx) }; - let reply = tokio::time::timeout(Duration::from_secs(1), transport.recv()) - .await - .unwrap() - .unwrap() - .unwrap(); - assert_eq!(reply.header.kind, MessageKind::Error); - assert!(transport.is_faulted()); + let reply = tokio::time::timeout(Duration::from_secs(1), transport.recv()).await; + let faulted = transport.is_faulted(); + // A failed receive may have no fault-owned cleanup job. Commit cleanup + // before reporting the captured result; this does not clear an actual fault. + transport.request_terminal(false); finish_closed_delivery_fixture(&transport, lock).await; + let reply = reply.unwrap().unwrap().unwrap(); + assert_eq!(reply.header.kind, MessageKind::Error); + assert!(faulted); } // Keep the shared vtable fixture serialized through its final global writes. From 5a0c17e9e0ffda2c282dc6e751065394fcced0ad Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 14:53:56 +0300 Subject: [PATCH 015/130] test(module): cover host stop drain timeout and cancellation Extend the host callback lifecycle tests to exercise stop drain timeout and cancellation paths alongside the existing complete-drain case, asserting that stop intent is retained, new admission is denied, and the module owner is kept until admitted work finishes. Add a test-only accessor for the native terminal state so the tests can observe that authority closes before provider work drains. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../module/host_callback_lifecycle_tests.rs | 111 +++++++++++++++--- crates/tinybus/src/module/transport.rs | 5 + 2 files changed, 98 insertions(+), 18 deletions(-) diff --git a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs index fc6bf84e..5aae9db1 100644 --- a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs +++ b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs @@ -5,6 +5,7 @@ struct CommittingCallback { entered: tokio::sync::mpsc::Sender<()>, release: Arc, committed: Arc, + context: Arc>>, } #[async_trait::async_trait] impl crate::Interface for CommittingCallback { @@ -25,6 +26,7 @@ impl crate::Interface for CommittingCallback { ) -> Result { assert!(context.is_confidential()); assert!(context.authenticated_sender().is_some()); + *self.context.lock().unwrap() = Some(context.clone()); self.entered.send(()).await.unwrap(); let _release = self.release.acquire().await.unwrap(); // A native port may already have admitted its transaction. Revoking @@ -34,9 +36,39 @@ impl crate::Interface for CommittingCallback { } } +#[derive(Clone, Copy)] +enum DrainStop { + Complete, + Timeout, + Cancel, +} + +struct CallbackRelease(Arc); +impl Drop for CallbackRelease { + fn drop(&mut self) { + self.0.add_permits(1); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires TINYBUS_TEST_CONTEXT_MODULE to point at the built cdylib"] async fn stopping_a_verified_module_drains_its_admitted_host_callback_before_native_shutdown() { + stop_after_admitted_callback(DrainStop::Complete).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTEXT_MODULE to point at the built cdylib"] +async fn host_stop_drain_timeout_retains_stop_intent_and_denies_new_admission() { + stop_after_admitted_callback(DrainStop::Timeout).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTEXT_MODULE to point at the built cdylib"] +async fn host_stop_drain_cancellation_retains_stop_intent_and_denies_new_admission() { + stop_after_admitted_callback(DrainStop::Cancel).await; +} + +async fn stop_after_admitted_callback(mode: DrainStop) { let artifact = PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTEXT_MODULE").unwrap()); let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); let (_dir, artifact) = staged_module(&artifact, &hash); @@ -52,6 +84,7 @@ async fn stopping_a_verified_module_drains_its_admitted_host_callback_before_nat let (entered, mut receiver) = tokio::sync::mpsc::channel(1); let release = Arc::new(tokio::sync::Semaphore::new(0)); let committed = Arc::new(AtomicUsize::new(0)); + let context = Arc::new(StdMutex::new(None)); binding .serve_at( "/callback".parse().unwrap(), @@ -59,6 +92,7 @@ async fn stopping_a_verified_module_drains_its_admitted_host_callback_before_nat entered, release: release.clone(), committed: committed.clone(), + context: context.clone(), }, ) .await @@ -75,8 +109,10 @@ async fn stopping_a_verified_module_drains_its_admitted_host_callback_before_nat ) .unwrap() .with_timeout(Duration::from_secs(2)); + let release_on_failure = CallbackRelease(release.clone()); + let first_proxy = proxy.clone(); let request = tokio::spawn(async move { - proxy + first_proxy .call_confidential::("ConfidentialCallback", ()) .await }); @@ -84,35 +120,74 @@ async fn stopping_a_verified_module_drains_its_admitted_host_callback_before_nat .await .unwrap() .unwrap(); - assert_eq!(binding.guard().running(), 1); let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); - let stopping = host.inner.stop(&loaded.name, Duration::from_secs(2)); - tokio::pin!(stopping); - let completed_early = std::future::poll_fn(|context| { - std::task::Poll::Ready(std::future::Future::poll(stopping.as_mut(), context).is_ready()) + let deadline = if matches!(mode, DrainStop::Timeout) { + Duration::ZERO + } else { + Duration::from_secs(2) + }; + let mut stopping = Some(Box::pin(host.inner.stop(&loaded.name, deadline))); + let mut early = std::future::poll_fn(|cx| { + std::task::Poll::Ready( + match std::future::Future::poll(stopping.as_mut().unwrap().as_mut(), cx) { + std::task::Poll::Ready(result) => Some(result), + std::task::Poll::Pending => None, + }, + ) }) .await; + if !matches!(mode, DrainStop::Complete) { + drop(stopping.take()); + } + let terminal_while_held = transport.native_terminal_state_for_test(); let shutdown_started_early = transport.stop_task_spawned(); + let physically_stopped_early = + terminal_while_held & crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED != 0; let still_running = binding.guard().running(); - // Always release the actual worker before asserting the intended RED. - release.add_permits(1); + let authority_denied = !binding + .guard() + .authorizes(context.lock().unwrap().as_ref().unwrap()); + let owner_retained = host + .inner + .loaded + .lock() + .unwrap() + .iter() + .any(|module| Arc::ptr_eq(&module.transport, &transport)); + let enable_denied = host.inner.enable(&loaded.name, true).is_err(); + let new_call_denied = proxy + .call_confidential::("ConfidentialCallback", ()) + .await + .is_err(); + // Release and consume real callback/physical cleanup before all assertions. + drop(release_on_failure); tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()) .await .unwrap(); - if !completed_early { - stopping.await.unwrap(); - } + let stopped = if let Some(stopping) = stopping { + if let Some(result) = early.take() { + result + } else { + stopping.await + } + } else { + host.inner.stop(&loaded.name, Duration::from_secs(2)).await + }; let _outcome = request.await.unwrap(); + assert!(stopped.is_ok()); assert_eq!(committed.load(Ordering::Acquire), 1); assert_eq!(still_running, 1); - assert!( - !completed_early, - "stop completed before admitted callback commit" - ); - assert!( - !shutdown_started_early, - "native shutdown started before callback commit" + assert!(authority_denied && owner_retained && enable_denied && new_call_denied); + assert!(!shutdown_started_early && !physically_stopped_early); + assert_eq!( + terminal_while_held, + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT, + "native authority must close before admitted provider work drains" ); + assert!(!transport.is_faulted()); + if matches!(mode, DrainStop::Timeout) { + assert!(early.is_some_and(|result| result.is_err())); + } } #[tokio::test] diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 7202f116..1973e2ba 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -1059,6 +1059,11 @@ impl ModuleTransport { code } + #[cfg(test)] + pub(crate) fn native_terminal_state_for_test(&self) -> u32 { + unsafe { host_native_terminal_state(std::ptr::from_ref(self.context).cast_mut().cast()) } + } + #[cfg(test)] pub(crate) fn lifecycle_is_owned_for_test(&self) -> bool { self.lifecycle.try_lock().is_err() From e8098c6f3203987a187bb457a066e60181d2c195 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 14:56:10 +0300 Subject: [PATCH 016/130] fix(module): drain host callbacks before physical teardown Bind the transport to its host callback state so physical cleanup can wait for in-flight user futures to finish before releasing native exclusion. Admission is now revoked immediately on module stop, and the callback lifetime lock tolerates poisoning so a panicking callback cannot wedge shutdown. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host.rs | 3 ++ crates/tinybus/src/module/host_callbacks.rs | 37 ++++++++++++++++++--- crates/tinybus/src/module/transport.rs | 29 ++++++++++++---- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/crates/tinybus/src/module/host.rs b/crates/tinybus/src/module/host.rs index 0b6dc483..a0264030 100644 --- a/crates/tinybus/src/module/host.rs +++ b/crates/tinybus/src/module/host.rs @@ -1501,6 +1501,9 @@ impl ModuleControl for ModuleHostInner { .lock() .expect("router lock") .revoke_callbacks_for_peer(&module.unique_name); + // Revoke native admission immediately; provider drain owns the + // later shutdown handoff, so this must not submit a physical job. + module.transport.commit_terminal(false); module.transition_from.get_or_insert(old); module.lifecycle_busy = true; (module.transport.clone(), module.callback_state.clone()) diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 3108561c..5a530ffe 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -35,9 +35,26 @@ pub(crate) struct CallbackState { module_transport: Mutex>>, lifetime: Mutex, drained: Notify, + physical_drained: std::sync::Condvar, } impl CallbackState { + /// Physical cleanup waits offruntime, without a watchdog or native exclusion. + pub(crate) fn wait_drained(&self) { + let mut state = self + .lifetime + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.active = false; + state.revoked = true; + while state.running != 0 { + state = self + .physical_drained + .wait(state) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + pub(crate) async fn drain(self: &Arc, deadline: Duration) -> Result<()> { tokio::time::timeout(deadline, HostCallbackGuard(self.clone()).drained()) .await @@ -352,10 +369,15 @@ struct GuardedInterface { struct Running(Arc); impl Drop for Running { fn drop(&mut self) { - let mut state = self.0.lifetime.lock().expect("callback lifetime lock"); + let mut state = self + .0 + .lifetime + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); state.running -= 1; if state.running == 0 { self.0.drained.notify_waiters(); + self.0.physical_drained.notify_all(); } } } @@ -393,9 +415,14 @@ impl Interface for GuardedInterface { state.running += 1; Running(self.state.clone()) }; - self.interface - .call_with_context(member, args, context) - .await + // The later-created user future is destroyed before Running on both + // completion and cancellation; no response queue owns this accounting. + let result = { + let user = self.interface.call_with_context(member, args, context); + user.await + }; + drop(_running); + result } } @@ -465,6 +492,7 @@ impl ModuleHost { registering: Vec::new(), }), drained: Notify::new(), + physical_drained: std::sync::Condvar::new(), }), }) } @@ -547,6 +575,7 @@ impl ModuleHost { let fingerprint = callback_config_fingerprint(&bytes); drop(config); let (transport, host) = ModuleTransport::new(admitted.name.clone(), bytes); + transport.bind_host_callbacks(binding.state.clone())?; #[cfg(test)] { // Transfer the fixture config borrow before C can enter, so diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 1973e2ba..8d73b58f 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -3,7 +3,7 @@ use std::collections::VecDeque; use std::ffi::c_void; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use std::sync::{Arc, Condvar, Mutex as StdMutex, Weak}; +use std::sync::{Arc, Condvar, Mutex as StdMutex, OnceLock, Weak}; use std::time::Duration; use async_trait::async_trait; @@ -74,6 +74,8 @@ pub(crate) struct ModuleTransport { native_lifecycle_pending: Arc, stop_task_spawned: Arc, callbacks: CallbackGate, + // One exact provider account. Its transport backlink is weak. + host_callbacks: OnceLock>, initialization_gate: StdMutex>, initialization_gate_changed: Notify, #[cfg(test)] @@ -288,6 +290,7 @@ impl ModuleTransport { native_lifecycle_pending: Arc::new(AtomicBool::new(false)), stop_task_spawned: Arc::new(AtomicBool::new(false)), callbacks: CallbackGate::default(), + host_callbacks: OnceLock::new(), initialization_gate: StdMutex::new(Some(true)), initialization_gate_changed: Notify::new(), #[cfg(test)] @@ -767,8 +770,18 @@ impl ModuleTransport { } } + pub(crate) fn bind_host_callbacks( + &self, + callbacks: Arc, + ) -> Result<()> { + self.host_callbacks + .set(callbacks) + .map_err(|_| Error::failed("module callback binding is already installed")) + } + pub(crate) fn native_lifecycle_pending(&self) -> bool { self.native_lifecycle_pending.load(Ordering::Acquire) + || (self.admission_closed() && !self.is_stopped()) || self .context .physical @@ -1277,12 +1290,14 @@ impl PhysicalJob { .expect("physical state lock"); } } - let guard = self - .guard - .lock() - .expect("physical guard lock") - .take() - .unwrap_or_else(|| transport.lifecycle.clone().blocking_lock_owned()); + let mut acquired = self.guard.lock().expect("physical guard lock").take(); + if let Some(callbacks) = transport.host_callbacks.get() { + // User futures may need lifecycle services. Retain this physical + // owner, but release native exclusion before their actual drain. + drop(acquired.take()); + callbacks.wait_drained(); + } + let guard = acquired.unwrap_or_else(|| transport.lifecycle.clone().blocking_lock_owned()); { let mut state = transport .context From 67c0006190f50adbbc542090506a999f09d6e2a4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 14:59:57 +0300 Subject: [PATCH 017/130] test(module): cover host drop during cancelled stop drain Add a test-only notification that fires when the physical drain wait begins, and use it to assert that dropping the host after a cancelled stop still waits for the provider future to be destroyed before native cleanup completes. Also add tests that a transport rejects rebinding a different callback account and that runtime drop destroys the provider future before off-runtime native cleanup. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../module/host_callback_lifecycle_tests.rs | 50 +++++++ crates/tinybus/src/module/host_callbacks.rs | 6 + .../src/module/host_callbacks_tests.rs | 130 ++++++++++++++++++ 3 files changed, 186 insertions(+) diff --git a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs index 5aae9db1..4f133ff0 100644 --- a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs +++ b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs @@ -41,6 +41,7 @@ enum DrainStop { Complete, Timeout, Cancel, + DropHost, } struct CallbackRelease(Arc); @@ -68,6 +69,12 @@ async fn host_stop_drain_cancellation_retains_stop_intent_and_denies_new_admissi stop_after_admitted_callback(DrainStop::Cancel).await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTEXT_MODULE to point at the built cdylib"] +async fn host_drop_after_cancelled_stop_waits_for_actual_provider_drain() { + stop_after_admitted_callback(DrainStop::DropHost).await; +} + async fn stop_after_admitted_callback(mode: DrainStop) { let artifact = PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTEXT_MODULE").unwrap()); let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); @@ -159,6 +166,49 @@ async fn stop_after_admitted_callback(mode: DrainStop) { .call_confidential::("ConfidentialCallback", ()) .await .is_err(); + if matches!(mode, DrainStop::DropHost) { + let account = host.inner.loaded.lock().unwrap()[0] + .callback_state + .clone() + .unwrap(); + let host_owner = Arc::downgrade(&host.inner); + drop(stopping); + drop(host); + let entered = tokio::time::timeout( + Duration::from_secs(2), + account.physical_wait_started.notified(), + ) + .await; + let terminal_after_drop = transport.native_terminal_state_for_test(); + let still_counted = binding.guard().running(); + let lifecycle_free = !transport.lifecycle_is_owned_for_test(); + drop(release_on_failure); + tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()) + .await + .unwrap(); + let completed = tokio::time::timeout(Duration::from_secs(2), async { + while transport.native_terminal_state_for_test() + & crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED + == 0 + { + tokio::task::yield_now().await; + } + }) + .await; + let _outcome = request.await; + assert!(completed.is_ok()); + assert!(entered.is_ok()); + assert!(host_owner.upgrade().is_none()); + assert_eq!(still_counted, 1); + assert_eq!( + terminal_after_drop, + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT + ); + assert!(lifecycle_free && authority_denied && new_call_denied); + assert_eq!(committed.load(Ordering::Acquire), 1); + assert!(!transport.is_faulted()); + return; + } // Release and consume real callback/physical cleanup before all assertions. drop(release_on_failure); tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()) diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 5a530ffe..34cc3921 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -36,6 +36,8 @@ pub(crate) struct CallbackState { lifetime: Mutex, drained: Notify, physical_drained: std::sync::Condvar, + #[cfg(test)] + pub(crate) physical_wait_started: Notify, } impl CallbackState { @@ -47,6 +49,8 @@ impl CallbackState { .unwrap_or_else(std::sync::PoisonError::into_inner); state.active = false; state.revoked = true; + #[cfg(test)] + self.physical_wait_started.notify_one(); while state.running != 0 { state = self .physical_drained @@ -493,6 +497,8 @@ impl ModuleHost { }), drained: Notify::new(), physical_drained: std::sync::Condvar::new(), + #[cfg(test)] + physical_wait_started: Notify::new(), }), }) } diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 00d1a6e2..937ca505 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -25,6 +25,8 @@ fn state(source: BusName, destination: BusName) -> Arc { registering: vec![], }), drained: Notify::new(), + physical_drained: std::sync::Condvar::new(), + physical_wait_started: Notify::new(), }) } fn call(state: &CallbackState) -> Message { @@ -1241,3 +1243,131 @@ async fn actual_callback_setup_error_still_faults_after_ordinary_stop() { async fn actual_callback_setup_unwind_still_faults_after_ordinary_stop() { native_callback_setup_failure_race(true).await; } + +#[test] +fn a_transport_rejects_replacement_of_its_exact_callback_account() { + let first = state(":1.1".parse().unwrap(), ":1.2".parse().unwrap()); + let second = state(":1.3".parse().unwrap(), ":1.4".parse().unwrap()); + let original = Arc::downgrade(&first); + let replacement = Arc::downgrade(&second); + let (transport, _) = ModuleTransport::new("one-account".into(), vec![]); + transport.bind_host_callbacks(first.clone()).unwrap(); + let rejected = transport.bind_host_callbacks(second.clone()); + drop(first); + drop(second); + assert!(rejected.is_err()); + assert!(original.upgrade().is_some()); + assert!(replacement.upgrade().is_none()); +} + +struct DestructionObserved { + guard: HostCallbackGuard, + observed: Arc, +} +impl Drop for DestructionObserved { + fn drop(&mut self) { + self.observed.store(self.guard.running(), Ordering::Release); + } +} +struct PendingUntilDestroyed { + entered: Arc, + guard: HostCallbackGuard, + observed: Arc, +} +#[async_trait] +impl Interface for PendingUntilDestroyed { + fn name(&self) -> InterfaceName { + endpoint().interface + } + fn members(&self) -> Vec { + endpoint().members + } + async fn call(&self, _: &MemberName, _: serde_json::Value) -> Result { + Err(Error::failed("context required")) + } + async fn call_with_context( + &self, + _: &MemberName, + _: serde_json::Value, + _: &CallContext, + ) -> Result { + let _destruction = DestructionObserved { + guard: self.guard.clone(), + observed: self.observed.clone(), + }; + self.entered.notify_one(); + std::future::pending().await + } +} + +#[test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +fn runtime_drop_destroys_provider_future_before_offruntime_native_cleanup() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let account = state(":1.1".parse().unwrap(), ":1.2".parse().unwrap()); + let observed = Arc::new(AtomicUsize::new(0)); + let entered = Arc::new(Notify::new()); + let (transport, view, barrier) = runtime.block_on(async { + let (transport, view, init, barrier) = crate::module::transport::tests::native_fixture(); + barrier.state.lock().unwrap().1 = true; + transport.bind_host_callbacks(account.clone()).unwrap(); + transport.defer_initialize(init, view); + transport.ensure_initialized().await.unwrap(); + barrier.entered.notified().await; + let guarded = GuardedInterface { + interface: PendingUntilDestroyed { + entered: entered.clone(), + guard: HostCallbackGuard(account.clone()), + observed: observed.clone(), + }, + endpoint: endpoint(), + state: account.clone(), + }; + let context = CallContext::brokered(account.module_peer().as_ref(), true); + tokio::spawn(async move { + guarded + .call_with_context(&endpoint().members[0], serde_json::json!([]), &context) + .await + }); + entered.notified().await; + transport.request_terminal(false); + account.physical_wait_started.notified().await; + (transport, view, barrier) + }); + let count_before_drop = HostCallbackGuard(account.clone()).running(); + let terminal_before_drop = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; + let lifecycle_free = !transport.lifecycle_is_owned_for_test(); + // This drops the actual GuardedInterface/user future, not a caller observer. + drop(runtime); + let deadline = std::time::Instant::now() + Duration::from_secs(2); + let completed = loop { + if unsafe { view.native_terminal_state.unwrap()(view.host_ctx) } + & crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED + != 0 + { + break true; + } + if std::time::Instant::now() >= deadline { + break false; + } + std::thread::yield_now(); + }; + assert!(completed, "actual native callback return remains pending"); + assert_eq!(count_before_drop, 1); + assert_eq!( + terminal_before_drop, + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT + ); + assert!(lifecycle_free); + assert_eq!( + observed.load(Ordering::Acquire), + 1, + "user future destruction precedes counter release" + ); + assert_eq!(HostCallbackGuard(account).running(), 0); + assert!(barrier.state.lock().unwrap().2); + assert!(!transport.is_faulted()); +} From 15ae3c13f36ce31a1fe7e3fe1240ae76db257744 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:03:26 +0300 Subject: [PATCH 018/130] chore: files changed crates/tinybus/src/module/host_callback_lifecycle_tests.rs,crates/tinybus/src/m Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../module/host_callback_lifecycle_tests.rs | 12 ++- .../src/module/host_callbacks_tests.rs | 100 +++++++++++++----- 2 files changed, 82 insertions(+), 30 deletions(-) diff --git a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs index 4f133ff0..14aa3a6f 100644 --- a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs +++ b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs @@ -143,6 +143,9 @@ async fn stop_after_admitted_callback(mode: DrainStop) { ) }) .await; + if matches!(mode, DrainStop::Timeout) && early.is_none() { + early = Some(stopping.take().unwrap().await); + } if !matches!(mode, DrainStop::Complete) { drop(stopping.take()); } @@ -197,13 +200,14 @@ async fn stop_after_admitted_callback(mode: DrainStop) { .await; let _outcome = request.await; assert!(completed.is_ok()); - assert!(entered.is_ok()); - assert!(host_owner.upgrade().is_none()); - assert_eq!(still_counted, 1); assert_eq!( terminal_after_drop, - crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT, + "opaque shutdown returned while provider work was still admitted" ); + assert!(entered.is_ok()); + assert!(host_owner.upgrade().is_none()); + assert_eq!(still_counted, 1); assert!(lifecycle_free && authority_denied && new_call_denied); assert_eq!(committed.load(Ordering::Acquire), 1); assert!(!transport.is_faulted()); diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 937ca505..83b151db 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -1270,6 +1270,7 @@ impl Drop for DestructionObserved { } } struct PendingUntilDestroyed { + complete: bool, entered: Arc, guard: HostCallbackGuard, observed: Arc, @@ -1296,7 +1297,11 @@ impl Interface for PendingUntilDestroyed { observed: self.observed.clone(), }; self.entered.notify_one(); - std::future::pending().await + if self.complete { + Ok(serde_json::Value::Null) + } else { + std::future::pending().await + } } } @@ -1310,33 +1315,52 @@ fn runtime_drop_destroys_provider_future_before_offruntime_native_cleanup() { let account = state(":1.1".parse().unwrap(), ":1.2".parse().unwrap()); let observed = Arc::new(AtomicUsize::new(0)); let entered = Arc::new(Notify::new()); - let (transport, view, barrier) = runtime.block_on(async { - let (transport, view, init, barrier) = crate::module::transport::tests::native_fixture(); - barrier.state.lock().unwrap().1 = true; - transport.bind_host_callbacks(account.clone()).unwrap(); - transport.defer_initialize(init, view); - transport.ensure_initialized().await.unwrap(); - barrier.entered.notified().await; - let guarded = GuardedInterface { - interface: PendingUntilDestroyed { - entered: entered.clone(), - guard: HostCallbackGuard(account.clone()), - observed: observed.clone(), - }, - endpoint: endpoint(), - state: account.clone(), - }; - let context = CallContext::brokered(account.module_peer().as_ref(), true); - tokio::spawn(async move { - guarded - .call_with_context(&endpoint().members[0], serde_json::json!([]), &context) + let (transport, view, barrier, user_entered, wait_entered, replacement_rejected) = runtime + .block_on(async { + let (transport, view, init, barrier) = + crate::module::transport::tests::native_fixture(); + barrier.state.lock().unwrap().1 = true; + transport.bind_host_callbacks(account.clone()).unwrap(); + transport.defer_initialize(init, view); + transport.ensure_initialized().await.unwrap(); + barrier.entered.notified().await; + let guarded = GuardedInterface { + interface: PendingUntilDestroyed { + complete: false, + entered: entered.clone(), + guard: HostCallbackGuard(account.clone()), + observed: observed.clone(), + }, + endpoint: endpoint(), + state: account.clone(), + }; + let context = CallContext::brokered(account.module_peer().as_ref(), true); + tokio::spawn(async move { + guarded + .call_with_context(&endpoint().members[0], serde_json::json!([]), &context) + .await + }); + let user_entered = tokio::time::timeout(Duration::from_secs(2), entered.notified()) .await + .is_ok(); + let replacement = state(":1.3".parse().unwrap(), ":1.4".parse().unwrap()); + let replacement_rejected = transport.bind_host_callbacks(replacement).is_err(); + transport.request_terminal(false); + let wait_entered = tokio::time::timeout( + Duration::from_secs(2), + account.physical_wait_started.notified(), + ) + .await + .is_ok(); + ( + transport, + view, + barrier, + user_entered, + wait_entered, + replacement_rejected, + ) }); - entered.notified().await; - transport.request_terminal(false); - account.physical_wait_started.notified().await; - (transport, view, barrier) - }); let count_before_drop = HostCallbackGuard(account.clone()).running(); let terminal_before_drop = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; let lifecycle_free = !transport.lifecycle_is_owned_for_test(); @@ -1356,6 +1380,7 @@ fn runtime_drop_destroys_provider_future_before_offruntime_native_cleanup() { std::thread::yield_now(); }; assert!(completed, "actual native callback return remains pending"); + assert!(user_entered && wait_entered && replacement_rejected); assert_eq!(count_before_drop, 1); assert_eq!( terminal_before_drop, @@ -1371,3 +1396,26 @@ fn runtime_drop_destroys_provider_future_before_offruntime_native_cleanup() { assert!(barrier.state.lock().unwrap().2); assert!(!transport.is_faulted()); } + +#[tokio::test] +async fn completing_provider_future_is_destroyed_before_its_running_account() { + let account = state(":1.1".parse().unwrap(), ":1.2".parse().unwrap()); + let observed = Arc::new(AtomicUsize::new(0)); + let guarded = GuardedInterface { + interface: PendingUntilDestroyed { + complete: true, + entered: Arc::new(Notify::new()), + guard: HostCallbackGuard(account.clone()), + observed: observed.clone(), + }, + endpoint: endpoint(), + state: account.clone(), + }; + let context = CallContext::brokered(account.module_peer().as_ref(), true); + let result = guarded + .call_with_context(&endpoint().members[0], serde_json::json!([]), &context) + .await; + assert!(result.is_ok()); + assert_eq!(observed.load(Ordering::Acquire), 1); + assert_eq!(HostCallbackGuard(account).running(), 0); +} From aab489a06bee54670a97f86b6bfa6ffdc37e2e36 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:07:57 +0300 Subject: [PATCH 019/130] docs(module): document callback lifecycle validation Rework the callback lifecycle validation notes to describe the single exact CallbackState retained by the loaded record and transport, the stop intent committed before draining admitted futures, and the last-host drop path that waits on the physical owner. The previous text no longer matched the implemented ordering and marker semantics. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../module/callback-lifecycle-validation.md | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/docs/modules/module/callback-lifecycle-validation.md b/docs/modules/module/callback-lifecycle-validation.md index 24b9b38e..eb022f27 100644 --- a/docs/modules/module/callback-lifecycle-validation.md +++ b/docs/modules/module/callback-lifecycle-validation.md @@ -25,18 +25,32 @@ artifact cases, plus the existing helper, admission and umask controls. ## Implementation -The loaded record retains its exact `CallbackState`, including operation -accounting, after routing revocation removes its route. Stop revokes first and -then waits for admitted user futures with its remaining deadline, before native -shutdown. No loaded, router or callback-state lock is held across that wait. +The loaded record and transport retain one exact `CallbackState`, including +operation accounting, after routing revocation removes its route. The account +is bound before callback admission and cannot be replaced. Its backlink to the +transport remains weak. -A marker owned by the actual blocking native callback survives public waiter -cancellation or timeout. Enable/disable checks it under the same loaded mutex -that admits lifecycle transitions. The marker drops before the native lifecycle -mutex unlocks, so a finishing callback cannot clear a successor's marker. Stop -and reinitialize keep their canonical queued semantics. +Stop revokes routing and commits native stop intent before waiting for admitted +user futures with its remaining deadline. This closes native admission without +submitting physical shutdown. Drain timeout or public waiter cancellation keeps +stop intent pending; enable cannot restore admission. Once the actual provider +account drains, stop submits the existing physical owner. Ordinary stop intent +is not upgraded to a fault. -## Fresh acceptance +Last-host drop also uses that physical owner. Its OS worker revokes the exact +account and waits for its actual running count to reach zero before acquiring +the native lifecycle mutex or calling opaque shutdown. This wait has no runtime +dependency or deadline. A provider that never releases its future keeps the +physical job pending. Provider futures are destroyed before their running +accounts decrement, including when their runtime is destroyed. No provider code +or destructor runs under the accounting mutex. + +Reinitialize retains its marker in the actual blocking callback until it +returns. Enable/disable also checks committed stop intent and the retained +physical job, including uncertain cleanup. Public waiter completion or timeout +is not evidence of physical reclamation. + +## Upstream acceptance before integration Run 87340 completed with exit 0 on source `372a0ddd97d23daf58203d59b95b7be79fc3393a`. All 3,068 output lines were inspected: From a358cc8e48c6f2382c3fcda9368eb886792d2709 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:10:25 +0300 Subject: [PATCH 020/130] chore: files changed crates/tinybus/src/module/host_callbacks.rs,crates/tinybus/src/module/host_call Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_callbacks.rs | 8 ++++++++ crates/tinybus/src/module/host_callbacks_tests.rs | 14 ++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 34cc3921..32c94fe3 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -175,6 +175,8 @@ pub struct HostCallbackBinding { fixture_controller: Mutex>>, #[cfg(test)] before_load_decision: Mutex, tokio::sync::oneshot::Receiver<()>)>>, + #[cfg(test)] + after_load_termination: Mutex>>, } /// Outgoing operations on the binding's private connection. @@ -482,6 +484,8 @@ impl ModuleHost { fixture_controller: Mutex::new(None), #[cfg(test)] before_load_decision: Mutex::new(None), + #[cfg(test)] + after_load_termination: Mutex::new(None), state: Arc::new(CallbackState { destination, endpoints, @@ -690,6 +694,10 @@ impl ModuleHost { let fault = transport.commit_admission_termination(cause); binding.revoke(); transport.request_terminal(fault); + #[cfg(test)] + if let Some(entered) = binding.after_load_termination.lock().unwrap().take() { + entered.notify_one(); + } let _ = transport.close().await; self.inner .loaded diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 83b151db..a0370875 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -902,6 +902,8 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { let binding = Arc::new(served_native_binding(&host).await); *binding.fixture_controller.lock().unwrap() = Some(barrier.clone()); let decision = Arc::new(Notify::new()); + let termination = Arc::new(Notify::new()); + *binding.after_load_termination.lock().unwrap() = Some(termination.clone()); let (resume, release) = tokio::sync::oneshot::channel(); *binding.before_load_decision.lock().unwrap() = Some((decision.clone(), release)); let mut cleanup = ControlledRelease::new(barrier.clone()); @@ -967,10 +969,13 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { fault_flags = Some(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }); } let resumed = cleanup.resume.take().unwrap().send(()); - // Rollback revokes before awaiting physical close or returning failure. - while !binding.state.lifetime.lock().unwrap().revoked && !load.is_finished() { - tokio::task::yield_now().await; - } + // The physical worker can revoke the account independently. Observe the + // loader's actual failure commit instead of treating revocation as proof. + let terminated = if !ready { + Some(tokio::time::timeout(Duration::from_secs(2), termination.notified()).await) + } else { + None + }; let held = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; cleanup.release(); // Keep virtual time runnable through the actual opaque return. @@ -982,6 +987,7 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { cleanup.transport = None; assert!(initialization.is_ok()); assert!(resumed.is_ok()); + assert!(terminated.is_none_or(|result| result.is_ok())); assert!(stopped.unwrap().is_ok()); assert!(loaded.unwrap().is_err()); assert_eq!(observed_ready, ready); From ccd70a385c85cfc249acce407ff05d6a1e1daa37 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:24:42 +0300 Subject: [PATCH 021/130] docs(plans): clarify native terminal stop commit points Host stop now commits ordinary intent before its bounded provider drain, with drain or cancellation preserving that intent and the loaded owner while native physical handoff waits for provider completion. Direct transport stop commits after acquiring the native lifecycle lock, and canceling that wait before acquisition makes no new commitment. Auto-committed-on: dragonfly Co-authored-by: Medulla --- docs/plans/native-terminal-authority.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/plans/native-terminal-authority.md b/docs/plans/native-terminal-authority.md index 59fd4a9a..49d4a8ed 100644 --- a/docs/plans/native-terminal-authority.md +++ b/docs/plans/native-terminal-authority.md @@ -26,8 +26,12 @@ owner scope/generation/receiver, local flags, live native query, then local flag again. Query success is admission linearization; later intent cannot undo an OS operation already granted. Query alone is not a publication permit. -Normal stop commits intent only after acquiring actual lifecycle lock, before -handing it to retained physical task. Canceled/unacquired stop makes no commitment. +Host stop commits ordinary intent before its bounded provider-callback drain. +Drain timeout or cancellation preserves that intent and the loaded owner; native +physical handoff waits for actual provider completion. An entirely unpolled stop +has no effect. Direct transport stop commits after acquiring the native lifecycle +lock, before handing it to the retained physical task; canceling that direct wait +before acquisition makes no new commitment. Actual fault, rollback and Host Drop commit immediately, even behind opaque C init or reinit. Fault takes public-state precedence; normal stop never becomes Faulted. Fault cleanup must not create ordinary transition_from or restart terminal modules. From b4ea8faa160c506cfa880b21620761430be3b624 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:46:07 +0300 Subject: [PATCH 022/130] test(tinybus-module): cover shutdown suppression of captured caller dispatch Add a test that captures the tracing dispatch active on the factory and configuration caller paths, then verifies that after shutdown the captured dispatch no longer forwards events and cannot select a replacement module's host callback. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/execution_context_tests.rs | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/crates/tinybus-module/src/execution_context_tests.rs b/crates/tinybus-module/src/execution_context_tests.rs index 7b0d052a..6e952607 100644 --- a/crates/tinybus-module/src/execution_context_tests.rs +++ b/crates/tinybus-module/src/execution_context_tests.rs @@ -111,3 +111,69 @@ fn unowned_and_reentrant_panic_contexts_never_select_a_host_callback() { assert_eq!(count.load(Ordering::SeqCst), 0); assert!(owner.panic_location.lock().unwrap().is_none()); } + +static CONFIG_DISPATCH: StdMutex> = StdMutex::new(None); +struct CapturedConfig; +impl<'de> serde::Deserialize<'de> for CapturedConfig { + fn deserialize>(deserializer: D) -> std::result::Result { + let _ = serde_json::Value::deserialize(deserializer)?; + *CONFIG_DISPATCH.lock().unwrap() = Some(tracing::dispatcher::get_default(Clone::clone)); + Ok(Self) + } +} + +#[test] +fn shutdown_suppresses_the_exact_factory_and_configuration_caller_dispatch() { + for factory in [true, false] { + let (fixture, count) = owner(); + let mut host = fixture.host.0; + host.config = tinybus::module::abi::TbSlice { ptr: b"{}".as_ptr(), len: 2 }; + let captured = Arc::new(StdMutex::new(None)); + let capture = captured.clone(); + let mut module = TbModuleVtable::default(); + let started = if factory { + unsafe { + start_reconfigurable_module_with_factory::( + &host, &mut module, 1, false, move || { + *capture.lock().unwrap() = Some(tracing::dispatcher::get_default(Clone::clone)); + |_, _| async { Ok(()) } + }, + ) + } + } else { + unsafe { + start_module_with_config::( + &host, &mut module, 1, false, |_, _| async { Ok(()) }, + ) + } + }; + let dispatch = if factory { captured.lock().unwrap().take() } else { CONFIG_DISPATCH.lock().unwrap().take() }; + let before = count.load(Ordering::SeqCst); + if let Some(dispatch) = &dispatch { + tracing::dispatcher::with_default(dispatch, || tracing::info!("captured caller before stop")); + } + let live = count.load(Ordering::SeqCst); + let stopped = if started == TB_OK { unsafe { (module.shutdown)(module.module_ctx, 1000) } } else { TB_CLOSED }; + let after_shutdown = count.load(Ordering::SeqCst); + let (replacement_fixture, replacement_count) = owner(); + let mut replacement_host = replacement_fixture.host.0; + replacement_host.config = host.config; + let mut replacement = TbModuleVtable::default(); + let replaced = unsafe { start_module(&replacement_host, &mut replacement, 1, false, |_| async { Ok(()) }) }; + let replacement_before = replacement_count.load(Ordering::SeqCst); + if let Some(dispatch) = &dispatch { + tracing::dispatcher::with_default(dispatch, || tracing::info!("captured caller after actual shutdown")); + } + let late = count.load(Ordering::SeqCst); + let replacement_after = replacement_count.load(Ordering::SeqCst); + let replacement_stopped = if replaced == TB_OK { unsafe { (replacement.shutdown)(replacement.module_ctx, 1000) } } else { TB_CLOSED }; + assert_eq!(started, TB_OK); + assert!(dispatch.is_some()); + assert!(live > before, "real caller dispatch forwards before shutdown"); + assert_eq!(stopped, TB_OK); + assert_eq!(replaced, TB_OK); + assert_eq!(replacement_stopped, TB_OK); + assert_eq!(late, after_shutdown, "stopped caller dispatch must not forward"); + assert_eq!(replacement_after, replacement_before, "old dispatch cannot select replacement"); + } +} From b27285f5c87425fdbc5b50b55d0f4e8e4e855ae2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:47:50 +0300 Subject: [PATCH 023/130] chore: files changed crates/tinybus-module/src/execution_context_tests.rs,crates/tinybus/src/module/ Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/execution_context_tests.rs | 1 + .../tinybus/src/module/host_reload_tests.rs | 36 +++++++++++++++---- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/crates/tinybus-module/src/execution_context_tests.rs b/crates/tinybus-module/src/execution_context_tests.rs index 6e952607..0a538468 100644 --- a/crates/tinybus-module/src/execution_context_tests.rs +++ b/crates/tinybus-module/src/execution_context_tests.rs @@ -167,6 +167,7 @@ fn shutdown_suppresses_the_exact_factory_and_configuration_caller_dispatch() { let late = count.load(Ordering::SeqCst); let replacement_after = replacement_count.load(Ordering::SeqCst); let replacement_stopped = if replaced == TB_OK { unsafe { (replacement.shutdown)(replacement.module_ctx, 1000) } } else { TB_CLOSED }; + eprintln!("factory={factory}: start={started}, stop={stopped}, before={before}, live={live}, after_shutdown={after_shutdown}, late={late}, replacement={replacement_before}/{replacement_after}, replacement_stop={replacement_stopped}"); assert_eq!(started, TB_OK); assert!(dispatch.is_some()); assert!(live > before, "real caller dispatch forwards before shutdown"); diff --git a/crates/tinybus/src/module/host_reload_tests.rs b/crates/tinybus/src/module/host_reload_tests.rs index 22e38331..1c65a1ba 100644 --- a/crates/tinybus/src/module/host_reload_tests.rs +++ b/crates/tinybus/src/module/host_reload_tests.rs @@ -66,13 +66,10 @@ static LOGS: OnceLock>>> = OnceLock::new(); unsafe extern "C" fn observe_log(ctx: *mut std::ffi::c_void, _: u32, ptr: *const u8, len: usize) { let line = String::from_utf8_lossy(unsafe { std::slice::from_raw_parts(ptr, len) }).into_owned(); - LOGS.get() - .unwrap() - .lock() - .unwrap() - .entry(ctx as usize) - .or_default() - .push(line); + record_log(LOGS.get(), ctx as usize, line); +} +fn record_log(storage: Option<&Mutex>>>, key: usize, line: String) { + storage.unwrap().lock().unwrap().entry(key).or_default().push(line); } fn observed_native(host: &ModuleHost, path: &Path, prefix: &str) -> usize { let artifact = loader::load(path, false).unwrap(); @@ -189,3 +186,28 @@ async fn native_replacement_logs_and_sanitized_panics_use_only_its_original_host second.shutdown(Duration::from_secs(3)).await; broker_task.abort(); } + + +#[test] +fn native_log_fixture_recovers_poison_without_unwinding_across_abi() { + let storage = Mutex::new(HashMap::new()); + let poisoned = std::panic::catch_unwind(|| { + let _guard = storage.lock().unwrap(); + panic!("controlled Rust fixture poison"); + }); + // Exercise the callback's Rust body first, safely containing baseline RED. + let recovered = std::panic::catch_unwind(|| record_log(Some(&storage), 1, "local".into())); + if recovered.is_ok() { + let logs = LOGS.get_or_init(|| Mutex::new(HashMap::new())); + let _ = std::panic::catch_unwind(|| { + let _guard = logs.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + panic!("controlled Rust fixture poison"); + }); + let line = b"actual ABI callback after fixture poison"; + unsafe { observe_log(1usize as *mut std::ffi::c_void, 1, line.as_ptr(), line.len()) }; + } + let recorded = LOGS.get().and_then(|logs| logs.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&1).cloned()); + assert!(poisoned.is_err()); + assert!(recovered.is_ok(), "callback body must recover poisoned fixture storage"); + assert!(recorded.is_some_and(|lines| lines.iter().any(|line| line.contains("actual ABI callback")))); +} From acceb53b3203700da1e01abd5368547d86aaf620 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:48:51 +0300 Subject: [PATCH 024/130] fix(module): validate start arguments before installing execution owner The module start path now validates its arguments and derives the execution owner before entering the runtime, so an invalid host or worker count returns TB_BAD_ARGUMENT instead of partially starting. Reload tests were reworked to read recorded log lines through a poison-tolerant helper, and a failing-init test now takes the shared fake-module lock to avoid cross-test interference. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 19 +++++--- .../tinybus/src/module/host_reload_tests.rs | 48 +++++++------------ crates/tinybus/src/module/host_tests.rs | 1 + 3 files changed, 30 insertions(+), 38 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 75ba18a0..6001d390 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -490,6 +490,12 @@ where F: FnOnce(Connection) -> Fut + Send + 'static, Fut: Future> + Send + 'static, { + let Some((_, compatible_host)) = + (unsafe { validate_start_arguments(host, out, worker_threads) }) + else { + return TB_BAD_ARGUMENT; + }; + let owner = Some(ExecutionOwner::new(HostCalls(compatible_host))); unsafe { start_module_runtime( host, @@ -498,7 +504,7 @@ where detach_on_panic, setup, None, - false, + owner, ) } } @@ -528,7 +534,7 @@ where detach_on_panic, setup, None, - true, + None, ) } } @@ -558,7 +564,7 @@ unsafe fn start_module_runtime( detach_on_panic: bool, setup: F, reinitialize_factory: Option, - linked: bool, + owner: Option, ) -> i32 where F: FnOnce(Connection) -> Fut + Send + 'static, @@ -572,7 +578,6 @@ where }; let host = HostCalls(host); - let owner = (!linked).then(|| ExecutionOwner::new(host)); if owner.is_some() { ExecutionOwner::install_panic_hook(); } @@ -739,12 +744,14 @@ where Err(_) => return TB_PANICKED, }; unsafe { - start_module( + start_module_runtime( host, out, worker_threads, detach_on_panic, move |connection| setup(connection, config), + None, + caller_owner, ) } } @@ -970,7 +977,7 @@ where detach_on_panic, move |connection| initial_setup(connection, config), Some(reinitialize_factory), - linked, + caller_owner, ) } } diff --git a/crates/tinybus/src/module/host_reload_tests.rs b/crates/tinybus/src/module/host_reload_tests.rs index 1c65a1ba..46ab5a58 100644 --- a/crates/tinybus/src/module/host_reload_tests.rs +++ b/crates/tinybus/src/module/host_reload_tests.rs @@ -69,7 +69,12 @@ unsafe extern "C" fn observe_log(ctx: *mut std::ffi::c_void, _: u32, ptr: *const record_log(LOGS.get(), ctx as usize, line); } fn record_log(storage: Option<&Mutex>>>, key: usize, line: String) { - storage.unwrap().lock().unwrap().entry(key).or_default().push(line); + if let Some(storage) = storage { + storage.lock().unwrap_or_else(std::sync::PoisonError::into_inner).entry(key).or_default().push(line); + } +} +fn recorded_lines(key: usize) -> Vec { + LOGS.get().and_then(|logs| logs.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&key).cloned()).unwrap_or_default() } fn observed_native(host: &ModuleHost, path: &Path, prefix: &str) -> usize { let artifact = loader::load(path, false).unwrap(); @@ -82,7 +87,7 @@ fn observed_native(host: &ModuleHost, path: &Path, prefix: &str) -> usize { let key = vtable.host_ctx as usize; LOGS.get_or_init(|| Mutex::new(HashMap::new())) .lock() - .unwrap() + .unwrap_or_else(std::sync::PoisonError::into_inner) .insert(key, Vec::new()); vtable.log = observe_log; transport.defer_initialize(artifact.init, vtable); @@ -128,11 +133,8 @@ async fn native_replacement_logs_and_sanitized_panics_use_only_its_original_host .unwrap() .starts_with("first:") ); - assert!( - LOGS.get().unwrap().lock().unwrap()[&first_key] - .iter() - .any(|line| line.contains("clock fixture served")) - ); + let first_lines = recorded_lines(first_key); + assert!(first_lines.iter().any(|line| line.contains("clock fixture served"))); first.shutdown(Duration::from_secs(3)).await; tokio::time::timeout(Duration::from_secs(3), async { while caller @@ -148,7 +150,7 @@ async fn native_replacement_logs_and_sanitized_panics_use_only_its_original_host .await .unwrap(); drop(first); - let first_count = LOGS.get().unwrap().lock().unwrap()[&first_key].len(); + let first_count = recorded_lines(first_key).len(); let second = ModuleHost::new(broker); let second_key = observed_native(&second, &artifact, "second:"); assert_ne!(first_key, second_key); @@ -160,34 +162,16 @@ async fn native_replacement_logs_and_sanitized_panics_use_only_its_original_host .starts_with("second:") ); assert!(proxy.call::("Panic", ()).await.is_err()); - { - let logs = LOGS.get().unwrap().lock().unwrap(); - assert_eq!( - logs[&first_key].len(), - first_count, - "B must not call A's original logging callback" - ); - assert!( - logs[&second_key] - .iter() - .any(|line| line.contains("clock fixture served")) - ); - assert!( - logs[&second_key] - .iter() - .any(|line| line.contains("module panicked at")) - ); - assert!( - logs[&second_key] - .iter() - .all(|line| !line.contains("secret-token")) - ); - } + let first_lines = recorded_lines(first_key); + let second_lines = recorded_lines(second_key); second.shutdown(Duration::from_secs(3)).await; broker_task.abort(); + assert_eq!(first_lines.len(), first_count, "B must not call A's original logging callback"); + assert!(second_lines.iter().any(|line| line.contains("clock fixture served"))); + assert!(second_lines.iter().any(|line| line.contains("module panicked at"))); + assert!(second_lines.iter().all(|line| !line.contains("secret-token"))); } - #[test] fn native_log_fixture_recovers_poison_without_unwinding_across_abi() { let storage = Mutex::new(HashMap::new()); diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index b4214ba9..ab11c4c6 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -746,6 +746,7 @@ fn a_sidecar_without_lazy_init_refuses_the_artifact_instead_of_loading_it_eagerl #[tokio::test] async fn a_module_whose_init_fails_is_terminal_and_is_never_initialized_again() { + let _test_guard = FAKE_MODULE_TEST_LOCK.lock().await; FAILED_INIT_COUNT.store(0, Ordering::Release); let bus = MemoryBus::new(); let broker = Broker::new(); From bfabd2d91e2bd5866627df2078913793764a27ca Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:50:07 +0300 Subject: [PATCH 025/130] test(module): harden shutdown and reload test fixtures The shutdown, host reload, and transport tests now release their worker fixtures through guards and watchdogs so a stalled cleanup cannot hang the suite, and the transport test additionally waits for the real lifecycle lock to be released before asserting. Assertions and formatting were reflowed to keep the intent of each check explicit. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/execution_context_tests.rs | 75 ++++++++++++++---- .../tinybus/src/module/host_reload_tests.rs | 77 ++++++++++++++++--- crates/tinybus/src/module/transport_tests.rs | 52 +++++++++++-- 3 files changed, 169 insertions(+), 35 deletions(-) diff --git a/crates/tinybus-module/src/execution_context_tests.rs b/crates/tinybus-module/src/execution_context_tests.rs index 0a538468..3a3b40c7 100644 --- a/crates/tinybus-module/src/execution_context_tests.rs +++ b/crates/tinybus-module/src/execution_context_tests.rs @@ -115,7 +115,9 @@ fn unowned_and_reentrant_panic_contexts_never_select_a_host_callback() { static CONFIG_DISPATCH: StdMutex> = StdMutex::new(None); struct CapturedConfig; impl<'de> serde::Deserialize<'de> for CapturedConfig { - fn deserialize>(deserializer: D) -> std::result::Result { + fn deserialize>( + deserializer: D, + ) -> std::result::Result { let _ = serde_json::Value::deserialize(deserializer)?; *CONFIG_DISPATCH.lock().unwrap() = Some(tracing::dispatcher::get_default(Clone::clone)); Ok(Self) @@ -127,15 +129,23 @@ fn shutdown_suppresses_the_exact_factory_and_configuration_caller_dispatch() { for factory in [true, false] { let (fixture, count) = owner(); let mut host = fixture.host.0; - host.config = tinybus::module::abi::TbSlice { ptr: b"{}".as_ptr(), len: 2 }; + host.config = tinybus::module::abi::TbSlice { + ptr: b"{}".as_ptr(), + len: 2, + }; let captured = Arc::new(StdMutex::new(None)); let capture = captured.clone(); let mut module = TbModuleVtable::default(); let started = if factory { unsafe { start_reconfigurable_module_with_factory::( - &host, &mut module, 1, false, move || { - *capture.lock().unwrap() = Some(tracing::dispatcher::get_default(Clone::clone)); + &host, + &mut module, + 1, + false, + move || { + *capture.lock().unwrap() = + Some(tracing::dispatcher::get_default(Clone::clone)); |_, _| async { Ok(()) } }, ) @@ -143,38 +153,73 @@ fn shutdown_suppresses_the_exact_factory_and_configuration_caller_dispatch() { } else { unsafe { start_module_with_config::( - &host, &mut module, 1, false, |_, _| async { Ok(()) }, + &host, + &mut module, + 1, + false, + |_, _| async { Ok(()) }, ) } }; - let dispatch = if factory { captured.lock().unwrap().take() } else { CONFIG_DISPATCH.lock().unwrap().take() }; + let dispatch = if factory { + captured.lock().unwrap().take() + } else { + CONFIG_DISPATCH.lock().unwrap().take() + }; let before = count.load(Ordering::SeqCst); if let Some(dispatch) = &dispatch { - tracing::dispatcher::with_default(dispatch, || tracing::info!("captured caller before stop")); + tracing::dispatcher::with_default(dispatch, || { + tracing::info!("captured caller before stop") + }); } let live = count.load(Ordering::SeqCst); - let stopped = if started == TB_OK { unsafe { (module.shutdown)(module.module_ctx, 1000) } } else { TB_CLOSED }; + let stopped = if started == TB_OK { + unsafe { (module.shutdown)(module.module_ctx, 1000) } + } else { + TB_CLOSED + }; let after_shutdown = count.load(Ordering::SeqCst); let (replacement_fixture, replacement_count) = owner(); let mut replacement_host = replacement_fixture.host.0; replacement_host.config = host.config; let mut replacement = TbModuleVtable::default(); - let replaced = unsafe { start_module(&replacement_host, &mut replacement, 1, false, |_| async { Ok(()) }) }; + let replaced = unsafe { + start_module(&replacement_host, &mut replacement, 1, false, |_| async { + Ok(()) + }) + }; let replacement_before = replacement_count.load(Ordering::SeqCst); if let Some(dispatch) = &dispatch { - tracing::dispatcher::with_default(dispatch, || tracing::info!("captured caller after actual shutdown")); + tracing::dispatcher::with_default(dispatch, || { + tracing::info!("captured caller after actual shutdown") + }); } let late = count.load(Ordering::SeqCst); let replacement_after = replacement_count.load(Ordering::SeqCst); - let replacement_stopped = if replaced == TB_OK { unsafe { (replacement.shutdown)(replacement.module_ctx, 1000) } } else { TB_CLOSED }; - eprintln!("factory={factory}: start={started}, stop={stopped}, before={before}, live={live}, after_shutdown={after_shutdown}, late={late}, replacement={replacement_before}/{replacement_after}, replacement_stop={replacement_stopped}"); + let replacement_stopped = if replaced == TB_OK { + unsafe { (replacement.shutdown)(replacement.module_ctx, 1000) } + } else { + TB_CLOSED + }; + eprintln!( + "factory={factory}: start={started}, stop={stopped}, before={before}, live={live}, after_shutdown={after_shutdown}, late={late}, replacement={replacement_before}/{replacement_after}, replacement_stop={replacement_stopped}" + ); assert_eq!(started, TB_OK); assert!(dispatch.is_some()); - assert!(live > before, "real caller dispatch forwards before shutdown"); + assert!( + live > before, + "real caller dispatch forwards before shutdown" + ); assert_eq!(stopped, TB_OK); assert_eq!(replaced, TB_OK); assert_eq!(replacement_stopped, TB_OK); - assert_eq!(late, after_shutdown, "stopped caller dispatch must not forward"); - assert_eq!(replacement_after, replacement_before, "old dispatch cannot select replacement"); + assert_eq!( + late, after_shutdown, + "stopped caller dispatch must not forward" + ); + assert_eq!( + replacement_after, replacement_before, + "old dispatch cannot select replacement" + ); } } diff --git a/crates/tinybus/src/module/host_reload_tests.rs b/crates/tinybus/src/module/host_reload_tests.rs index 46ab5a58..dce1492e 100644 --- a/crates/tinybus/src/module/host_reload_tests.rs +++ b/crates/tinybus/src/module/host_reload_tests.rs @@ -70,11 +70,23 @@ unsafe extern "C" fn observe_log(ctx: *mut std::ffi::c_void, _: u32, ptr: *const } fn record_log(storage: Option<&Mutex>>>, key: usize, line: String) { if let Some(storage) = storage { - storage.lock().unwrap_or_else(std::sync::PoisonError::into_inner).entry(key).or_default().push(line); + storage + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .entry(key) + .or_default() + .push(line); } } fn recorded_lines(key: usize) -> Vec { - LOGS.get().and_then(|logs| logs.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&key).cloned()).unwrap_or_default() + LOGS.get() + .and_then(|logs| { + logs.lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&key) + .cloned() + }) + .unwrap_or_default() } fn observed_native(host: &ModuleHost, path: &Path, prefix: &str) -> usize { let artifact = loader::load(path, false).unwrap(); @@ -134,7 +146,11 @@ async fn native_replacement_logs_and_sanitized_panics_use_only_its_original_host .starts_with("first:") ); let first_lines = recorded_lines(first_key); - assert!(first_lines.iter().any(|line| line.contains("clock fixture served"))); + assert!( + first_lines + .iter() + .any(|line| line.contains("clock fixture served")) + ); first.shutdown(Duration::from_secs(3)).await; tokio::time::timeout(Duration::from_secs(3), async { while caller @@ -166,10 +182,26 @@ async fn native_replacement_logs_and_sanitized_panics_use_only_its_original_host let second_lines = recorded_lines(second_key); second.shutdown(Duration::from_secs(3)).await; broker_task.abort(); - assert_eq!(first_lines.len(), first_count, "B must not call A's original logging callback"); - assert!(second_lines.iter().any(|line| line.contains("clock fixture served"))); - assert!(second_lines.iter().any(|line| line.contains("module panicked at"))); - assert!(second_lines.iter().all(|line| !line.contains("secret-token"))); + assert_eq!( + first_lines.len(), + first_count, + "B must not call A's original logging callback" + ); + assert!( + second_lines + .iter() + .any(|line| line.contains("clock fixture served")) + ); + assert!( + second_lines + .iter() + .any(|line| line.contains("module panicked at")) + ); + assert!( + second_lines + .iter() + .all(|line| !line.contains("secret-token")) + ); } #[test] @@ -184,14 +216,35 @@ fn native_log_fixture_recovers_poison_without_unwinding_across_abi() { if recovered.is_ok() { let logs = LOGS.get_or_init(|| Mutex::new(HashMap::new())); let _ = std::panic::catch_unwind(|| { - let _guard = logs.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _guard = logs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); panic!("controlled Rust fixture poison"); }); let line = b"actual ABI callback after fixture poison"; - unsafe { observe_log(1usize as *mut std::ffi::c_void, 1, line.as_ptr(), line.len()) }; + unsafe { + observe_log( + 1usize as *mut std::ffi::c_void, + 1, + line.as_ptr(), + line.len(), + ) + }; } - let recorded = LOGS.get().and_then(|logs| logs.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&1).cloned()); + let recorded = LOGS.get().and_then(|logs| { + logs.lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&1) + .cloned() + }); assert!(poisoned.is_err()); - assert!(recovered.is_ok(), "callback body must recover poisoned fixture storage"); - assert!(recorded.is_some_and(|lines| lines.iter().any(|line| line.contains("actual ABI callback")))); + assert!( + recovered.is_ok(), + "callback body must recover poisoned fixture storage" + ); + assert!(recorded.is_some_and(|lines| { + lines + .iter() + .any(|line| line.contains("actual ABI callback")) + })); } diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index be0796cd..02926c20 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -2260,7 +2260,15 @@ async fn rejected_observer_after_ordinary_stop_preserves_nonfault_cause() { #[tokio::test] async fn completed_shutdown_has_already_released_its_native_lifecycle_marker() { - let _guard = VTABLE_TEST_LOCK + struct WorkerRelease(Option>); + impl Drop for WorkerRelease { + fn drop(&mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } + } + let guard = VTABLE_TEST_LOCK .get_or_init(|| tokio::sync::Mutex::new(())) .lock() .await; @@ -2271,17 +2279,45 @@ async fn completed_shutdown_has_already_released_its_native_lifecycle_marker() { let published = Arc::new(Notify::new()); let entered = published.clone(); let (release, worker) = std::sync::mpsc::channel(); + let release = WorkerRelease(Some(release)); *transport.after_cleanup_publication.lock().unwrap() = Some(Box::new(move || { entered.notify_one(); - worker.recv().unwrap(); + let _ = worker.recv(); })); - let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); - published.notified().await; - let result = stopping.await.unwrap(); + let mut stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + let publication = tokio::time::timeout(Duration::from_secs(2), published.notified()).await; + let result = tokio::time::timeout(Duration::from_secs(2), &mut stopping).await; let still_pending = transport.native_lifecycle_pending(); - // Always release the worker before asserting, even against the unfixed code. - release.send(()).unwrap(); - assert_eq!(result.unwrap(), TB_OK); + drop(release); + transport.request_terminal(false); + let cleanup = tokio::time::timeout(Duration::from_secs(2), async { + loop { + let changed = transport.context.physical_notify.notified(); + if matches!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Complete | CleanupState::NoCallbackRequired + ) { + break; + } + changed.await; + } + // Publication precedes native lifecycle unlock. Observe that actual + // exclusion release separately; this is not an OS-worker-exit claim. + let lifecycle = transport.lifecycle.clone().lock_owned().await; + drop(lifecycle); + }) + .await; + if cleanup.is_err() { + // Keep fixture exclusion with the actual transport/job if cleanup is + // unresolved. Neither timeout nor dropped async waiter reclaims it. + *transport.fixture_controller.lock().unwrap() = Some(Arc::new(guard)); + } + assert!(publication.is_ok(), "cleanup publication watchdog expired"); + assert!( + cleanup.is_ok(), + "actual cleanup/lifecycle release remains pending" + ); + assert_eq!(result.unwrap().unwrap().unwrap(), TB_OK); assert!( !still_pending, "completed cleanup still reports native work" From 5add882c859af67400f749458737238afcda61c5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:52:06 +0300 Subject: [PATCH 026/130] test(module): cover unavailable log storage and document runtime owner Extend the native log fixture to exercise the callback with no storage available, asserting it returns without unwinding across the ABI, and key the recorded log lookup off a stable pointer instead of a literal. Add a comment in start_module_runtime explaining that the validated entry caller's diagnostic owner must be carried into the runtime so factory, config dispatch, and shutdown share the same live bit. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 2 ++ crates/tinybus/src/module/host_reload_tests.rs | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 6001d390..2e919592 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -557,6 +557,8 @@ unsafe fn validate_start_arguments( Some((out_capacity, host)) } +// Carry the validated entry caller's exact diagnostic owner into the runtime. +// Captured factory/config dispatch and shutdown must share the same live bit. unsafe fn start_module_runtime( host: *const TbHostVtable, out: *mut TbModuleVtable, diff --git a/crates/tinybus/src/module/host_reload_tests.rs b/crates/tinybus/src/module/host_reload_tests.rs index dce1492e..c49babad 100644 --- a/crates/tinybus/src/module/host_reload_tests.rs +++ b/crates/tinybus/src/module/host_reload_tests.rs @@ -207,12 +207,15 @@ async fn native_replacement_logs_and_sanitized_panics_use_only_its_original_host #[test] fn native_log_fixture_recovers_poison_without_unwinding_across_abi() { let storage = Mutex::new(HashMap::new()); + let context = 0u8; + let key = std::ptr::from_ref(&context) as usize; let poisoned = std::panic::catch_unwind(|| { let _guard = storage.lock().unwrap(); panic!("controlled Rust fixture poison"); }); // Exercise the callback's Rust body first, safely containing baseline RED. let recovered = std::panic::catch_unwind(|| record_log(Some(&storage), 1, "local".into())); + let unavailable = std::panic::catch_unwind(|| record_log(None, 1, "unavailable".into())); if recovered.is_ok() { let logs = LOGS.get_or_init(|| Mutex::new(HashMap::new())); let _ = std::panic::catch_unwind(|| { @@ -234,10 +237,14 @@ fn native_log_fixture_recovers_poison_without_unwinding_across_abi() { let recorded = LOGS.get().and_then(|logs| { logs.lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .get(&1) + .get(&key) .cloned() }); assert!(poisoned.is_err()); + assert!( + unavailable.is_ok(), + "unavailable fixture storage must not unwind" + ); assert!( recovered.is_ok(), "callback body must recover poisoned fixture storage" From 09820cb9a0b192df38e640818ca57d8f08eeb210 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 15:53:29 +0300 Subject: [PATCH 027/130] test(host_reload): use key variable in poison recovery fixture The ABI callback fixture now passes the existing key variable to observe_log instead of a hardcoded pointer, keeping the test consistent with the surrounding setup. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/src/module/host_reload_tests.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/crates/tinybus/src/module/host_reload_tests.rs b/crates/tinybus/src/module/host_reload_tests.rs index c49babad..14e7b67d 100644 --- a/crates/tinybus/src/module/host_reload_tests.rs +++ b/crates/tinybus/src/module/host_reload_tests.rs @@ -225,14 +225,7 @@ fn native_log_fixture_recovers_poison_without_unwinding_across_abi() { panic!("controlled Rust fixture poison"); }); let line = b"actual ABI callback after fixture poison"; - unsafe { - observe_log( - 1usize as *mut std::ffi::c_void, - 1, - line.as_ptr(), - line.len(), - ) - }; + unsafe { observe_log(key as *mut std::ffi::c_void, 1, line.as_ptr(), line.len()) }; } let recorded = LOGS.get().and_then(|logs| { logs.lock() From 3f4807e8d8d86e4ee7514206811c3467222fe4ff Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 16:20:47 +0300 Subject: [PATCH 028/130] test(module): cover stop completion with a held denial probe Add a fixture mode that holds the denial probe observer while native stop completes, asserting that stop does not wait on the unrelated observer and that native cleanup and lock release are observed independently. The existing drain assertions now await the denial task and verify actual native cleanup rather than treating a stop error as evidence of reclamation. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../module/host_callback_lifecycle_tests.rs | 100 +++++++++++++++--- 1 file changed, 88 insertions(+), 12 deletions(-) diff --git a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs index 14aa3a6f..da9e9db5 100644 --- a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs +++ b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs @@ -39,6 +39,7 @@ impl crate::Interface for CommittingCallback { #[derive(Clone, Copy)] enum DrainStop { Complete, + HeldProbeObserver, Timeout, Cancel, DropHost, @@ -75,6 +76,22 @@ async fn host_drop_after_cancelled_stop_waits_for_actual_provider_drain() { stop_after_admitted_callback(DrainStop::DropHost).await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTEXT_MODULE to point at the built cdylib"] +async fn native_stop_completion_does_not_wait_for_the_denial_probe_observer() { + stop_after_admitted_callback(DrainStop::HeldProbeObserver).await; +} + +async fn native_cleanup_finished(transport: &crate::module::transport::ModuleTransport) { + while transport.native_terminal_state_for_test() + & crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED + == 0 + || transport.lifecycle_is_owned_for_test() + { + tokio::task::yield_now().await; + } +} + async fn stop_after_admitted_callback(mode: DrainStop) { let artifact = PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTEXT_MODULE").unwrap()); let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); @@ -133,6 +150,7 @@ async fn stop_after_admitted_callback(mode: DrainStop) { } else { Duration::from_secs(2) }; + let stop_started = std::time::Instant::now(); let mut stopping = Some(Box::pin(host.inner.stop(&loaded.name, deadline))); let mut early = std::future::poll_fn(|cx| { std::task::Poll::Ready( @@ -146,7 +164,7 @@ async fn stop_after_admitted_callback(mode: DrainStop) { if matches!(mode, DrainStop::Timeout) && early.is_none() { early = Some(stopping.take().unwrap().await); } - if !matches!(mode, DrainStop::Complete) { + if !matches!(mode, DrainStop::Complete | DrainStop::HeldProbeObserver) { drop(stopping.take()); } let terminal_while_held = transport.native_terminal_state_for_test(); @@ -165,10 +183,44 @@ async fn stop_after_admitted_callback(mode: DrainStop) { .iter() .any(|module| Arc::ptr_eq(&module.transport, &transport)); let enable_denied = host.inner.enable(&loaded.name, true).is_err(); - let new_call_denied = proxy - .call_confidential::("ConfidentialCallback", ()) - .await - .is_err(); + // This observation is independently owned: its scheduling must not consume + // the stop's semantic deadline while the fixture deliberately holds work. + let probe_entered = Arc::new(tokio::sync::Semaphore::new(0)); + let probe_release = Arc::new(tokio::sync::Semaphore::new(0)); + let probe_release_on_failure = CallbackRelease(probe_release.clone()); + let held_probe = matches!(mode, DrainStop::HeldProbeObserver); + let entered = probe_entered.clone(); + let release_probe = probe_release.clone(); + let denial = tokio::spawn(async move { + let denied = proxy + .call_confidential::("ConfidentialCallback", ()) + .await + .is_err(); + if held_probe { + entered.add_permits(1); + let _permit = release_probe.acquire().await; + } + denied + }); + let observer = if held_probe { + let transport = transport.clone(); + Some(tokio::spawn(async move { + // Always release the observer, including a failed watchdog. + let _release = CallbackRelease(probe_release); + let entered = tokio::time::timeout( + Duration::from_secs(2), + probe_entered.acquire(), + ).await; + let completed = tokio::time::timeout( + Duration::from_secs(1), + native_cleanup_finished(&transport), + ).await; + (entered.is_ok(), completed.is_ok()) + })) + } else { + None + }; + let new_call_denied = denial.await; if matches!(mode, DrainStop::DropHost) { let account = host.inner.loaded.lock().unwrap()[0] .callback_state @@ -199,6 +251,8 @@ async fn stop_after_admitted_callback(mode: DrainStop) { }) .await; let _outcome = request.await; + drop(probe_release_on_failure); + assert!(new_call_denied.is_ok_and(|denied| denied)); assert!(completed.is_ok()); assert_eq!( terminal_after_drop, @@ -208,16 +262,15 @@ async fn stop_after_admitted_callback(mode: DrainStop) { assert!(entered.is_ok()); assert!(host_owner.upgrade().is_none()); assert_eq!(still_counted, 1); - assert!(lifecycle_free && authority_denied && new_call_denied); + assert!(lifecycle_free && authority_denied); assert_eq!(committed.load(Ordering::Acquire), 1); assert!(!transport.is_faulted()); return; } // Release and consume real callback/physical cleanup before all assertions. drop(release_on_failure); - tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()) - .await - .unwrap(); + let drained = tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()) + .await; let stopped = if let Some(stopping) = stopping { if let Some(result) = early.take() { result @@ -227,11 +280,34 @@ async fn stop_after_admitted_callback(mode: DrainStop) { } else { host.inner.stop(&loaded.name, Duration::from_secs(2)).await }; - let _outcome = request.await.unwrap(); - assert!(stopped.is_ok()); + let stop_elapsed = stop_started.elapsed(); + let stop_code = stopped.as_ref().err().map(|error| error.wire_name().to_owned()); + // An error is not evidence of reclamation. Complete the retained ordinary + // terminal job and independently observe native shutdown and lock release. + transport.request_terminal(false); + let cleanup = tokio::time::timeout( + Duration::from_secs(2), + native_cleanup_finished(&transport), + ).await; + let request_outcome = request.await; + drop(probe_release_on_failure); + let observer_outcome = match observer { + Some(observer) => Some(observer.await), + None => None, + }; + eprintln!("stop fixture: code={stop_code:?} elapsed={stop_elapsed:?} cleanup={}", + cleanup.is_ok()); + assert!(cleanup.is_ok(), "actual native cleanup did not finish"); + assert!(request_outcome.is_ok()); + assert!(drained.is_ok()); + if let Some(outcome) = observer_outcome { + assert!(outcome.is_ok_and(|(entered, completed)| entered && completed), + "actual native cleanup waited for the unrelated denial observer"); + } + assert!(stopped.is_ok(), "stop code={stop_code:?}, elapsed={stop_elapsed:?}"); assert_eq!(committed.load(Ordering::Acquire), 1); assert_eq!(still_running, 1); - assert!(authority_denied && owner_retained && enable_denied && new_call_denied); + assert!(authority_denied && owner_retained && enable_denied && new_call_denied.is_ok_and(|denied| denied)); assert!(!shutdown_started_early && !physically_stopped_early); assert_eq!( terminal_while_held, From aaada9aad8bbe74abb2c9a6bb6b68da6e7487c6b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 16:22:55 +0300 Subject: [PATCH 029/130] test(module): reorder denial observation after stop and cleanup Move the denial observer await to after the stop and real cleanup complete, so the denial is measured at the task's actual execution phase rather than before the stop begins. Also raise the observer entry timeout and log its outcome to make failures easier to diagnose. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/module/host_callback_lifecycle_tests.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs index da9e9db5..00a31164 100644 --- a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs +++ b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs @@ -208,19 +208,20 @@ async fn stop_after_admitted_callback(mode: DrainStop) { // Always release the observer, including a failed watchdog. let _release = CallbackRelease(probe_release); let entered = tokio::time::timeout( - Duration::from_secs(2), + Duration::from_secs(5), probe_entered.acquire(), ).await; let completed = tokio::time::timeout( Duration::from_secs(1), native_cleanup_finished(&transport), ).await; + eprintln!("denial observer: entered={} actual_cleanup_while_held={}", + entered.is_ok(), completed.is_ok()); (entered.is_ok(), completed.is_ok()) })) } else { None }; - let new_call_denied = denial.await; if matches!(mode, DrainStop::DropHost) { let account = host.inner.loaded.lock().unwrap()[0] .callback_state @@ -251,6 +252,7 @@ async fn stop_after_admitted_callback(mode: DrainStop) { }) .await; let _outcome = request.await; + let new_call_denied = denial.await; drop(probe_release_on_failure); assert!(new_call_denied.is_ok_and(|denied| denied)); assert!(completed.is_ok()); @@ -269,8 +271,6 @@ async fn stop_after_admitted_callback(mode: DrainStop) { } // Release and consume real callback/physical cleanup before all assertions. drop(release_on_failure); - let drained = tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()) - .await; let stopped = if let Some(stopping) = stopping { if let Some(result) = early.take() { result @@ -282,6 +282,8 @@ async fn stop_after_admitted_callback(mode: DrainStop) { }; let stop_elapsed = stop_started.elapsed(); let stop_code = stopped.as_ref().err().map(|error| error.wire_name().to_owned()); + let drained = tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()) + .await; // An error is not evidence of reclamation. Complete the retained ordinary // terminal job and independently observe native shutdown and lock release. transport.request_terminal(false); @@ -290,6 +292,9 @@ async fn stop_after_admitted_callback(mode: DrainStop) { native_cleanup_finished(&transport), ).await; let request_outcome = request.await; + // Observe the unrelated RPC only after the original stop and real cleanup. + // Its denial is measured at the task's actual execution phase. + let new_call_denied = denial.await; drop(probe_release_on_failure); let observer_outcome = match observer { Some(observer) => Some(observer.await), From 5fc7575dc52025d5c7181a2eba3fe1f0d4ad8435 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 16:23:40 +0300 Subject: [PATCH 030/130] chore: files changed crates/tinybus/src/module/host_callback_lifecycle_tests.rs Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../module/host_callback_lifecycle_tests.rs | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs index 00a31164..ac770696 100644 --- a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs +++ b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs @@ -207,16 +207,16 @@ async fn stop_after_admitted_callback(mode: DrainStop) { Some(tokio::spawn(async move { // Always release the observer, including a failed watchdog. let _release = CallbackRelease(probe_release); - let entered = tokio::time::timeout( - Duration::from_secs(5), - probe_entered.acquire(), - ).await; - let completed = tokio::time::timeout( - Duration::from_secs(1), - native_cleanup_finished(&transport), - ).await; - eprintln!("denial observer: entered={} actual_cleanup_while_held={}", - entered.is_ok(), completed.is_ok()); + let entered = + tokio::time::timeout(Duration::from_secs(5), probe_entered.acquire()).await; + let completed = + tokio::time::timeout(Duration::from_secs(1), native_cleanup_finished(&transport)) + .await; + eprintln!( + "denial observer: entered={} actual_cleanup_while_held={}", + entered.is_ok(), + completed.is_ok() + ); (entered.is_ok(), completed.is_ok()) })) } else { @@ -281,16 +281,16 @@ async fn stop_after_admitted_callback(mode: DrainStop) { host.inner.stop(&loaded.name, Duration::from_secs(2)).await }; let stop_elapsed = stop_started.elapsed(); - let stop_code = stopped.as_ref().err().map(|error| error.wire_name().to_owned()); - let drained = tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()) - .await; + let stop_code = stopped + .as_ref() + .err() + .map(|error| error.wire_name().to_owned()); + let drained = tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()).await; // An error is not evidence of reclamation. Complete the retained ordinary // terminal job and independently observe native shutdown and lock release. transport.request_terminal(false); - let cleanup = tokio::time::timeout( - Duration::from_secs(2), - native_cleanup_finished(&transport), - ).await; + let cleanup = + tokio::time::timeout(Duration::from_secs(2), native_cleanup_finished(&transport)).await; let request_outcome = request.await; // Observe the unrelated RPC only after the original stop and real cleanup. // Its denial is measured at the task's actual execution phase. @@ -300,19 +300,31 @@ async fn stop_after_admitted_callback(mode: DrainStop) { Some(observer) => Some(observer.await), None => None, }; - eprintln!("stop fixture: code={stop_code:?} elapsed={stop_elapsed:?} cleanup={}", - cleanup.is_ok()); + eprintln!( + "stop fixture: code={stop_code:?} elapsed={stop_elapsed:?} cleanup={}", + cleanup.is_ok() + ); assert!(cleanup.is_ok(), "actual native cleanup did not finish"); assert!(request_outcome.is_ok()); assert!(drained.is_ok()); if let Some(outcome) = observer_outcome { - assert!(outcome.is_ok_and(|(entered, completed)| entered && completed), - "actual native cleanup waited for the unrelated denial observer"); + assert!( + outcome.is_ok_and(|(entered, completed)| entered && completed), + "actual native cleanup waited for the unrelated denial observer" + ); } - assert!(stopped.is_ok(), "stop code={stop_code:?}, elapsed={stop_elapsed:?}"); + assert!( + stopped.is_ok(), + "stop code={stop_code:?}, elapsed={stop_elapsed:?}" + ); assert_eq!(committed.load(Ordering::Acquire), 1); assert_eq!(still_running, 1); - assert!(authority_denied && owner_retained && enable_denied && new_call_denied.is_ok_and(|denied| denied)); + assert!( + authority_denied + && owner_retained + && enable_denied + && new_call_denied.is_ok_and(|denied| denied) + ); assert!(!shutdown_started_early && !physically_stopped_early); assert_eq!( terminal_while_held, From 73557c4a91c216fcaf9986746bc9b6733168bed6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 16:44:47 +0300 Subject: [PATCH 031/130] feat(module): add terminal runtime hooks and completion types Expose a terminal module providing runtime hooks and completion types so modules can observe native instance liveness and terminal reasons. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 3 ++ crates/tinybus-module/src/terminal.rs | 43 +++++++++++++++++++++ crates/tinybus-module/src/terminal_tests.rs | 25 ++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 crates/tinybus-module/src/terminal.rs create mode 100644 crates/tinybus-module/src/terminal_tests.rs diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 2e919592..c2cf85bf 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -26,6 +26,9 @@ use tinybus::module::abi::{ use tinybus::{Connection, Error, Result, Transport}; use tokio::sync::{Mutex, mpsc}; +mod terminal; +pub use terminal::{NativeInstanceLiveness, RuntimeHooks, TerminalCompletion, TerminalReason}; + mod execution_context; use execution_context::ExecutionOwner; diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs new file mode 100644 index 00000000..9800341f --- /dev/null +++ b/crates/tinybus-module/src/terminal.rs @@ -0,0 +1,43 @@ +//! Opted-in instance terminal authority and bounded physical cleanup ownership. +use super::*; +use tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE; + +/// Cause of irreversible instance admission closure. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TerminalReason { Stop, HostClosed, Fault, SetupFailed } +/// Actual controller cleanup outcome, independent of native admission. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TerminalCompletion { PhysicallyStopped, Uncertain } +/// Controller callbacks supplied by the terminal factory. +pub struct RuntimeHooks { + request: Arc, + finish: Box TerminalCompletion + Send>, +} +impl RuntimeHooks { + /// `request` must only close admission and signal bounded cancel/wake. It + /// must not join, perform I/O, or wait for a gate. `finish` independently + /// fences and joins the controller after SDK setup cancellation has settled; + /// it must not depend on those cancelled futures or on host shutdown. + pub fn new(request: R, finish: F) -> Self + where R: Fn(TerminalReason) + Send + Sync + 'static, + F: FnOnce() -> TerminalCompletion + Send + 'static { + Self { request: Arc::new(request), finish: Box::new(finish) } + } +} +/// A retained view of one actual admitted native instance. +#[derive(Clone)] +pub struct NativeInstanceLiveness { inner: Arc } +impl NativeInstanceLiveness { + /// Only a supported, exactly-live native query allows new admission. + pub fn is_live(&self) -> bool { + self.inner.host.0.native_terminal_state.is_some_and(|query| unsafe { + query(self.inner.host.0.host_ctx) == TB_NATIVE_TERMINAL_LIVE + }) + } + /// Persist genuine fault and close independent native admission first. + pub fn request_fault(&self) { self.inner.host.fault(); } +} +struct TerminalState { host: HostCalls } +#[cfg(test)] +#[path = "terminal_tests.rs"] +mod tests; diff --git a/crates/tinybus-module/src/terminal_tests.rs b/crates/tinybus-module/src/terminal_tests.rs new file mode 100644 index 00000000..2d1184f5 --- /dev/null +++ b/crates/tinybus-module/src/terminal_tests.rs @@ -0,0 +1,25 @@ +use super::*; +use std::sync::atomic::AtomicU32; +use tinybus::module::abi::{TbSlice, TB_NATIVE_TERMINAL_STOP_INTENT}; +static FLAGS: AtomicU32 = AtomicU32::new(TB_NATIVE_TERMINAL_LIVE); +unsafe extern "C" fn query(_: *mut c_void) -> u32 { FLAGS.load(Ordering::Acquire) } +unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { TB_OK } +unsafe extern "C" fn notify(_: *mut c_void) {} +unsafe extern "C" fn fault(_: *mut c_void, _: *const u8, _: usize) { FLAGS.store(4, Ordering::Release); } +unsafe extern "C" fn log(_: *mut c_void, _: u32, _: *const u8, _: usize) {} +fn host() -> HostCalls { + HostCalls(TbHostVtable { size: size_of::() as u32, _reserved: 0, + host_ctx: std::ptr::null_mut(), send, wake: notify, log, ready: notify, + fault, config: TbSlice { ptr: std::ptr::null(), len: 0 }, broker_routing: None, + native_terminal_state: Some(query) }) +} +#[test] +fn observed_native_terminal_never_reopens_a_retained_view() { + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + let view = NativeInstanceLiveness { inner: Arc::new(TerminalState { host: host() }) }; + assert!(view.is_live()); + FLAGS.store(TB_NATIVE_TERMINAL_STOP_INTENT, Ordering::Release); + assert!(!view.is_live()); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + assert!(!view.is_live(), "an observed terminal tombstone must stay closed"); +} From 5ee28a3e1a54f6282cb509e83570c45666d43477 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 16:45:41 +0300 Subject: [PATCH 032/130] fix(terminal): latch native liveness closed once it stops being live NativeInstanceLiveness now records a closed flag so a retained view cannot report live again after a failed query or a requested fault. The flag is set on the first non-live observation and on request_fault, and is_live checks it before and after the native query. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/terminal.rs | 11 +++++++---- crates/tinybus-module/src/terminal_tests.rs | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 9800341f..6cf85b9c 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -30,14 +30,17 @@ pub struct NativeInstanceLiveness { inner: Arc } impl NativeInstanceLiveness { /// Only a supported, exactly-live native query allows new admission. pub fn is_live(&self) -> bool { - self.inner.host.0.native_terminal_state.is_some_and(|query| unsafe { + if self.inner.closed.load(Ordering::Acquire) { return false; } + let live = self.inner.host.0.native_terminal_state.is_some_and(|query| unsafe { query(self.inner.host.0.host_ctx) == TB_NATIVE_TERMINAL_LIVE - }) + }); + if !live { self.inner.closed.store(true, Ordering::Release); } + live && !self.inner.closed.load(Ordering::Acquire) } /// Persist genuine fault and close independent native admission first. - pub fn request_fault(&self) { self.inner.host.fault(); } + pub fn request_fault(&self) { self.inner.host.fault(); self.inner.closed.store(true, Ordering::Release); } } -struct TerminalState { host: HostCalls } +struct TerminalState { host: HostCalls, closed: AtomicBool } #[cfg(test)] #[path = "terminal_tests.rs"] mod tests; diff --git a/crates/tinybus-module/src/terminal_tests.rs b/crates/tinybus-module/src/terminal_tests.rs index 2d1184f5..a5c90e9b 100644 --- a/crates/tinybus-module/src/terminal_tests.rs +++ b/crates/tinybus-module/src/terminal_tests.rs @@ -16,7 +16,7 @@ fn host() -> HostCalls { #[test] fn observed_native_terminal_never_reopens_a_retained_view() { FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); - let view = NativeInstanceLiveness { inner: Arc::new(TerminalState { host: host() }) }; + let view = NativeInstanceLiveness { inner: Arc::new(TerminalState { host: host(), closed: AtomicBool::new(false) }) }; assert!(view.is_live()); FLAGS.store(TB_NATIVE_TERMINAL_STOP_INTENT, Ordering::Release); assert!(!view.is_live()); From cfe905bb6450ff7ca4cba1a9e15a08621ccdf069 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 16:49:33 +0300 Subject: [PATCH 033/130] feat(terminal): add bounded instance admission and cleanup ownership Replace the single closed flag with a slot-based admission table that caps live instances and retains each instance's permit until its state is truly dropped, so a pending runtime teardown can never recycle a slot. Terminal state now records a reason with fault precedence, runs request and finish hooks under panic containment, and reports PhysicallyStopped only when fence, join, and resource release are all proven. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/terminal.rs | 337 +++++++++++++++++++- crates/tinybus-module/src/terminal_tests.rs | 6 +- 2 files changed, 326 insertions(+), 17 deletions(-) diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 6cf85b9c..f448a96f 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -1,8 +1,13 @@ //! Opted-in instance terminal authority and bounded physical cleanup ownership. use super::*; -use tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE; +use std::any::Any; +use std::pin::Pin; +use std::sync::{Condvar, Weak}; +use std::sync::atomic::{AtomicU8, AtomicUsize}; +use tinybus::module::abi::{TB_NATIVE_TERMINAL_LIVE, TB_NATIVE_TERMINAL_FAULT, TB_NATIVE_TERMINAL_STOP_INTENT}; -/// Cause of irreversible instance admission closure. +/// Cause of irreversible instance admission closure. Genuine fault takes +/// precedence over ordinary stop or host disappearance. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum TerminalReason { Stop, HostClosed, Fault, SetupFailed } /// Actual controller cleanup outcome, independent of native admission. @@ -15,32 +20,332 @@ pub struct RuntimeHooks { } impl RuntimeHooks { /// `request` must only close admission and signal bounded cancel/wake. It - /// must not join, perform I/O, or wait for a gate. `finish` independently - /// fences and joins the controller after SDK setup cancellation has settled; - /// it must not depend on those cancelled futures or on host shutdown. + /// must not join, perform I/O, reenter SDK APIs, or wait for a gate. `finish` + /// independently fences and joins the controller after SDK setup cancellation + /// has settled; it must not depend on cancelled futures or host shutdown. + /// Returning `PhysicallyStopped` asserts actual fence/join, not a timeout. pub fn new(request: R, finish: F) -> Self where R: Fn(TerminalReason) + Send + Sync + 'static, F: FnOnce() -> TerminalCompletion + Send + 'static { Self { request: Arc::new(request), finish: Box::new(finish) } } } -/// A retained view of one actual admitted native instance. +/// A retained view of one actual admitted native instance. Its lifetime retains +/// the instance's admission permit even after successful physical cleanup. #[derive(Clone)] -pub struct NativeInstanceLiveness { inner: Arc } +pub struct NativeInstanceLiveness { pub(super) inner: Arc } impl NativeInstanceLiveness { /// Only a supported, exactly-live native query allows new admission. - pub fn is_live(&self) -> bool { - if self.inner.closed.load(Ordering::Acquire) { return false; } - let live = self.inner.host.0.native_terminal_state.is_some_and(|query| unsafe { - query(self.inner.host.0.host_ctx) == TB_NATIVE_TERMINAL_LIVE + /// Observed terminal state never reopens. This is not a publication lease. + pub fn is_live(&self) -> bool { self.inner.is_live() } + /// Persist genuine fault and close independent native admission first. + pub fn request_fault(&self) { self.inner.signal(TerminalReason::Fault); } +} +const INSTANCE_LIMIT: usize = 256; +#[derive(Default)] +struct Slot { reserved: bool, anchor: Option> } +struct Admission { slots: StdMutex<[Slot; INSTANCE_LIMIT]> } +impl Admission { + fn new() -> Arc { + Arc::new(Self { slots: StdMutex::new(std::array::from_fn(|_| Slot::default())) }) + } +} +static ADMISSION: OnceLock> = OnceLock::new(); +struct SlotLease { admission: Weak, index: usize } +impl Drop for SlotLease { + fn drop(&mut self) { + if let Some(admission) = self.admission.upgrade() { + let mut slots = admission.slots.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + // The permit ends only at actual State last-Drop, never at anchor retirement. + slots[self.index].reserved = false; + } + } +} +#[derive(Clone, Copy, Eq, PartialEq)] +enum RequestState { Unregistered, Idle, Running, Done, Panicked } +#[derive(Clone, Copy, Eq, PartialEq)] +enum FinishState { Idle, Submitted, Running, Complete, SpawnFailed } +struct FinishJob { state: Weak } +impl FinishJob { + fn run(self: Arc, state: Arc) { state.run_finish(); } +} +type Payload = Box; +#[derive(Default)] +struct PrimaryPayloads { + request: Option, finish: Option, initial: Option, + reinit: Option, factory: Option, startup: Option, + cleanup: Option, +} +#[derive(Clone, Copy)] +pub(super) enum PayloadKind { Request, Finish, Initial, Reinit, Factory, Startup, Cleanup } +impl PrimaryPayloads { + fn cell(&mut self, kind: PayloadKind) -> &mut Option { + match kind { PayloadKind::Request => &mut self.request, PayloadKind::Finish => &mut self.finish, + PayloadKind::Initial => &mut self.initial, PayloadKind::Reinit => &mut self.reinit, + PayloadKind::Factory => &mut self.factory, PayloadKind::Startup => &mut self.startup, + PayloadKind::Cleanup => &mut self.cleanup } + } +} +struct Control { + request: Option>, + finish: Option TerminalCompletion + Send>>, + request_state: RequestState, finish_state: FinishState, + published: bool, completion: Option, job: Option>, + payloads: PrimaryPayloads, +} +type Setup = Box Pin> + Send>> + Send>; +/// Resources are attached here before the next fallible startup boundary. +/// Taking values is mechanical; all foreign destruction occurs off this lock. +pub(super) struct StartupResources { + pub runtime: Option, + pub setup: Option, + pub retained_factory: Option>, + pub config: Option>, + pub reinitializer_factory: Option, + pub reinitialize: Arc>>, + pub connection: Arc>>, + pub inbound: Option>, + pub initial: Option>>, +} +impl Default for StartupResources { + fn default() -> Self { + Self { runtime: None, setup: None, retained_factory: None, config: None, + reinitializer_factory: None, reinitialize: Arc::new(StdMutex::new(None)), + connection: Arc::new(StdMutex::new(None)), inbound: None, initial: None } + } +} +pub(super) struct TerminalState { + pub host: HostCalls, pub owner: Option, + reason: AtomicU8, uncertain: AtomicBool, + control: StdMutex, changed: Condvar, + pub terminal_notify: tokio::sync::Notify, + pub resources: StdMutex, + sdk_owners: AtomicUsize, + lease: SlotLease, +} +/// Every surviving SDK transport/Connection/task retains this actual State. +/// A pending runtime teardown may therefore never recycle its slot. +pub(super) struct SdkOwner { state: Arc } +impl Drop for SdkOwner { + fn drop(&mut self) { self.state.sdk_owners.fetch_sub(1, Ordering::AcqRel); } +} +impl TerminalState { + pub fn admit(host: HostCalls, owner: Option) -> Option> { + Self::admit_in(ADMISSION.get_or_init(Admission::new), host, owner) + } + fn admit_in(admission: &Arc, host: HostCalls, owner: Option) -> Option> { + let mut slots = admission.slots.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let index = slots.iter().position(|slot| !slot.reserved)?; + slots[index].reserved = true; + let state = Arc::new(Self { + host, owner, reason: AtomicU8::new(0), uncertain: AtomicBool::new(false), + control: StdMutex::new(Control { request: None, finish: None, + request_state: RequestState::Unregistered, finish_state: FinishState::Idle, + published: false, completion: None, job: None, payloads: PrimaryPayloads::default() }), + changed: Condvar::new(), terminal_notify: tokio::sync::Notify::new(), + resources: StdMutex::new(StartupResources::default()), sdk_owners: AtomicUsize::new(0), + lease: SlotLease { admission: Arc::downgrade(admission), index }, }); - if !live { self.inner.closed.store(true, Ordering::Release); } - live && !self.inner.closed.load(Ordering::Acquire) + slots[index].anchor = Some(state.clone()); + Some(state) + } + pub fn view(self: &Arc) -> NativeInstanceLiveness { NativeInstanceLiveness { inner: self.clone() } } + pub fn sdk_owner(self: &Arc) -> SdkOwner { + self.sdk_owners.fetch_add(1, Ordering::AcqRel); SdkOwner { state: self.clone() } + } + pub fn is_terminal(&self) -> bool { self.reason.load(Ordering::Acquire) != 0 } + pub fn is_live(self: &Arc) -> bool { + if self.is_terminal() { return false; } + let flags = self.host.0.native_terminal_state.map(|query| unsafe { query(self.host.0.host_ctx) }); + if flags != Some(TB_NATIVE_TERMINAL_LIVE) { + let reason = if flags.is_some_and(|flags| flags & TB_NATIVE_TERMINAL_FAULT != 0) { + TerminalReason::Fault + } else if flags.is_some_and(|flags| flags & TB_NATIVE_TERMINAL_STOP_INTENT != 0) { + TerminalReason::Stop + } else { TerminalReason::HostClosed }; + self.signal(reason); + return false; + } + !self.is_terminal() + } + fn reason(&self) -> TerminalReason { + match self.reason.load(Ordering::Acquire) { 1 => TerminalReason::Stop, 2 => TerminalReason::HostClosed, + 4 => TerminalReason::SetupFailed, _ => TerminalReason::Fault } + } + fn commit(&self, reason: TerminalReason) { + let next = match reason { TerminalReason::Stop => 1, TerminalReason::HostClosed => 2, + TerminalReason::Fault => 3, TerminalReason::SetupFailed => 4 }; + let _ = self.reason.fetch_update(Ordering::AcqRel, Ordering::Acquire, |old| + if old == 0 || (old < 3 && next >= 3) { Some(next) } else { None }); + self.terminal_notify.notify_waiters(); + } + pub fn signal(self: &Arc, reason: TerminalReason) { + if matches!(reason, TerminalReason::Fault | TerminalReason::SetupFailed) { self.host.fault(); } + self.commit(reason); + self.run_request(); + self.schedule(); + } + pub fn source_panic(self: &Arc, kind: PayloadKind, payload: Payload) { + // Atomic authority fence must precede payload-cell locks and foreign Drop. + self.host.fault(); self.commit(TerminalReason::Fault); + self.uncertain.store(true, Ordering::Release); + self.keep_payload(kind, payload); + self.run_request(); self.schedule(); + } + fn lock(&self) -> std::sync::MutexGuard<'_, Control> { + self.control.lock().unwrap_or_else(|poison| { + self.host.fault(); self.commit(TerminalReason::Fault); + self.uncertain.store(true, Ordering::Release); poison.into_inner() + }) + } + fn keep_payload(&self, kind: PayloadKind, payload: Payload) { + let mut control = self.lock(); + let cell = control.payloads.cell(kind); + // Once/terminal admission prevents a second primary in this category. + // A violated internal invariant cannot discard a foreign payload here. + if cell.is_none() { *cell = Some(payload); } + else { std::mem::forget(payload); } + } + pub fn register(self: &Arc, hooks: RuntimeHooks) { + { let mut control = self.lock(); + control.request = Some(hooks.request); control.finish = Some(hooks.finish); + control.request_state = RequestState::Idle; + } + if self.is_terminal() { self.run_request(); self.schedule(); } + } + fn run_request(self: &Arc) { + let request = { + let mut control = self.lock(); + if !self.is_terminal() || control.request_state != RequestState::Idle { return; } + control.request_state = RequestState::Running; + control.request.clone() + }; + let _scope = self.owner.as_ref().map(ExecutionOwner::enter); + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Some(request) = request { request(self.reason()); } + })); + match outcome { + Ok(()) => { self.lock().request_state = RequestState::Done; self.changed.notify_all(); } + Err(payload) => { + self.host.fault(); self.commit(TerminalReason::Fault); + self.uncertain.store(true, Ordering::Release); + self.lock().request_state = RequestState::Panicked; + self.changed.notify_all(); + self.keep_payload(PayloadKind::Request, payload); + } + } + } + pub fn publish(self: &Arc) { + self.lock().published = true; self.changed.notify_all(); + if self.is_terminal() { self.run_request(); self.schedule(); } + } + fn schedule(self: &Arc) { + let job = { + let mut control = self.lock(); + if control.finish_state != FinishState::Idle { return; } + let job = Arc::new(FinishJob { state: Arc::downgrade(self) }); + control.job = Some(job.clone()); control.finish_state = FinishState::Submitted; job + }; + let state = job.state.upgrade().expect("anchored terminal state"); + if std::thread::Builder::new().name("tinybus-terminal".into()).spawn(move || job.run(state)).is_err() { + self.uncertain.store(true, Ordering::Release); + let mut control = self.lock(); control.finish_state = FinishState::SpawnFailed; + control.completion = Some(TerminalCompletion::Uncertain); drop(control); self.changed.notify_all(); + } + } + fn remove_anchor(&self) { + if let Some(admission) = self.lease.admission.upgrade() { + let anchor = { + let mut slots = admission.slots.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + slots[self.lease.index].anchor.take() + }; + drop(anchor); + } + } + pub fn empty_factory_failure(self: &Arc) { + // No callable/controller was transferred. Retained views still hold the permit. + self.commit(TerminalReason::SetupFailed); self.remove_anchor(); + } + fn run_finish(self: &Arc) { + let _scope = self.owner.as_ref().map(ExecutionOwner::enter); + let finish = { + let mut control = self.lock(); + while !control.published || matches!(control.request_state, RequestState::Idle | RequestState::Running) { + control = self.changed.wait(control).unwrap_or_else(std::sync::PoisonError::into_inner); + } + control.finish_state = FinishState::Running; control.finish.take() + }; + let cleanup = catch_unwind(AssertUnwindSafe(|| self.settle_setup())); + let setup_settled = match cleanup { + Ok(settled) => settled, + Err(payload) => { self.source_panic(PayloadKind::Cleanup, payload); false } + }; + let completed = if setup_settled { + if let Some(owner) = &self.owner { owner.stop(); } + match catch_unwind(AssertUnwindSafe(|| finish.map_or(TerminalCompletion::PhysicallyStopped, |finish| finish()))) { + Ok(completion) => completion, + Err(payload) => { self.source_panic(PayloadKind::Finish, payload); TerminalCompletion::Uncertain } + } + } else { + // If setup destruction could not be proved, keep the original uninvoked finish owner. + self.lock().finish = finish; TerminalCompletion::Uncertain + }; + let released = if completed == TerminalCompletion::PhysicallyStopped { + match catch_unwind(AssertUnwindSafe(|| self.release_resources())) { + Ok(released) => released, + Err(payload) => { self.source_panic(PayloadKind::Cleanup, payload); false } + } + } else { false }; + if !released { self.uncertain.store(true, Ordering::Release); } + let completion = if self.uncertain.load(Ordering::Acquire) { TerminalCompletion::Uncertain } else { completed }; + { let mut control = self.lock(); control.finish_state = FinishState::Complete; control.completion = Some(completion); } + self.changed.notify_all(); + if completion == TerminalCompletion::PhysicallyStopped { self.remove_anchor(); } + } + fn settle_setup(self: &Arc) -> bool { + let (initial, handle, reinitialize) = { + let mut resources = self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + (resources.initial.take(), resources.runtime.as_ref().map(|runtime| runtime.handle().clone()), resources.reinitialize.clone()) + }; + if let Some(initial) = initial { + initial.abort(); + let Some(handle) = handle else { self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).initial = Some(initial); return false; }; + match handle.block_on(initial) { + Ok(Ok(())) => {}, + Ok(Err(_error)) => self.signal(TerminalReason::SetupFailed), + Err(error) if error.is_cancelled() => {}, + Err(error) => { + if error.is_panic() { self.source_panic(PayloadKind::Initial, error.into_panic()); } + else { self.signal(TerminalReason::SetupFailed); } + } + } + } + let callback = reinitialize.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + if let Some(owner) = &self.owner { owner.stop(); } + drop(callback); + true + } + fn release_resources(&self) -> bool { + let (setup, factory, config, reinit_factory, connection, inbound) = { + let mut resources = self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + (resources.setup.take(), resources.retained_factory.take(), resources.config.take(), + resources.reinitializer_factory.take(), resources.connection.clone(), resources.inbound.take()) + }; + let connection = connection.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + drop((setup, factory, config, reinit_factory, connection, inbound)); + let runtime = self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).runtime.take(); + if let Some(runtime) = runtime { runtime.shutdown_timeout(Duration::from_secs(4)); } + // Timeout is not proof: any surviving SDK transport/task keeps State + // and its resource count. Unknown surviving ownership stays anchored. + self.sdk_owners.load(Ordering::Acquire) == 0 + } + pub fn wait(&self, deadline: Duration) -> TerminalCompletion { + let control = self.lock(); + let (control, _) = self.changed.wait_timeout_while(control, deadline, |control| control.completion.is_none()) + .unwrap_or_else(std::sync::PoisonError::into_inner); + control.completion.unwrap_or(TerminalCompletion::Uncertain) } - /// Persist genuine fault and close independent native admission first. - pub fn request_fault(&self) { self.inner.host.fault(); self.inner.closed.store(true, Ordering::Release); } } -struct TerminalState { host: HostCalls, closed: AtomicBool } #[cfg(test)] #[path = "terminal_tests.rs"] mod tests; diff --git a/crates/tinybus-module/src/terminal_tests.rs b/crates/tinybus-module/src/terminal_tests.rs index a5c90e9b..8222f563 100644 --- a/crates/tinybus-module/src/terminal_tests.rs +++ b/crates/tinybus-module/src/terminal_tests.rs @@ -16,10 +16,14 @@ fn host() -> HostCalls { #[test] fn observed_native_terminal_never_reopens_a_retained_view() { FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); - let view = NativeInstanceLiveness { inner: Arc::new(TerminalState { host: host(), closed: AtomicBool::new(false) }) }; + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let view = state.view(); assert!(view.is_live()); FLAGS.store(TB_NATIVE_TERMINAL_STOP_INTENT, Ordering::Release); assert!(!view.is_live()); FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); assert!(!view.is_live(), "an observed terminal tombstone must stay closed"); + state.publish(); + assert_eq!(state.wait(Duration::from_secs(3)), TerminalCompletion::PhysicallyStopped); } From d01d1dac39f95effcaeb7e2075793906442067c5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 16:50:46 +0300 Subject: [PATCH 034/130] test(terminal): cover permit anchoring and request completion Add tests for terminal admission and shutdown edge cases: a retired anchor must not refund its permit while an old view survives, a panicking request payload is retained and finish is attempted exactly once, physical finish waits for bounded request completion and startup publication, and a surviving SDK-owned transport keeps cleanup uncertain with the permit anchored. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/terminal_tests.rs | 81 +++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/crates/tinybus-module/src/terminal_tests.rs b/crates/tinybus-module/src/terminal_tests.rs index 8222f563..464586ac 100644 --- a/crates/tinybus-module/src/terminal_tests.rs +++ b/crates/tinybus-module/src/terminal_tests.rs @@ -27,3 +27,84 @@ fn observed_native_terminal_never_reopens_a_retained_view() { state.publish(); assert_eq!(state.wait(Duration::from_secs(3)), TerminalCompletion::PhysicallyStopped); } + +#[test] +fn retired_anchor_cannot_refund_permit_while_old_view_survives() { + let admission = Admission::new(); + let mut views = Vec::new(); + for _ in 0..INSTANCE_LIMIT { + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + views.push(state.view()); + state.empty_factory_failure(); + } + assert!(TerminalState::admit_in(&admission, host(), None).is_none()); + views.pop(); + let replacement = TerminalState::admit_in(&admission, host(), None).unwrap(); + replacement.empty_factory_failure(); + drop(replacement); + drop(views); + assert!(admission.slots.lock().unwrap().iter().all(|slot| !slot.reserved)); +} + +#[test] +fn request_primary_payload_is_retained_and_finish_still_attempted_once() { + use std::sync::atomic::AtomicUsize; + struct PayloadDrop(Arc); + impl Drop for PayloadDrop { fn drop(&mut self) { self.0.fetch_add(1, Ordering::SeqCst); } } + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let dropped = Arc::new(AtomicUsize::new(0)); + let requests = Arc::new(AtomicUsize::new(0)); + let finishes = Arc::new(AtomicUsize::new(0)); + let (request_count, finish_count, payload) = (requests.clone(), finishes.clone(), dropped.clone()); + state.register(RuntimeHooks::new(move |_| { + request_count.fetch_add(1, Ordering::SeqCst); + std::panic::panic_any(PayloadDrop(payload.clone())); + }, move || { + finish_count.fetch_add(1, Ordering::SeqCst); + TerminalCompletion::PhysicallyStopped + })); + state.publish(); + state.signal(TerminalReason::Stop); + assert_eq!(state.wait(Duration::from_secs(3)), TerminalCompletion::Uncertain); + state.signal(TerminalReason::Fault); + assert_eq!(requests.load(Ordering::SeqCst), 1); + assert_eq!(finishes.load(Ordering::SeqCst), 1); + assert_eq!(dropped.load(Ordering::SeqCst), 0); + assert!(state.lock().request_state == RequestState::Panicked); + assert!(admission.slots.lock().unwrap()[state.lease.index].anchor.is_some()); +} + +#[test] +fn physical_finish_waits_for_bounded_request_completion_and_startup_publication() { + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let release = Arc::new(std::sync::Barrier::new(2)); + let request_release = release.clone(); + let finished = Arc::new(AtomicBool::new(false)); + let finish_seen = finished.clone(); + state.register(RuntimeHooks::new(move |_| { + entered_tx.send(()).unwrap(); request_release.wait(); + }, move || { finish_seen.store(true, Ordering::Release); TerminalCompletion::PhysicallyStopped })); + let caller_state = state.clone(); + let caller = std::thread::spawn(move || caller_state.signal(TerminalReason::Stop)); + entered_rx.recv().unwrap(); + state.publish(); + assert!(!finished.load(Ordering::Acquire)); + release.wait(); caller.join().unwrap(); + assert_eq!(state.wait(Duration::from_secs(3)), TerminalCompletion::PhysicallyStopped); + assert!(finished.load(Ordering::Acquire)); +} + +#[test] +fn surviving_sdk_owned_transport_keeps_cleanup_uncertain_and_permit_anchored() { + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let sdk_owner = state.sdk_owner(); + state.register(RuntimeHooks::new(|_| {}, || TerminalCompletion::PhysicallyStopped)); + state.publish(); state.signal(TerminalReason::Stop); + assert_eq!(state.wait(Duration::from_secs(3)), TerminalCompletion::Uncertain); + drop(sdk_owner); + assert!(admission.slots.lock().unwrap()[state.lease.index].anchor.is_some()); +} From d211fc3ab259caceda3e8601f5131332246ce7cb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 16:51:26 +0300 Subject: [PATCH 035/130] chore: files changed crates/tinybus-module/src/terminal.rs Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/terminal.rs | 28 +++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index f448a96f..53b9908d 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -93,7 +93,9 @@ struct Control { published: bool, completion: Option, job: Option>, payloads: PrimaryPayloads, } -type Setup = Box Pin> + Send>> + Send>; +pub(super) type Setup = Box Pin> + Send>> + Send>; +pub(super) type SetupFutureSlot = Arc> + Send>>>>>; + /// Resources are attached here before the next fallible startup boundary. /// Taking values is mechanical; all foreign destruction occurs off this lock. pub(super) struct StartupResources { @@ -106,12 +108,14 @@ pub(super) struct StartupResources { pub connection: Arc>>, pub inbound: Option>, pub initial: Option>>, + pub initial_future: SetupFutureSlot, } impl Default for StartupResources { fn default() -> Self { Self { runtime: None, setup: None, retained_factory: None, config: None, reinitializer_factory: None, reinitialize: Arc::new(StdMutex::new(None)), - connection: Arc::new(StdMutex::new(None)), inbound: None, initial: None } + connection: Arc::new(StdMutex::new(None)), inbound: None, initial: None, + initial_future: Arc::new(StdMutex::new(None)) } } } pub(super) struct TerminalState { @@ -175,8 +179,12 @@ impl TerminalState { fn commit(&self, reason: TerminalReason) { let next = match reason { TerminalReason::Stop => 1, TerminalReason::HostClosed => 2, TerminalReason::Fault => 3, TerminalReason::SetupFailed => 4 }; - let _ = self.reason.fetch_update(Ordering::AcqRel, Ordering::Acquire, |old| - if old == 0 || (old < 3 && next >= 3) { Some(next) } else { None }); + let mut old = self.reason.load(Ordering::Acquire); + while old == 0 || (old < 3 && next >= 3) { + match self.reason.compare_exchange_weak(old, next, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => break, Err(actual) => old = actual, + } + } self.terminal_notify.notify_waiters(); } pub fn signal(self: &Arc, reason: TerminalReason) { @@ -203,8 +211,8 @@ impl TerminalState { let cell = control.payloads.cell(kind); // Once/terminal admission prevents a second primary in this category. // A violated internal invariant cannot discard a foreign payload here. - if cell.is_none() { *cell = Some(payload); } - else { std::mem::forget(payload); } + assert!(cell.is_none(), "terminal primary boundary is once-only"); + *cell = Some(payload); } pub fn register(self: &Arc, hooks: RuntimeHooks) { { let mut control = self.lock(); @@ -303,9 +311,9 @@ impl TerminalState { if completion == TerminalCompletion::PhysicallyStopped { self.remove_anchor(); } } fn settle_setup(self: &Arc) -> bool { - let (initial, handle, reinitialize) = { + let (initial, handle, reinitialize, future) = { let mut resources = self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - (resources.initial.take(), resources.runtime.as_ref().map(|runtime| runtime.handle().clone()), resources.reinitialize.clone()) + (resources.initial.take(), resources.runtime.as_ref().map(|runtime| runtime.handle().clone()), resources.reinitialize.clone(), resources.initial_future.clone()) }; if let Some(initial) = initial { initial.abort(); @@ -320,6 +328,8 @@ impl TerminalState { } } } + let future = future.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + drop(future); let callback = reinitialize.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); if let Some(owner) = &self.owner { owner.stop(); } drop(callback); @@ -339,6 +349,8 @@ impl TerminalState { // and its resource count. Unknown surviving ownership stays anchored. self.sdk_owners.load(Ordering::Acquire) == 0 } + pub fn has_fault(&self) -> bool { self.reason.load(Ordering::Acquire) >= 3 } + pub fn mark_uncertain(&self) { self.uncertain.store(true, Ordering::Release); } pub fn wait(&self, deadline: Duration) -> TerminalCompletion { let control = self.lock(); let (control, _) = self.changed.wait_timeout_while(control, deadline, |control| control.completion.is_none()) From 91a96e06ae5346cf8d3e89a6214e0d1ab83d19bd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 16:54:30 +0300 Subject: [PATCH 036/130] feat(module): add terminal reconfigurable module entry point Add a native entry point that admits an opted-in instance through the terminal state and invokes a caller-supplied factory with a liveness view. The factory result is discarded and the instance is closed, so unsupported hosts reject before the factory runs. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 27 ++++++++++++++++++ .../tinybus-module/src/lib_terminal_tests.rs | 28 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 crates/tinybus-module/src/lib_terminal_tests.rs diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index c2cf85bf..93e3004a 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1242,3 +1242,30 @@ mod setup_factory_test; #[cfg(test)] #[path = "lib_lifecycle_tests.rs"] mod lifecycle_test; + +/// Initialize an opted-in instance with native admission and physical hooks. +/// +/// # Safety +/// Uphold [`start_module`]'s ABI and output contracts. The compatible native +/// query and copied host context must remain valid for every retained view, +/// including after host destruction. Only the actual native query grants +/// authority; unsupported hosts reject the factory before invoking it. +#[doc(hidden)] +pub unsafe fn start_terminal_reconfigurable_module_with_factory( + host: *const TbHostVtable, out: *mut TbModuleVtable, worker_threads: usize, + _detach_on_panic: bool, factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, +) -> i32 +where C: serde::de::DeserializeOwned + Send + 'static, + F: Fn(Connection, C) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, +{ + let Some((_, compatible)) = (unsafe { validate_start_arguments(host, out, worker_threads) }) else { return TB_BAD_ARGUMENT; }; + let Some(state) = terminal::TerminalState::admit(HostCalls(compatible), None) else { return TB_CLOSED; }; + let _result = factory(state.view()); + state.empty_factory_failure(); + TB_CLOSED +} + +#[cfg(test)] +#[path = "lib_terminal_tests.rs"] +mod terminal_runtime_tests; diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs new file mode 100644 index 00000000..9a9dec60 --- /dev/null +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -0,0 +1,28 @@ +use super::*; +use std::sync::atomic::AtomicUsize; +use tinybus::module::abi::TbSlice; +unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { TB_OK } +unsafe extern "C" fn notify(_: *mut c_void) {} +unsafe extern "C" fn fault(_: *mut c_void, _: *const u8, _: usize) {} +unsafe extern "C" fn log(_: *mut c_void, _: u32, _: *const u8, _: usize) {} +fn host() -> TbHostVtable { + TbHostVtable { size: size_of::() as u32, _reserved: 0, + host_ctx: std::ptr::null_mut(), send, wake: notify, log, ready: notify, fault, + config: TbSlice { ptr: std::ptr::null(), len: 0 }, broker_routing: None, native_terminal_state: None } +} +fn output() -> TbModuleVtable { + TbModuleVtable { size: size_of::() as u32, _reserved: 0, + module_ctx: std::ptr::null_mut(), deliver, shutdown, reinitialize: None } +} +#[test] +fn native_query_is_required_before_terminal_factory_evaluation() { + let calls = AtomicUsize::new(0); + let code = unsafe { start_terminal_reconfigurable_module_with_factory::( + &host(), &mut output(), 1, true, |_| { + calls.fetch_add(1, Ordering::SeqCst); + Ok((|_: Connection, _: serde_json::Value| async { Ok(()) }, + RuntimeHooks::new(|_| {}, || TerminalCompletion::PhysicallyStopped))) + }) }; + assert_eq!(code, TB_CLOSED); + assert_eq!(calls.load(Ordering::SeqCst), 0, "missing native query must leave the factory untouched"); +} From 2bed745104e14eb919f0e7674f767d923f7b0294 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 16:57:39 +0300 Subject: [PATCH 037/130] feat(module): add linked terminal factory startup mode Adds a linked variant of the terminal reconfigurable module factory that retains the embedding process's tracing subscriber and panic hook, and routes both startup paths through a shared implementation. The terminal runtime now owns the module vtable callbacks and tracks the reinit future so it can be dropped during physical cleanup, and the once-only payload assertion is replaced with a comment since call sites already guarantee Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 240 +++++++++++++++++++++++++- crates/tinybus-module/src/terminal.rs | 12 +- 2 files changed, 243 insertions(+), 9 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 93e3004a..b8521cf2 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -269,6 +269,7 @@ struct ModuleTransport { inbound: Mutex>, provenance: Arc, detach_on_panic: bool, + _terminal_owner: Option, } #[async_trait] @@ -606,6 +607,7 @@ where inbound: Mutex::new(inbound_rx), provenance: Arc::new(AtomicBool::new(false)), detach_on_panic, + _terminal_owner: None, }); let connection_slot = Arc::new(StdMutex::new(None)); let task_connection_slot = connection_slot.clone(); @@ -1259,11 +1261,239 @@ where C: serde::de::DeserializeOwned + Send + 'static, F: Fn(Connection, C) -> Fut + Send + Sync + 'static, Fut: Future> + Send + 'static, { - let Some((_, compatible)) = (unsafe { validate_start_arguments(host, out, worker_threads) }) else { return TB_BAD_ARGUMENT; }; - let Some(state) = terminal::TerminalState::admit(HostCalls(compatible), None) else { return TB_CLOSED; }; - let _result = factory(state.view()); - state.empty_factory_failure(); - TB_CLOSED + unsafe { start_terminal_factory_mode(host, out, worker_threads, factory, false) } +} + +/// Linked counterpart with explicit native authority from the supplied host. +/// It retains the embedding process's tracing subscriber and panic hook. +/// +/// # Safety +/// Uphold [`start_terminal_reconfigurable_module_with_factory`]'s contracts. +#[doc(hidden)] +pub unsafe fn start_linked_terminal_reconfigurable_module_with_factory( + host: *const TbHostVtable, out: *mut TbModuleVtable, worker_threads: usize, + _detach_on_panic: bool, factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, +) -> i32 +where C: serde::de::DeserializeOwned + Send + 'static, + F: Fn(Connection, C) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, +{ + unsafe { start_terminal_factory_mode(host, out, worker_threads, factory, true) } +} + +// Process-lifetime C tombstone does not retain the actual State after proved +// successful shutdown. Surviving liveness views retain their own slot permits. +struct TerminalRuntimeContext { state: StdMutex>> } +impl TerminalRuntimeContext { + fn state(&self) -> Option> { + self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone() + } +} +unsafe extern "C" fn terminal_deliver(ctx: *mut c_void, ptr: *const u8, len: usize) -> i32 { + if ctx.is_null() || ptr.is_null() || len > MAX_FRAME_LEN { return TB_BAD_ARGUMENT; } + let context = unsafe { &*ctx.cast::() }; + let Some(state) = context.state() else { return TB_CLOSED; }; + if !state.is_live() { return TB_CLOSED; } + let bytes = unsafe { std::slice::from_raw_parts(ptr, len) }.to_vec(); + let brokered = state.host.is_broker_routed(); + let sender = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).inbound.clone(); + match sender.map(|sender| sender.try_send((bytes, brokered))) { + Some(Ok(())) => TB_OK, + Some(Err(mpsc::error::TrySendError::Full(_))) => TB_BACKPRESSURE, + _ => TB_CLOSED, + } +} +unsafe extern "C" fn terminal_shutdown(ctx: *mut c_void, deadline_ms: u64) -> i32 { + if ctx.is_null() { return TB_BAD_ARGUMENT; } + let context = unsafe { &*ctx.cast::() }; + let Some(state) = context.state() else { return TB_CLOSED; }; + let _scope = state.owner.as_ref().map(ExecutionOwner::enter); + state.is_live(); // sample actual host Fault before selecting ordinary Stop + state.signal(TerminalReason::Stop); + let completion = state.wait(Duration::from_millis(deadline_ms)); + // Genuine source/native Fault cannot earn accepted-vtable physical success. + state.is_live(); + if completion != TerminalCompletion::PhysicallyStopped || state.has_fault() { return TB_PANICKED; } + let retired = context.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + drop(retired); + TB_OK +} +unsafe extern "C" fn terminal_reinitialize(ctx: *mut c_void, ptr: *const u8, len: usize) -> i32 { + if ctx.is_null() || len > 1024 * 1024 || (ptr.is_null() && len != 0) { return TB_BAD_ARGUMENT; } + let context = unsafe { &*ctx.cast::() }; + let Some(state) = context.state() else { return TB_CLOSED; }; + if !state.is_live() { return TB_CLOSED; } + let _scope = state.owner.as_ref().map(ExecutionOwner::enter); + let callback = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reinitialize.clone(); + let callback = callback.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !state.is_live() { return TB_CLOSED; } + let Some(callback) = callback.as_ref() else { return TB_CLOSED; }; + match catch_unwind(AssertUnwindSafe(|| callback(ptr, len))) { + Ok(code) => code, + Err(payload) => { state.source_panic(terminal::PayloadKind::Reinit, payload); TB_PANICKED } + } +} + +// The task never owns the foreign future across a yield or on cancellation. +// Each poll takes/polls/reattaches without holding a resource mutex over user +// code. Physical cleanup joins the sole task, then destroys this retained future. +struct TerminalSetupPoll { + slot: terminal::SetupFutureSlot, state: Arc, kind: terminal::PayloadKind, +} +impl Future for TerminalSetupPoll { + type Output = Result<()>; + fn poll(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll { + let mut future = self.slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + let outcome = catch_unwind(AssertUnwindSafe(|| { + future.as_mut().map_or(std::task::Poll::Ready(Err(Error::ConnectionClosed)), |future| future.as_mut().poll(cx)) + })); + *self.slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = future; + match outcome { + Ok(std::task::Poll::Ready(Err(error))) => { + self.state.signal(TerminalReason::SetupFailed); std::task::Poll::Ready(Err(error)) + } + Ok(poll) => poll, + Err(payload) => { + self.state.source_panic(self.kind, payload); + std::task::Poll::Ready(Err(Error::failed("module setup panicked"))) + } + } + } +} +async fn terminal_notification(state: &terminal::TerminalState) { + loop { + let notification = state.terminal_notify.notified(); + tokio::pin!(notification); + notification.as_mut().enable(); + if state.is_terminal() { return; } + notification.await; + } +} + +unsafe fn start_terminal_factory_mode( + host: *const TbHostVtable, out: *mut TbModuleVtable, worker_threads: usize, + factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, linked: bool, +) -> i32 +where C: serde::de::DeserializeOwned + Send + 'static, + F: Fn(Connection, C) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, +{ + let Some((capacity, compatible)) = (unsafe { validate_start_arguments(host, out, worker_threads) }) else { return TB_BAD_ARGUMENT; }; + // Reject unavailable, unknown and terminal authority before any factory. + let Some(query) = compatible.native_terminal_state else { return TB_CLOSED; }; + if unsafe { query(compatible.host_ctx) } != tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE { return TB_CLOSED; } + let owner = (!linked).then(|| ExecutionOwner::new(HostCalls(compatible))); + let Some(state) = terminal::TerminalState::admit(HostCalls(compatible), owner) else { return TB_CLOSED; }; + if state.owner.is_some() { ExecutionOwner::install_panic_hook(); } + let _scope = state.owner.as_ref().map(ExecutionOwner::enter); + let result = catch_unwind(AssertUnwindSafe(|| { + let config = match unsafe { parse_config::(host) } { Ok(config) => config, Err(code) => return code }; + state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).config = Some(Box::new(config)); + let (setup, hooks) = match catch_unwind(AssertUnwindSafe(|| factory(state.view()))) { + Ok(Ok(returned)) => returned, + Ok(Err(_error)) => { state.signal(TerminalReason::SetupFailed); return TB_CLOSED; } + Err(payload) => { state.source_panic(terminal::PayloadKind::Factory, payload); return TB_PANICKED; } + }; + let setup = Arc::new(setup); + state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).retained_factory = Some(Box::new(setup.clone())); + state.register(hooks); + if !state.is_live() { return TB_CLOSED; } + let config = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).config.take(); + let config = *config.expect("attached config").downcast::().unwrap_or_else(|_| unreachable!("private config type")); + let initial_setup = setup.clone(); + state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).setup = Some(Box::new(move |connection| Box::pin(initial_setup(connection, config)))); + let weak = Arc::downgrade(&state); + let reinitializer: ReinitializeFactory = Box::new(move |handle, connection_slot| { + Box::new(move |ptr, len| { + let Some(state) = weak.upgrade() else { return TB_CLOSED; }; + if !state.is_live() { return TB_CLOSED; } + let bytes = if len == 0 { b"{}".as_slice() } else { unsafe { std::slice::from_raw_parts(ptr, len) } }; + let config = match serde_json::from_slice::(bytes) { Ok(config) => config, Err(_) => return TB_BAD_ARGUMENT }; + let connection = connection_slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone(); + let Some(connection) = connection else { return TB_CLOSED; }; + let slot = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reinit_future.clone(); + let previous = slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + drop(previous); + if !state.is_live() { return TB_CLOSED; } + let future = Box::pin(setup(connection, config)); + *slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); + let result = handle.block_on(async { + tokio::select! { biased; + result = TerminalSetupPoll { slot, state: state.clone(), kind: terminal::PayloadKind::Reinit } => result, + () = terminal_notification(&state) => Err(Error::ConnectionClosed), + } + }); + match result { Ok(()) if state.is_live() => TB_OK, Ok(()) | Err(_) => TB_CLOSED } + }) + }); + state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reinitializer_factory = Some(reinitializer); + let mut builder = tokio::runtime::Builder::new_multi_thread(); + if let Some(owner) = &state.owner { owner.configure(&mut builder); } + let runtime = match builder.worker_threads(worker_threads).enable_all().build() { + Ok(runtime) => runtime, Err(_) => { state.signal(TerminalReason::SetupFailed); return TB_CLOSED; } + }; + // Immediate mechanical transfer: no later startup failure drops Runtime on ABI. + state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).runtime = Some(runtime); + let (inbound_tx, inbound_rx) = mpsc::channel(MODULE_QUEUE_CAPACITY); + state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).inbound = Some(inbound_tx); + let transport = Arc::new(ModuleTransport { host: state.host, inbound: Mutex::new(inbound_rx), + provenance: Arc::new(AtomicBool::new(false)), detach_on_panic: false, _terminal_owner: Some(state.sdk_owner()) }); + let (handle, connection_slot, factory) = { + let mut resources = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + (resources.runtime.as_ref().expect("attached runtime").handle().clone(), resources.connection.clone(), resources.reinitializer_factory.take()) + }; + let callback = factory.map(|factory| factory(handle.clone(), connection_slot.clone())); + let callback_slot = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reinitialize.clone(); + *callback_slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = callback; + let task_state = state.clone(); + let task = handle.spawn(async move { + let connection = module_connection(transport); + if let Err(error) = connection.handshake().await { task_state.signal(TerminalReason::SetupFailed); return Err(error); } + connection.__set_panic_handler(Arc::new(|| Error::MethodFailed { + name: MODULE_PANIC_ERROR.to_owned(), message: "a module method panicked".to_owned(), + })); + let observer_state = Arc::downgrade(&task_state); + connection.__set_catch_observer(Arc::new(move || { + if let Some(state) = observer_state.upgrade() { + state.mark_uncertain(); state.signal(TerminalReason::Fault); + } + })); + // Connection is persisted independently of initial task lifetime. + *connection_slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(connection.clone()); + if !task_state.is_live() { return Ok(()); } + let setup = task_state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).setup.take(); + let future = match catch_unwind(AssertUnwindSafe(|| setup.map(|setup| setup(connection)))) { + Ok(Some(future)) => future, + Ok(None) => return Err(Error::ConnectionClosed), + Err(payload) => { task_state.source_panic(terminal::PayloadKind::Initial, payload); return Err(Error::failed("module setup construction panicked")); } + }; + let slot = task_state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).initial_future.clone(); + *slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); + let result = TerminalSetupPoll { slot, state: task_state.clone(), kind: terminal::PayloadKind::Initial }.await; + if result.is_ok() && task_state.is_live() { task_state.host.ready(); } + result + }); + state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).initial = Some(task); + let context = Box::into_raw(Box::new(TerminalRuntimeContext { state: StdMutex::new(Some(state.clone())) })).cast::(); + if capacity >= size_of::() as u32 { + unsafe { out.write(TbModuleVtable { size: size_of::() as u32, _reserved: 0, + module_ctx: context, deliver: terminal_deliver, shutdown: terminal_shutdown, reinitialize: Some(terminal_reinitialize) }); } + } else { + unsafe { out.cast::().write(TbModuleVtableV1Prefix { + size: TB_MODULE_VTABLE_BASE_SIZE, _reserved: 0, module_ctx: context, deliver: terminal_deliver, shutdown: terminal_shutdown }); } + } + TB_OK + })); + let code = match result { + Ok(code) => code, + Err(payload) => { state.source_panic(terminal::PayloadKind::Startup, payload); TB_PANICKED } + }; + if code != TB_OK { + if let Some(owner) = &state.owner { owner.stop(); } + if !state.is_terminal() { state.signal(TerminalReason::SetupFailed); } + } + state.publish(); + code } #[cfg(test)] diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 53b9908d..23651de6 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -109,13 +109,14 @@ pub(super) struct StartupResources { pub inbound: Option>, pub initial: Option>>, pub initial_future: SetupFutureSlot, + pub reinit_future: SetupFutureSlot, } impl Default for StartupResources { fn default() -> Self { Self { runtime: None, setup: None, retained_factory: None, config: None, reinitializer_factory: None, reinitialize: Arc::new(StdMutex::new(None)), connection: Arc::new(StdMutex::new(None)), inbound: None, initial: None, - initial_future: Arc::new(StdMutex::new(None)) } + initial_future: Arc::new(StdMutex::new(None)), reinit_future: Arc::new(StdMutex::new(None)) } } } pub(super) struct TerminalState { @@ -211,7 +212,8 @@ impl TerminalState { let cell = control.payloads.cell(kind); // Once/terminal admission prevents a second primary in this category. // A violated internal invariant cannot discard a foreign payload here. - assert!(cell.is_none(), "terminal primary boundary is once-only"); + // Call sites admit this category once and deny all later invocation. + // No assertion/unwind, replacement retry, or payload disposal path. *cell = Some(payload); } pub fn register(self: &Arc, hooks: RuntimeHooks) { @@ -311,9 +313,9 @@ impl TerminalState { if completion == TerminalCompletion::PhysicallyStopped { self.remove_anchor(); } } fn settle_setup(self: &Arc) -> bool { - let (initial, handle, reinitialize, future) = { + let (initial, handle, reinitialize, future, reinit_future) = { let mut resources = self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - (resources.initial.take(), resources.runtime.as_ref().map(|runtime| runtime.handle().clone()), resources.reinitialize.clone(), resources.initial_future.clone()) + (resources.initial.take(), resources.runtime.as_ref().map(|runtime| runtime.handle().clone()), resources.reinitialize.clone(), resources.initial_future.clone(), resources.reinit_future.clone()) }; if let Some(initial) = initial { initial.abort(); @@ -333,6 +335,8 @@ impl TerminalState { let callback = reinitialize.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); if let Some(owner) = &self.owner { owner.stop(); } drop(callback); + let future = reinit_future.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + drop(future); true } fn release_resources(&self) -> bool { From a81fcd4a15f904d85fd03c8af8d6ab9ea83f87dd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 16:58:47 +0300 Subject: [PATCH 038/130] refactor(tinybus-module): split terminal transport ownership from module transport The terminal SDK owner is moved out of ModuleTransport into a new TerminalTransport wrapper so that only opted-in Connections retain terminal State, leaving the legacy transport path unchanged. The terminal attach path now also selects the provenance-aware connection constructor when broker routing is enabled. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index b8521cf2..e2ed7b2c 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -269,7 +269,6 @@ struct ModuleTransport { inbound: Mutex>, provenance: Arc, detach_on_panic: bool, - _terminal_owner: Option, } #[async_trait] @@ -607,7 +606,6 @@ where inbound: Mutex::new(inbound_rx), provenance: Arc::new(AtomicBool::new(false)), detach_on_panic, - _terminal_owner: None, }); let connection_slot = Arc::new(StdMutex::new(None)); let task_connection_slot = connection_slot.clone(); @@ -1281,6 +1279,16 @@ where C: serde::de::DeserializeOwned + Send + 'static, unsafe { start_terminal_factory_mode(host, out, worker_threads, factory, true) } } +// Legacy transport remains unchanged; only opted-in Connections retain State. +struct TerminalTransport { inner: ModuleTransport, _owner: terminal::SdkOwner } +#[async_trait] +impl Transport for TerminalTransport { + async fn send(&self, message: Message) -> Result<()> { self.inner.send(message).await } + async fn recv(&self) -> Result> { self.inner.recv().await } + async fn close(&self) -> Result<()> { self.inner.close().await } + fn describe(&self) -> String { self.inner.describe() } +} + // Process-lifetime C tombstone does not retain the actual State after proved // successful shutdown. Surviving liveness views retain their own slot permits. struct TerminalRuntimeContext { state: StdMutex>> } @@ -1436,8 +1444,8 @@ where C: serde::de::DeserializeOwned + Send + 'static, state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).runtime = Some(runtime); let (inbound_tx, inbound_rx) = mpsc::channel(MODULE_QUEUE_CAPACITY); state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).inbound = Some(inbound_tx); - let transport = Arc::new(ModuleTransport { host: state.host, inbound: Mutex::new(inbound_rx), - provenance: Arc::new(AtomicBool::new(false)), detach_on_panic: false, _terminal_owner: Some(state.sdk_owner()) }); + let transport = Arc::new(TerminalTransport { inner: ModuleTransport { host: state.host, inbound: Mutex::new(inbound_rx), + provenance: Arc::new(AtomicBool::new(false)), detach_on_panic: false }, _owner: state.sdk_owner() }); let (handle, connection_slot, factory) = { let mut resources = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner); (resources.runtime.as_ref().expect("attached runtime").handle().clone(), resources.connection.clone(), resources.reinitializer_factory.take()) @@ -1447,7 +1455,12 @@ where C: serde::de::DeserializeOwned + Send + 'static, *callback_slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = callback; let task_state = state.clone(); let task = handle.spawn(async move { - let connection = module_connection(transport); + let provenance = transport.inner.provenance.clone(); + let brokered = transport.inner.host.0.broker_routing.is_some(); + let connection = if brokered { + // SAFETY: same private per-frame provenance contract as module_connection. + unsafe { Connection::__attach_module_with_provenance(transport, provenance) } + } else { Connection::attach(transport) }; if let Err(error) = connection.handshake().await { task_state.signal(TerminalReason::SetupFailed); return Err(error); } connection.__set_panic_handler(Arc::new(|| Error::MethodFailed { name: MODULE_PANIC_ERROR.to_owned(), message: "a module method panicked".to_owned(), From b776e9a5d1838c7186610a4be6cb794774fcca5e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:00:19 +0300 Subject: [PATCH 039/130] test(module): cover shutdown rechecking native fault after local stop Add a test asserting that a native terminal fault observed after a local stop and physical finish still surfaces as a panic rather than being masked by the local stop state. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 9a9dec60..235b7374 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -26,3 +26,30 @@ fn native_query_is_required_before_terminal_factory_evaluation() { assert_eq!(code, TB_CLOSED); assert_eq!(calls.load(Ordering::SeqCst), 0, "missing native query must leave the factory untouched"); } + +unsafe extern "C" fn native_query(ctx: *mut c_void) -> u32 { + unsafe { &*ctx.cast::() }.load(Ordering::Acquire) +} +#[test] +fn shutdown_rechecks_actual_native_fault_after_local_stop_and_physical_finish() { + let flags = Arc::new(std::sync::atomic::AtomicU32::new(tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE)); + let mut host = host(); + host.host_ctx = Arc::as_ptr(&flags).cast_mut().cast(); + host.native_terminal_state = Some(native_query); + let state = terminal::TerminalState::admit(HostCalls(host), None).unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let release = Arc::new(std::sync::Barrier::new(2)); + let worker_release = release.clone(); + state.register(RuntimeHooks::new(|_| {}, move || { + entered_tx.send(()).unwrap(); worker_release.wait(); TerminalCompletion::PhysicallyStopped + })); + state.publish(); + let context = Box::new(TerminalRuntimeContext { state: StdMutex::new(Some(state.clone())) }); + let ptr = (&*context as *const TerminalRuntimeContext) as usize; + let caller = std::thread::spawn(move || unsafe { terminal_shutdown(ptr as *mut c_void, 3000) }); + entered_rx.recv().unwrap(); + flags.store(tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, Ordering::Release); + release.wait(); + let code = caller.join().unwrap(); + assert_eq!(code, TB_PANICKED, "actual native Fault cannot disappear behind local Stop"); +} From c230af36c13f5388ce5b28796ac8d0c5a82880b8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:01:36 +0300 Subject: [PATCH 040/130] fix(module): sample host fault during terminal shutdown Terminal shutdown now calls observe_host instead of is_live so a native fault is still sampled after a local Stop has already closed admission. The host observation was split out of is_live, and teardown now drops the request and callback in an order that avoids holding locks across owner shutdown. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 4 ++-- crates/tinybus-module/src/terminal.rs | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index e2ed7b2c..a23136f7 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1316,11 +1316,11 @@ unsafe extern "C" fn terminal_shutdown(ctx: *mut c_void, deadline_ms: u64) -> i3 let context = unsafe { &*ctx.cast::() }; let Some(state) = context.state() else { return TB_CLOSED; }; let _scope = state.owner.as_ref().map(ExecutionOwner::enter); - state.is_live(); // sample actual host Fault before selecting ordinary Stop + state.observe_host(); // sample actual host Fault before selecting ordinary Stop state.signal(TerminalReason::Stop); let completion = state.wait(Duration::from_millis(deadline_ms)); // Genuine source/native Fault cannot earn accepted-vtable physical success. - state.is_live(); + state.observe_host(); if completion != TerminalCompletion::PhysicallyStopped || state.has_fault() { return TB_PANICKED; } let retired = context.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); drop(retired); diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 23651de6..b04bb962 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -161,6 +161,12 @@ impl TerminalState { pub fn is_terminal(&self) -> bool { self.reason.load(Ordering::Acquire) != 0 } pub fn is_live(self: &Arc) -> bool { if self.is_terminal() { return false; } + self.observe_host(); + !self.is_terminal() + } + // Cleanup must sample even when local Stop already closed admission: a real + // later native Fault is orthogonal to that local admission predicate. + pub fn observe_host(self: &Arc) { let flags = self.host.0.native_terminal_state.map(|query| unsafe { query(self.host.0.host_ctx) }); if flags != Some(TB_NATIVE_TERMINAL_LIVE) { let reason = if flags.is_some_and(|flags| flags & TB_NATIVE_TERMINAL_FAULT != 0) { @@ -169,9 +175,7 @@ impl TerminalState { TerminalReason::Stop } else { TerminalReason::HostClosed }; self.signal(reason); - return false; } - !self.is_terminal() } fn reason(&self) -> TerminalReason { match self.reason.load(Ordering::Acquire) { 1 => TerminalReason::Stop, 2 => TerminalReason::HostClosed, @@ -330,11 +334,13 @@ impl TerminalState { } } } - let future = future.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); - drop(future); let callback = reinitialize.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); if let Some(owner) = &self.owner { owner.stop(); } + let future = future.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + drop(future); drop(callback); + let request = self.lock().request.take(); + drop(request); let future = reinit_future.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); drop(future); true From bb33a48679f527164b070b9f1a93f454437fa6ad Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:02:01 +0300 Subject: [PATCH 041/130] chore: files changed crates/tinybus-module/src/lib.rs Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index a23136f7..95021a27 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1140,6 +1140,17 @@ macro_rules! module_export { unsafe { $start(host, out, $threads, true, $setup) } } }; + ( + terminal_setup_factory = $factory:path, + config = $config:ty, + $($rest:tt)* + ) => { + $crate::module_export! { + @configured export = {#[unsafe(no_mangle)]}, + start = $crate::start_terminal_reconfigurable_module_with_factory, + setup = $factory, config = $config, $($rest)* + } + }; ( setup_factory = $factory:path, config = $config:ty, @@ -1181,6 +1192,14 @@ macro_rules! module_export { /// Both configured `setup` and per-instance `setup_factory` forms are supported. #[macro_export] macro_rules! module_export_static { + (terminal_setup_factory = $factory:path, config = $config:ty, $($rest:tt)*) => { + $crate::module_export! { + @configured export = {}, + start = $crate::start_linked_terminal_reconfigurable_module_with_factory, + setup = $factory, config = $config, $($rest)* + } + $crate::module_export! { @linked } + }; (setup_factory = $factory:path, config = $config:ty, $($rest:tt)*) => { $crate::module_export! { @configured export = {}, From 7b8404d0faf845ec645bc55e7035f76d12385a5e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:05:15 +0300 Subject: [PATCH 042/130] refactor(module): resolve terminal callbacks through generation tokens Replace the heap-allocated runtime context with a bounded slot and generation token stored in the admission registry, so callbacks look up the exact live state instead of dereferencing a raw pointer. Retired slots now deny stale callbacks permanently and exhausted generations never wrap, which removes the tombstone lifetime hazard. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 23 +++-------- .../tinybus-module/src/lib_terminal_tests.rs | 3 +- crates/tinybus-module/src/terminal.rs | 30 +++++++++++++-- crates/tinybus-module/src/terminal_tests.rs | 38 +++++++++++++++++++ 4 files changed, 72 insertions(+), 22 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 95021a27..2546e1fa 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1308,18 +1308,11 @@ impl Transport for TerminalTransport { fn describe(&self) -> String { self.inner.describe() } } -// Process-lifetime C tombstone does not retain the actual State after proved -// successful shutdown. Surviving liveness views retain their own slot permits. -struct TerminalRuntimeContext { state: StdMutex>> } -impl TerminalRuntimeContext { - fn state(&self) -> Option> { - self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone() - } -} +// The existing opaque module_ctx field is a bounded slot/generation token, +// never a dereferenced pointer. Retirement denies old callbacks permanently. unsafe extern "C" fn terminal_deliver(ctx: *mut c_void, ptr: *const u8, len: usize) -> i32 { if ctx.is_null() || ptr.is_null() || len > MAX_FRAME_LEN { return TB_BAD_ARGUMENT; } - let context = unsafe { &*ctx.cast::() }; - let Some(state) = context.state() else { return TB_CLOSED; }; + let Some(state) = terminal::TerminalState::lookup(ctx) else { return TB_CLOSED; }; if !state.is_live() { return TB_CLOSED; } let bytes = unsafe { std::slice::from_raw_parts(ptr, len) }.to_vec(); let brokered = state.host.is_broker_routed(); @@ -1332,8 +1325,7 @@ unsafe extern "C" fn terminal_deliver(ctx: *mut c_void, ptr: *const u8, len: usi } unsafe extern "C" fn terminal_shutdown(ctx: *mut c_void, deadline_ms: u64) -> i32 { if ctx.is_null() { return TB_BAD_ARGUMENT; } - let context = unsafe { &*ctx.cast::() }; - let Some(state) = context.state() else { return TB_CLOSED; }; + let Some(state) = terminal::TerminalState::lookup(ctx) else { return TB_CLOSED; }; let _scope = state.owner.as_ref().map(ExecutionOwner::enter); state.observe_host(); // sample actual host Fault before selecting ordinary Stop state.signal(TerminalReason::Stop); @@ -1341,14 +1333,11 @@ unsafe extern "C" fn terminal_shutdown(ctx: *mut c_void, deadline_ms: u64) -> i3 // Genuine source/native Fault cannot earn accepted-vtable physical success. state.observe_host(); if completion != TerminalCompletion::PhysicallyStopped || state.has_fault() { return TB_PANICKED; } - let retired = context.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); - drop(retired); TB_OK } unsafe extern "C" fn terminal_reinitialize(ctx: *mut c_void, ptr: *const u8, len: usize) -> i32 { if ctx.is_null() || len > 1024 * 1024 || (ptr.is_null() && len != 0) { return TB_BAD_ARGUMENT; } - let context = unsafe { &*ctx.cast::() }; - let Some(state) = context.state() else { return TB_CLOSED; }; + let Some(state) = terminal::TerminalState::lookup(ctx) else { return TB_CLOSED; }; if !state.is_live() { return TB_CLOSED; } let _scope = state.owner.as_ref().map(ExecutionOwner::enter); let callback = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reinitialize.clone(); @@ -1506,7 +1495,7 @@ where C: serde::de::DeserializeOwned + Send + 'static, result }); state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).initial = Some(task); - let context = Box::into_raw(Box::new(TerminalRuntimeContext { state: StdMutex::new(Some(state.clone())) })).cast::(); + let context = state.context(); if capacity >= size_of::() as u32 { unsafe { out.write(TbModuleVtable { size: size_of::() as u32, _reserved: 0, module_ctx: context, deliver: terminal_deliver, shutdown: terminal_shutdown, reinitialize: Some(terminal_reinitialize) }); } diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 235b7374..52cac573 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -44,8 +44,7 @@ fn shutdown_rechecks_actual_native_fault_after_local_stop_and_physical_finish() entered_tx.send(()).unwrap(); worker_release.wait(); TerminalCompletion::PhysicallyStopped })); state.publish(); - let context = Box::new(TerminalRuntimeContext { state: StdMutex::new(Some(state.clone())) }); - let ptr = (&*context as *const TerminalRuntimeContext) as usize; + let ptr = state.context().addr(); let caller = std::thread::spawn(move || unsafe { terminal_shutdown(ptr as *mut c_void, 3000) }); entered_rx.recv().unwrap(); flags.store(tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, Ordering::Release); diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index b04bb962..a2685c37 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -43,7 +43,7 @@ impl NativeInstanceLiveness { } const INSTANCE_LIMIT: usize = 256; #[derive(Default)] -struct Slot { reserved: bool, anchor: Option> } +struct Slot { reserved: bool, exhausted: bool, generation: usize, anchor: Option> } struct Admission { slots: StdMutex<[Slot; INSTANCE_LIMIT]> } impl Admission { fn new() -> Arc { @@ -127,6 +127,7 @@ pub(super) struct TerminalState { pub resources: StdMutex, sdk_owners: AtomicUsize, lease: SlotLease, + token: usize, } /// Every surviving SDK transport/Connection/task retains this actual State. /// A pending runtime teardown may therefore never recycle its slot. @@ -140,8 +141,16 @@ impl TerminalState { } fn admit_in(admission: &Arc, host: HostCalls, owner: Option) -> Option> { let mut slots = admission.slots.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let index = slots.iter().position(|slot| !slot.reserved)?; + let index = slots.iter_mut().enumerate().find_map(|(index, slot)| { + if slot.reserved || slot.exhausted { return None; } + let generation = slot.generation.checked_add(1).filter(|generation| *generation <= usize::MAX >> 8); + match generation { + Some(generation) => { slot.generation = generation; Some(index) }, + None => { slot.exhausted = true; None }, + } + })?; slots[index].reserved = true; + let token = (slots[index].generation << 8) | index; let state = Arc::new(Self { host, owner, reason: AtomicU8::new(0), uncertain: AtomicBool::new(false), control: StdMutex::new(Control { request: None, finish: None, @@ -149,11 +158,25 @@ impl TerminalState { published: false, completion: None, job: None, payloads: PrimaryPayloads::default() }), changed: Condvar::new(), terminal_notify: tokio::sync::Notify::new(), resources: StdMutex::new(StartupResources::default()), sdk_owners: AtomicUsize::new(0), - lease: SlotLease { admission: Arc::downgrade(admission), index }, + lease: SlotLease { admission: Arc::downgrade(admission), index }, token, }); slots[index].anchor = Some(state.clone()); Some(state) } + /// Opaque address token only; no allocation or dereference occurs. + pub fn context(&self) -> *mut c_void { std::ptr::with_exposed_provenance_mut(self.token) } + pub fn lookup(context: *mut c_void) -> Option> { + Self::lookup_in(ADMISSION.get()?, context.addr()) + } + fn lookup_in(admission: &Arc, token: usize) -> Option> { + let generation = token >> 8; + if generation == 0 { return None; } + let slots = admission.slots.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let slot = &slots[token & (INSTANCE_LIMIT - 1)]; + if slot.generation != generation { return None; } + // Strong exact State is acquired before releasing registry exclusion. + slot.anchor.as_ref().filter(|state| state.token == token).cloned() + } pub fn view(self: &Arc) -> NativeInstanceLiveness { NativeInstanceLiveness { inner: self.clone() } } pub fn sdk_owner(self: &Arc) -> SdkOwner { self.sdk_owners.fetch_add(1, Ordering::AcqRel); SdkOwner { state: self.clone() } @@ -276,6 +299,7 @@ impl TerminalState { drop(anchor); } } + #[cfg(test)] pub fn empty_factory_failure(self: &Arc) { // No callable/controller was transferred. Retained views still hold the permit. self.commit(TerminalReason::SetupFailed); self.remove_anchor(); diff --git a/crates/tinybus-module/src/terminal_tests.rs b/crates/tinybus-module/src/terminal_tests.rs index 464586ac..9f713cfc 100644 --- a/crates/tinybus-module/src/terminal_tests.rs +++ b/crates/tinybus-module/src/terminal_tests.rs @@ -108,3 +108,41 @@ fn surviving_sdk_owned_transport_keeps_cleanup_uncertain_and_permit_anchored() { drop(sdk_owner); assert!(admission.slots.lock().unwrap()[state.lease.index].anchor.is_some()); } + +#[test] +fn stale_context_never_selects_replacement_and_unknown_contexts_are_closed() { + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let old = state.context().addr(); + state.empty_factory_failure(); drop(state); + let replacement = TerminalState::admit_in(&admission, host(), None).unwrap(); + assert_eq!(replacement.lease.index, old & 255); + assert_ne!(replacement.context().addr(), old); + assert!(TerminalState::lookup_in(&admission, old).is_none()); + for invalid in [0, 1, 255, usize::MAX] { assert!(TerminalState::lookup_in(&admission, invalid).is_none()); } + assert!(Arc::ptr_eq(&TerminalState::lookup_in(&admission, replacement.token).unwrap(), &replacement)); + replacement.empty_factory_failure(); +} +#[test] +fn exhausted_slot_generation_never_wraps_or_selects_old_context() { + let admission = Admission::new(); + { let mut slots = admission.slots.lock().unwrap(); + slots[0].generation = usize::MAX >> 8; + for slot in &mut slots[1..] { slot.exhausted = true; } + } + assert!(TerminalState::admit_in(&admission, host(), None).is_none()); + assert!(admission.slots.lock().unwrap()[0].exhausted); + assert!(TerminalState::lookup_in(&admission, (usize::MAX >> 8) << 8).is_none()); +} +#[test] +fn callback_lookup_holds_exact_state_across_retirement_until_actual_last_drop() { + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let admitted = TerminalState::lookup_in(&admission, state.token).unwrap(); + let index = state.lease.index; + state.empty_factory_failure(); drop(state); + assert!(admission.slots.lock().unwrap()[index].reserved); + assert_eq!(admitted.lease.index, index); + drop(admitted); + assert!(!admission.slots.lock().unwrap()[index].reserved); +} From 545911b6289d497835d67fbee4f96cf6fb5cd333 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:06:45 +0300 Subject: [PATCH 043/130] test(tinybus-module): cover linked terminal cancellation and setup panics Add controlled-fixture tests that drive the linked terminal path through a real worker thread, asserting that initial cancellation drops the setup future before finishing and joins the worker exactly once, and that setup construction and poll panics still complete the worker once with the instance left not live. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 52cac573..79246e86 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -52,3 +52,144 @@ fn shutdown_rechecks_actual_native_fault_after_local_stop_and_physical_finish() let code = caller.join().unwrap(); assert_eq!(code, TB_PANICKED, "actual native Fault cannot disappear behind local Stop"); } + +mod controlled { + use super::*; + use std::pin::Pin; + use std::task::{Context, Poll}; + #[derive(serde::Deserialize)] + pub struct Config { pub mode: u8 } + #[derive(Debug)] + pub enum Event { Poll(u8), Request, FutureDrop(String), Finish, FactoryDrop(String) } + pub struct Fixture { + pub events: std::sync::mpsc::Sender, + pub view: StdMutex>, + wake: (StdMutex, std::sync::Condvar), + worker: StdMutex>>, + pub requests: AtomicUsize, pub joins: AtomicUsize, + } + static NEXT: StdMutex>> = StdMutex::new(None); + pub static TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + pub fn fixture() -> (Arc, std::sync::mpsc::Receiver) { + let (events, receiver) = std::sync::mpsc::channel(); + let fixture = Arc::new(Fixture { events, view: StdMutex::new(None), + wake: (StdMutex::new(false), std::sync::Condvar::new()), + worker: StdMutex::new(None), requests: AtomicUsize::new(0), joins: AtomicUsize::new(0) }); + *NEXT.lock().unwrap() = Some(fixture.clone()); + (fixture, receiver) + } + struct Capture(Arc); + impl Drop for Capture { + fn drop(&mut self) { + let _ = self.0.events.send(Event::FactoryDrop(std::thread::current().name().unwrap_or("unnamed").to_owned())); + } + } + struct SetupFuture { fixture: Arc, mode: u8, first: bool } + impl Future for SetupFuture { + type Output = Result<()>; + fn poll(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + if self.first { self.first = false; self.fixture.events.send(Event::Poll(self.mode)).unwrap(); } + match self.mode { 0 => Poll::Ready(Ok(())), 2 => panic!("controlled setup poll"), + 3 => Poll::Ready(Err(Error::failed("controlled setup failure"))), _ => Poll::Pending } + } + } + impl Drop for SetupFuture { + fn drop(&mut self) { + let _ = self.fixture.events.send(Event::FutureDrop(std::thread::current().name().unwrap_or("unnamed").to_owned())); + } + } + fn factory(view: NativeInstanceLiveness) -> Result<(impl Fn(Connection, Config) -> Pin> + Send>> + Send + Sync, RuntimeHooks)> { + let fixture = NEXT.lock().unwrap().take().unwrap(); + *fixture.view.lock().unwrap() = Some(view.clone()); + let (request, finish) = (fixture.clone(), fixture.clone()); + let capture = Capture(fixture.clone()); + let setup = move |_: Connection, config: Config| { + let _ = &capture; + let mut worker = fixture.worker.lock().unwrap(); + if worker.is_none() { + let work = fixture.clone(); let worker_view = view.clone(); + *worker = Some(std::thread::Builder::new().name("controlled-module-worker".into()).spawn(move || { + let mut cancelled = work.wake.0.lock().unwrap(); + while !*cancelled { cancelled = work.wake.1.wait(cancelled).unwrap(); } + drop(cancelled); + !worker_view.is_live() + }).unwrap()); + } + drop(worker); + if config.mode == 4 { panic!("controlled setup construction"); } + Box::pin(SetupFuture { fixture: fixture.clone(), mode: config.mode, first: true }) as Pin> + Send>> + }; + let hooks = RuntimeHooks::new(move |_| { + request.requests.fetch_add(1, Ordering::SeqCst); + *request.wake.0.lock().unwrap() = true; request.wake.1.notify_all(); + request.events.send(Event::Request).unwrap(); + }, move || { + let worker = finish.worker.lock().unwrap().take(); + if let Some(worker) = worker { + let denied = worker.join().unwrap(); + finish.joins.fetch_add(1, Ordering::SeqCst); + if !denied { return TerminalCompletion::Uncertain; } + } + finish.events.send(Event::Finish).unwrap(); + TerminalCompletion::PhysicallyStopped + }); + Ok((setup, hooks)) + } + module_export_static! { + terminal_setup_factory = factory, config = Config, worker_threads = 1, + provides = ["ai.tinyhumans.tinybus.Terminal"], methods = [], signals = [], + requires = [], optional = [], lazy = false, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_initial_cancellation_drops_future_and_joins_worker_on_physical_thread() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = tinybus::module::ModuleHost::new(broker); + host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":1})).unwrap(); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let shutdown = host.shutdown(Duration::from_secs(5)); + shutdown.await; + let remainder: Vec<_> = events.try_iter().collect(); + let request_count = fixture.requests.load(Ordering::SeqCst); + let joins = fixture.joins.load(Ordering::SeqCst); + // All actual workers/futures settled before asserting behavior. + assert!(matches!(entered, controlled::Event::Poll(1))); + assert_eq!(request_count, 1); assert_eq!(joins, 1); + assert!(!view.is_live()); + let future_drop = remainder.iter().position(|event| matches!(event, controlled::Event::FutureDrop(name) if name == "tinybus-terminal")).unwrap(); + let finish = remainder.iter().position(|event| matches!(event, controlled::Event::Finish)).unwrap(); + assert!(future_drop < finish); + assert!(remainder.iter().any(|event| matches!(event, controlled::Event::FactoryDrop(name) if name == "tinybus-terminal"))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_setup_construction_and_poll_panics_still_finish_real_worker_once() { + let _serial = controlled::TEST_LOCK.lock().await; + for mode in [2, 4] { + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = tinybus::module::ModuleHost::new(broker); + host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":mode})).unwrap(); + // Consume through actual join notification before assertions or host teardown. + let mut seen = Vec::new(); + loop { + let event = events.recv_timeout(Duration::from_secs(5)).unwrap(); + let finished = matches!(event, controlled::Event::Finish); seen.push(event); + if finished { break; } + } + host.shutdown(Duration::from_secs(5)).await; + let view = fixture.view.lock().unwrap().take().unwrap(); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + assert_eq!(fixture.requests.load(Ordering::SeqCst), 1); + assert!(!view.is_live()); + if mode == 2 { + assert!(seen.iter().any(|event| matches!(event, controlled::Event::FutureDrop(name) if name == "tinybus-terminal"))); + } + } +} From aeaefd3c30dfdd3fd444999d7a28b652af0bdc68 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:13:15 +0300 Subject: [PATCH 044/130] fix(module): retain terminal anchor when a published vtable faults Terminal shutdown now keeps the exact state anchor alive when a genuine native fault is observed after the vtable has been published, so a C callback or retained view can still perform the final fault recheck instead of the permit being refunded. Startup failures that occur before any factory, hook, or config owner transfer are also routed through the empty-factory path, and the vtable publication flag is set before the Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 9 +- .../tinybus-module/src/lib_terminal_tests.rs | 4 +- crates/tinybus-module/src/terminal.rs | 20 +++- crates/tinybus/Cargo.toml | 5 + crates/tinybus/examples/module_terminal.rs | 108 ++++++++++++++++++ 5 files changed, 141 insertions(+), 5 deletions(-) create mode 100644 crates/tinybus/examples/module_terminal.rs diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 2546e1fa..4ce36f98 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1332,7 +1332,9 @@ unsafe extern "C" fn terminal_shutdown(ctx: *mut c_void, deadline_ms: u64) -> i3 let completion = state.wait(Duration::from_millis(deadline_ms)); // Genuine source/native Fault cannot earn accepted-vtable physical success. state.observe_host(); - if completion != TerminalCompletion::PhysicallyStopped || state.has_fault() { return TB_PANICKED; } + if completion != TerminalCompletion::PhysicallyStopped || state.has_fault() { + state.retain_uncertain_anchor(); return TB_PANICKED; + } TB_OK } unsafe extern "C" fn terminal_reinitialize(ctx: *mut c_void, ptr: *const u8, len: usize) -> i32 { @@ -1496,6 +1498,7 @@ where C: serde::de::DeserializeOwned + Send + 'static, }); state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).initial = Some(task); let context = state.context(); + state.mark_vtable_published(); if capacity >= size_of::() as u32 { unsafe { out.write(TbModuleVtable { size: size_of::() as u32, _reserved: 0, module_ctx: context, deliver: terminal_deliver, shutdown: terminal_shutdown, reinitialize: Some(terminal_reinitialize) }); } @@ -1509,6 +1512,10 @@ where C: serde::de::DeserializeOwned + Send + 'static, Ok(code) => code, Err(payload) => { state.source_panic(terminal::PayloadKind::Startup, payload); TB_PANICKED } }; + if code == TB_BAD_ARGUMENT && state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).config.is_none() { + // Parsing failed before any factory/hook/config owner transfer. + state.empty_factory_failure(); return code; + } if code != TB_OK { if let Some(owner) = &state.owner { owner.stop(); } if !state.is_terminal() { state.signal(TerminalReason::SetupFailed); } diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 79246e86..11f01013 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -43,7 +43,7 @@ fn shutdown_rechecks_actual_native_fault_after_local_stop_and_physical_finish() state.register(RuntimeHooks::new(|_| {}, move || { entered_tx.send(()).unwrap(); worker_release.wait(); TerminalCompletion::PhysicallyStopped })); - state.publish(); + state.mark_vtable_published(); state.publish(); let ptr = state.context().addr(); let caller = std::thread::spawn(move || unsafe { terminal_shutdown(ptr as *mut c_void, 3000) }); entered_rx.recv().unwrap(); @@ -51,6 +51,8 @@ fn shutdown_rechecks_actual_native_fault_after_local_stop_and_physical_finish() release.wait(); let code = caller.join().unwrap(); assert_eq!(code, TB_PANICKED, "actual native Fault cannot disappear behind local Stop"); + drop(state); + assert!(terminal::TerminalState::lookup(std::ptr::with_exposed_provenance_mut(ptr)).is_some(), "published Fault must retain the exact anchor instead of refunding its permit"); } mod controlled { diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index a2685c37..41ddf1d2 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -126,6 +126,7 @@ pub(super) struct TerminalState { pub terminal_notify: tokio::sync::Notify, pub resources: StdMutex, sdk_owners: AtomicUsize, + vtable_published: AtomicBool, lease: SlotLease, token: usize, } @@ -157,7 +158,7 @@ impl TerminalState { request_state: RequestState::Unregistered, finish_state: FinishState::Idle, published: false, completion: None, job: None, payloads: PrimaryPayloads::default() }), changed: Condvar::new(), terminal_notify: tokio::sync::Notify::new(), - resources: StdMutex::new(StartupResources::default()), sdk_owners: AtomicUsize::new(0), + resources: StdMutex::new(StartupResources::default()), sdk_owners: AtomicUsize::new(0), vtable_published: AtomicBool::new(false), lease: SlotLease { admission: Arc::downgrade(admission), index }, token, }); slots[index].anchor = Some(state.clone()); @@ -177,6 +178,17 @@ impl TerminalState { // Strong exact State is acquired before releasing registry exclusion. slot.anchor.as_ref().filter(|state| state.token == token).cloned() } + pub fn mark_vtable_published(&self) { self.vtable_published.store(true, Ordering::Release); } + pub fn retain_uncertain_anchor(self: &Arc) { + self.uncertain.store(true, Ordering::Release); + if let Some(admission) = self.lease.admission.upgrade() { + let mut slots = admission.slots.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let slot = &mut slots[self.lease.index]; + // This State's still-live lease forbids reuse while a C callback or + // retained view performs the final genuine-fault recheck. + if slot.generation == self.token >> 8 && slot.anchor.is_none() { slot.anchor = Some(self.clone()); } + } + } pub fn view(self: &Arc) -> NativeInstanceLiveness { NativeInstanceLiveness { inner: self.clone() } } pub fn sdk_owner(self: &Arc) -> SdkOwner { self.sdk_owners.fetch_add(1, Ordering::AcqRel); SdkOwner { state: self.clone() } @@ -299,7 +311,6 @@ impl TerminalState { drop(anchor); } } - #[cfg(test)] pub fn empty_factory_failure(self: &Arc) { // No callable/controller was transferred. Retained views still hold the permit. self.commit(TerminalReason::SetupFailed); self.remove_anchor(); @@ -334,7 +345,10 @@ impl TerminalState { Err(payload) => { self.source_panic(PayloadKind::Cleanup, payload); false } } } else { false }; - if !released { self.uncertain.store(true, Ordering::Release); } + self.observe_host(); + if !released || (self.has_fault() && self.vtable_published.load(Ordering::Acquire)) { + self.uncertain.store(true, Ordering::Release); + } let completion = if self.uncertain.load(Ordering::Acquire) { TerminalCompletion::Uncertain } else { completed }; { let mut control = self.lock(); control.finish_state = FinishState::Complete; control.completion = Some(completion); } self.changed.notify_all(); diff --git a/crates/tinybus/Cargo.toml b/crates/tinybus/Cargo.toml index b98b3681..d698f3d3 100644 --- a/crates/tinybus/Cargo.toml +++ b/crates/tinybus/Cargo.toml @@ -140,3 +140,8 @@ required-features = ["modules"] name = "module_controlled_init" crate-type = ["cdylib"] required-features = ["modules"] + +[[example]] +name = "module_terminal" +crate-type = ["cdylib"] +required-features = ["modules"] diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs new file mode 100644 index 00000000..25cf0379 --- /dev/null +++ b/crates/tinybus/examples/module_terminal.rs @@ -0,0 +1,108 @@ +//! Actual opted-in runtime fixture. Controls are trusted test-only C callbacks. +use std::{ffi::c_void, future::Future, pin::Pin, sync::{Arc, Condvar, Mutex}, task::{Context, Poll}}; +use tinybus::{Connection, Result}; +use tinybus::module::abi::*; +use tinybus_module::{NativeInstanceLiveness, RuntimeHooks, TerminalCompletion}; +#[derive(serde::Deserialize)] +struct Config { mode: u8, test_controls: Vec } +#[repr(C)] +#[derive(Clone, Copy)] +struct Controls { ctx: *mut c_void, event: unsafe extern "C" fn(*mut c_void, u32, u32) } +unsafe impl Send for Controls {} +unsafe impl Sync for Controls {} +impl Controls { + fn emit(self, event: u32, value: u32) { unsafe { (self.event)(self.ctx, event, value) } } +} +struct Controller { + controls: Mutex>, view: NativeInstanceLiveness, + wake: (Mutex, Condvar), worker: Mutex>>, +} +impl Controller { + fn emit(&self, event: u32, value: u32) { + let controls = *self.controls.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(controls) = controls { controls.emit(event, value); } + } +} +impl Drop for Controller { + fn drop(&mut self) { self.emit(6, u32::from(std::thread::current().name() == Some("tinybus-terminal"))); } +} +struct SetupFuture { controller: Arc, mode: u8, first: bool } +impl Future for SetupFuture { + type Output = Result<()>; + fn poll(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + if self.first { self.first = false; self.controller.emit(2, u32::from(self.mode)); } + match self.mode { + 0 => Poll::Ready(Ok(())), 2 => panic!("controlled native setup poll"), + 5 => Poll::Ready(Err(tinybus::Error::failed("controlled native setup error"))), + _ => Poll::Pending, + } + } +} +impl Drop for SetupFuture { + fn drop(&mut self) { self.controller.emit(4, u32::from(std::thread::current().name() == Some("tinybus-terminal"))); } +} +fn factory(view: NativeInstanceLiveness) -> Result<(impl Fn(Connection, Config) -> Pin> + Send>> + Send + Sync, RuntimeHooks)> { + let controller = Arc::new(Controller { controls: Mutex::new(None), view, + wake: (Mutex::new(false), Condvar::new()), worker: Mutex::new(None) }); + let (request, finish) = (controller.clone(), controller.clone()); + let setup = move |_: Connection, config: Config| { + if config.test_controls.len() != size_of::() { return Box::pin(async { Err(tinybus::Error::failed("bad test controls")) }) as Pin> + Send>>; } + // SAFETY: this explicit fixture accepts trusted in-process test pointers. + let controls = unsafe { config.test_controls.as_ptr().cast::().read_unaligned() }; + *controller.controls.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(controls); + let mut worker = controller.worker.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if worker.is_none() { + let work = controller.clone(); + *worker = Some(std::thread::Builder::new().name("native-terminal-controller".into()).spawn(move || { + let mut cancelled = work.wake.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + while !*cancelled { cancelled = work.wake.1.wait(cancelled).unwrap_or_else(std::sync::PoisonError::into_inner); } + drop(cancelled); + let denied = !work.view.is_live(); work.emit(7, u32::from(denied)); denied + }).expect("controlled worker spawn")); + } + drop(worker); + if config.mode == 4 { panic!("controlled native setup construction"); } + Box::pin(SetupFuture { controller: controller.clone(), mode: config.mode, first: true }) as Pin> + Send>> + }; + let hooks = RuntimeHooks::new(move |_| { + *request.wake.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = true; + request.wake.1.notify_all(); request.emit(3, 1); + }, move || { + let worker = finish.worker.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + let outcome = worker.map(|worker| worker.join()); + let joined = matches!(outcome, Some(Ok(true))); + finish.emit(5, u32::from(joined)); + if joined { TerminalCompletion::PhysicallyStopped } else { TerminalCompletion::Uncertain } + }); + Ok((setup, hooks)) +} +#[unsafe(no_mangle)] +pub static TINYBUS_MODULE_ABI_V1: TbAbiDescriptor = TbAbiDescriptor::current("module-terminal", env!("CARGO_PKG_VERSION")); +#[unsafe(no_mangle)] +pub extern "C" fn tinybus_module_manifest_v1() -> TbSlice { + tinybus_module::manifest_slice(tinybus_module::ManifestDeclaration { name: "module-terminal", version: env!("CARGO_PKG_VERSION"), + provides: &["ai.tinyhumans.TerminalFixture"], methods: &[], signals: &[], requires: &[], optional: &[], lazy: false, worker_threads: 1 }) +} +#[derive(Clone, Copy)] +struct Table(TbModuleVtable); +unsafe impl Send for Table {} +static LATEST: Mutex> = Mutex::new(None); +/// # Safety +/// Standard trusted module init contract plus live test controls in config. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tinybus_module_init_v1(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { + let code = unsafe { tinybus_module::start_terminal_reconfigurable_module_with_factory::(host, out, 1, true, factory) }; + if code == TB_OK { *LATEST.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Table(unsafe { *out })); } + code +} +/// Test-only stale-context probe, using the same actual SDK callback. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_context() -> usize { + LATEST.lock().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref().map_or(0, |table| table.0.module_ctx.addr()) +} +/// Test-only copied callback probe; the token is never dereferenced. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_old_context(token: usize) -> i32 { + let table = *LATEST.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + table.map_or(TB_CLOSED, |table| unsafe { (table.0.shutdown)(std::ptr::with_exposed_provenance_mut(token), 1000) }) +} From b467d2fd4fba83b3f46b6c1ecbe22383e217314e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:15:51 +0300 Subject: [PATCH 045/130] test(module): cover native terminal cancellation and reinit Add ignored and linked tests exercising a compiled native terminal module, verifying that initial cancellation, panics, worker joins, and stale context handling behave correctly across reinitialization. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 11f01013..10c2e541 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -195,3 +195,111 @@ async fn actual_linked_setup_construction_and_poll_panics_still_finish_real_work } } } + +#[repr(C)] +#[derive(Clone, Copy)] +struct NativeControls { ctx: *mut c_void, event: unsafe extern "C" fn(*mut c_void, u32, u32) } +unsafe extern "C" fn native_event(ctx: *mut c_void, event: u32, value: u32) { + let sender = unsafe { &*ctx.cast::>() }; + let _ = sender.send((event, value)); +} +// Only fixture symbols are looked up; TinyBus itself verifies/adopts the pinned +// private artifact through the actual ModuleHost loader first. It remains mapped. +#[cfg(unix)] +unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32) { + use std::os::unix::ffi::OsStrExt; + #[cfg_attr(target_os = "linux", link(name = "dl"))] + unsafe extern "C" { fn dlopen(path: *const std::ffi::c_char, flags: i32) -> *mut c_void; fn dlsym(handle: *mut c_void, name: *const std::ffi::c_char) -> *mut c_void; } + let path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + let handle = unsafe { dlopen(path.as_ptr(), 2) }; + assert!(!handle.is_null()); + let context = unsafe { dlsym(handle, c"terminal_test_context".as_ptr()) }; + let probe = unsafe { dlsym(handle, c"terminal_test_old_context".as_ptr()) }; + assert!(!context.is_null() && !probe.is_null()); + unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe)) } +} +#[cfg(windows)] +unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32) { + use std::os::windows::ffi::OsStrExt; + #[link(name = "kernel32")] + unsafe extern "system" { fn LoadLibraryW(path: *const u16) -> *mut c_void; fn GetProcAddress(handle: *mut c_void, name: *const u8) -> *mut c_void; } + let path: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let handle = unsafe { LoadLibraryW(path.as_ptr()) }; + assert!(!handle.is_null()); + let context = unsafe { GetProcAddress(handle, c"terminal_test_context".as_ptr().cast()) }; + let probe = unsafe { GetProcAddress(handle, c"terminal_test_old_context".as_ptr().cast()) }; + assert!(!context.is_null() && !probe.is_null()); + unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe)) } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a privately pinned TINYBUS_TEST_TERMINAL_MODULE compiled cdylib"] +async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stale_context() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let digest = tinybus::module::sha256_file(&path).unwrap(); + let name = path.file_name().unwrap().to_str().unwrap(); + let pins = std::fs::read_to_string(path.parent().unwrap().join("modules.toml")).unwrap(); + assert!(pins.contains(&format!("\"{name}\" = \"{digest}\""))); + let mut old_context = None; + for mode in [1u8, 0, 2, 4] { + let (events, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let events = Box::new(events); + let controls = NativeControls { ctx: (&*events as *const std::sync::mpsc::Sender<(u32, u32)>).cast_mut().cast(), event: native_event }; + let encoded = unsafe { std::slice::from_raw_parts((&controls as *const NativeControls).cast::(), size_of::()) }.to_vec(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = tinybus::module::ModuleHost::new(broker); + host.load_file_with_config(&path, serde_json::json!({"mode":mode,"test_controls":encoded})).unwrap(); + let (context, old_probe) = unsafe { terminal_fixture_symbols(&path) }; + let actual_context = context(); + let first = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); + let stale_result = old_context.map(old_probe); + host.shutdown(Duration::from_secs(5)).await; + let mut seen = vec![first]; + while !seen.iter().any(|(event, _)| *event == 6) { seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); } + // All actual controller workers and SDK captures are destroyed before assertions. + if let Some(code) = stale_result { assert_eq!(code, TB_CLOSED); } + assert_eq!(seen.iter().filter(|(event, _)| *event == 3).count(), 1); + assert_eq!(seen.iter().filter(|(event, value)| *event == 5 && *value == 1).count(), 1); + assert_eq!(seen.iter().filter(|(event, value)| *event == 7 && *value == 1).count(), 1); + assert!(seen.iter().any(|(event, value)| *event == 6 && *value == 1)); + if mode != 4 { + let drop = seen.iter().position(|(event, value)| *event == 4 && *value == 1).unwrap(); + let finish = seen.iter().position(|(event, _)| *event == 5).unwrap(); + assert!(drop < finish); + } + old_context = Some(actual_context); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_reinit_terminal_cancellation_settles_before_worker_join() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = tinybus::module::ModuleHost::new(broker); + host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + let initial = events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { tokio::task::yield_now().await; } + }).await.unwrap(); + let reinit_host = host.clone(); + let reinit = tokio::spawn(async move { reinit_host.reinitialize("tinybus-module", serde_json::json!({"mode":1})).await }); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + view.request_fault(); + let reinit_result = reinit.await.unwrap(); + host.shutdown(Duration::from_secs(5)).await; + let remainder: Vec<_> = events.try_iter().collect(); + assert!(matches!(initial, controlled::Event::Poll(0))); + assert!(matches!(entered, controlled::Event::Poll(1))); + assert!(reinit_result.is_err()); + assert_eq!(fixture.requests.load(Ordering::SeqCst), 1); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + assert!(!view.is_live()); + let drops: Vec<_> = remainder.iter().enumerate().filter_map(|(index, event)| matches!(event, controlled::Event::FutureDrop(name) if name == "tinybus-terminal").then_some(index)).collect(); + let finish = remainder.iter().position(|event| matches!(event, controlled::Event::Finish)).unwrap(); + assert_eq!(drops.len(), 2); + assert!(drops.into_iter().all(|index| index < finish)); +} From 20e918abe8cba7201c288c5aa63bd27e4923b3a0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:16:27 +0300 Subject: [PATCH 046/130] test(module): wrap module host in arc for terminal tests The terminal tests now construct the module host inside an Arc so the host can be shared across the spawned tasks that drive cancellation and join assertions. A closure was also made explicit in the stale probe call to satisfy the expected function signature. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 10c2e541..d94d79ef 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -150,7 +150,7 @@ async fn actual_linked_initial_cancellation_drops_future_and_joins_worker_on_phy let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); - let host = tinybus::module::ModuleHost::new(broker); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":1})).unwrap(); let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); let view = fixture.view.lock().unwrap().take().unwrap(); @@ -176,7 +176,7 @@ async fn actual_linked_setup_construction_and_poll_panics_still_finish_real_work let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); - let host = tinybus::module::ModuleHost::new(broker); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":mode})).unwrap(); // Consume through actual join notification before assertions or host teardown. let mut seen = Vec::new(); @@ -248,12 +248,12 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal let encoded = unsafe { std::slice::from_raw_parts((&controls as *const NativeControls).cast::(), size_of::()) }.to_vec(); let broker = tinybus::broker::Broker::new(); broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); - let host = tinybus::module::ModuleHost::new(broker); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); host.load_file_with_config(&path, serde_json::json!({"mode":mode,"test_controls":encoded})).unwrap(); let (context, old_probe) = unsafe { terminal_fixture_symbols(&path) }; let actual_context = context(); let first = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); - let stale_result = old_context.map(old_probe); + let stale_result = old_context.map(|token| old_probe(token)); host.shutdown(Duration::from_secs(5)).await; let mut seen = vec![first]; while !seen.iter().any(|(event, _)| *event == 6) { seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); } @@ -278,7 +278,7 @@ async fn actual_linked_reinit_terminal_cancellation_settles_before_worker_join() let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); - let host = tinybus::module::ModuleHost::new(broker); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); let initial = events.recv_timeout(Duration::from_secs(3)).unwrap(); tokio::time::timeout(Duration::from_secs(3), async { From 19833338ef34147df6b022400d69d9df1a9ebc3d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:19:15 +0300 Subject: [PATCH 047/130] test(tinybus-module): wait for factory drop event in reinit test The terminal reinit cancellation test collected events with try_iter, which could miss the factory drop event if it had not yet been delivered. It now receives events with a timeout until the factory drop is observed, making the assertion deterministic. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index d94d79ef..cd5c5794 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -291,7 +291,10 @@ async fn actual_linked_reinit_terminal_cancellation_settles_before_worker_join() view.request_fault(); let reinit_result = reinit.await.unwrap(); host.shutdown(Duration::from_secs(5)).await; - let remainder: Vec<_> = events.try_iter().collect(); + let mut remainder = Vec::new(); + while !remainder.iter().any(|event| matches!(event, controlled::Event::FactoryDrop(_))) { + remainder.push(events.recv_timeout(Duration::from_secs(5)).unwrap()); + } assert!(matches!(initial, controlled::Event::Poll(0))); assert!(matches!(entered, controlled::Event::Poll(1))); assert!(reinit_result.is_err()); From c96ff689fa35d62ffb57092e6932d45322d733f7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:24:12 +0300 Subject: [PATCH 048/130] test(module): verify terminal fixture teardown via drop counter The terminal fixture now exports a drop counter and the test waits for it to advance after shutdown, proving the controller is actually destroyed rather than merely signalled. The fixture callback owner is intentionally leaked on controlled failures so a live foreign context is never freed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 33 ++++++++++++------- crates/tinybus/examples/module_terminal.rs | 8 ++++- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index cd5c5794..4b782b32 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -206,30 +206,34 @@ unsafe extern "C" fn native_event(ctx: *mut c_void, event: u32, value: u32) { // Only fixture symbols are looked up; TinyBus itself verifies/adopts the pinned // private artifact through the actual ModuleHost loader first. It remains mapped. #[cfg(unix)] -unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32) { +unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32, extern "C" fn() -> usize) { use std::os::unix::ffi::OsStrExt; #[cfg_attr(target_os = "linux", link(name = "dl"))] - unsafe extern "C" { fn dlopen(path: *const std::ffi::c_char, flags: i32) -> *mut c_void; fn dlsym(handle: *mut c_void, name: *const std::ffi::c_char) -> *mut c_void; } + unsafe extern "C" { fn dlopen(path: *const std::ffi::c_char, flags: i32) -> *mut c_void; fn dlsym(handle: *mut c_void, name: *const std::ffi::c_char) -> *mut c_void; fn dlclose(handle: *mut c_void) -> i32; } let path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); let handle = unsafe { dlopen(path.as_ptr(), 2) }; assert!(!handle.is_null()); let context = unsafe { dlsym(handle, c"terminal_test_context".as_ptr()) }; let probe = unsafe { dlsym(handle, c"terminal_test_old_context".as_ptr()) }; - assert!(!context.is_null() && !probe.is_null()); - unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe)) } + let drops = unsafe { dlsym(handle, c"terminal_test_drops".as_ptr()) }; + assert!(!context.is_null() && !probe.is_null() && !drops.is_null()); + unsafe { dlclose(handle); } + unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops)) } } #[cfg(windows)] -unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32) { +unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32, extern "C" fn() -> usize) { use std::os::windows::ffi::OsStrExt; #[link(name = "kernel32")] - unsafe extern "system" { fn LoadLibraryW(path: *const u16) -> *mut c_void; fn GetProcAddress(handle: *mut c_void, name: *const u8) -> *mut c_void; } + unsafe extern "system" { fn LoadLibraryW(path: *const u16) -> *mut c_void; fn GetProcAddress(handle: *mut c_void, name: *const u8) -> *mut c_void; fn FreeLibrary(handle: *mut c_void) -> i32; } let path: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); let handle = unsafe { LoadLibraryW(path.as_ptr()) }; assert!(!handle.is_null()); let context = unsafe { GetProcAddress(handle, c"terminal_test_context".as_ptr().cast()) }; let probe = unsafe { GetProcAddress(handle, c"terminal_test_old_context".as_ptr().cast()) }; - assert!(!context.is_null() && !probe.is_null()); - unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe)) } + let drops = unsafe { GetProcAddress(handle, c"terminal_test_drops".as_ptr().cast()) }; + assert!(!context.is_null() && !probe.is_null() && !drops.is_null()); + unsafe { FreeLibrary(handle); } + unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops)) } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -243,20 +247,27 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal let mut old_context = None; for mode in [1u8, 0, 2, 4] { let (events, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); - let events = Box::new(events); - let controls = NativeControls { ctx: (&*events as *const std::sync::mpsc::Sender<(u32, u32)>).cast_mut().cast(), event: native_event }; + // If a controlled failure cannot prove teardown, conservatively retain + // this one fixture callback owner; never free live foreign context. + let events = Box::into_raw(Box::new(events)); + let controls = NativeControls { ctx: events.cast(), event: native_event }; let encoded = unsafe { std::slice::from_raw_parts((&controls as *const NativeControls).cast::(), size_of::()) }.to_vec(); let broker = tinybus::broker::Broker::new(); broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); host.load_file_with_config(&path, serde_json::json!({"mode":mode,"test_controls":encoded})).unwrap(); - let (context, old_probe) = unsafe { terminal_fixture_symbols(&path) }; + let (context, old_probe, drops) = unsafe { terminal_fixture_symbols(&path) }; + let dropped_before = drops(); let actual_context = context(); let first = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); let stale_result = old_context.map(|token| old_probe(token)); host.shutdown(Duration::from_secs(5)).await; let mut seen = vec![first]; while !seen.iter().any(|(event, _)| *event == 6) { seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); } + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == dropped_before { tokio::task::yield_now().await; } + }).await.unwrap(); + unsafe { drop(Box::from_raw(events)); } // All actual controller workers and SDK captures are destroyed before assertions. if let Some(code) = stale_result { assert_eq!(code, TB_CLOSED); } assert_eq!(seen.iter().filter(|(event, _)| *event == 3).count(), 1); diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 25cf0379..c8e613d7 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -1,6 +1,8 @@ //! Actual opted-in runtime fixture. Controls are trusted test-only C callbacks. use std::{ffi::c_void, future::Future, pin::Pin, sync::{Arc, Condvar, Mutex}, task::{Context, Poll}}; use tinybus::{Connection, Result}; +use std::sync::atomic::{AtomicUsize, Ordering}; +static DROPPED: AtomicUsize = AtomicUsize::new(0); use tinybus::module::abi::*; use tinybus_module::{NativeInstanceLiveness, RuntimeHooks, TerminalCompletion}; #[derive(serde::Deserialize)] @@ -24,7 +26,7 @@ impl Controller { } } impl Drop for Controller { - fn drop(&mut self) { self.emit(6, u32::from(std::thread::current().name() == Some("tinybus-terminal"))); } + fn drop(&mut self) { self.emit(6, u32::from(std::thread::current().name() == Some("tinybus-terminal"))); DROPPED.fetch_add(1, Ordering::Release); } } struct SetupFuture { controller: Arc, mode: u8, first: bool } impl Future for SetupFuture { @@ -106,3 +108,7 @@ pub extern "C" fn terminal_test_old_context(token: usize) -> i32 { let table = *LATEST.lock().unwrap_or_else(std::sync::PoisonError::into_inner); table.map_or(TB_CLOSED, |table| unsafe { (table.0.shutdown)(std::ptr::with_exposed_provenance_mut(token), 1000) }) } + +/// Test-only acquire fence after the final controller event callback returned. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_drops() -> usize { DROPPED.load(Ordering::Acquire) } From 207adfd449c4c42a3d08249636919b5b210e75aa Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:29:01 +0300 Subject: [PATCH 049/130] test(tinybus-module): track cumulative drop counts across terminal modes The terminal ownership test now accumulates the drop counter across loop iterations instead of re-reading it per mode, so the wait for teardown compares against a running total. The stale-context assertion gained a message naming the replacement mode, and the retired context is only retained after the first mode. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 4b782b32..11ee0fa8 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -245,6 +245,7 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal let pins = std::fs::read_to_string(path.parent().unwrap().join("modules.toml")).unwrap(); assert!(pins.contains(&format!("\"{name}\" = \"{digest}\""))); let mut old_context = None; + let mut dropped_total = 0; for mode in [1u8, 0, 2, 4] { let (events, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); // If a controlled failure cannot prove teardown, conservatively retain @@ -257,7 +258,7 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal let host = Arc::new(tinybus::module::ModuleHost::new(broker)); host.load_file_with_config(&path, serde_json::json!({"mode":mode,"test_controls":encoded})).unwrap(); let (context, old_probe, drops) = unsafe { terminal_fixture_symbols(&path) }; - let dropped_before = drops(); + let dropped_before = dropped_total; let actual_context = context(); let first = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); let stale_result = old_context.map(|token| old_probe(token)); @@ -267,9 +268,10 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal tokio::time::timeout(Duration::from_secs(5), async { while drops() == dropped_before { tokio::task::yield_now().await; } }).await.unwrap(); + dropped_total = drops(); unsafe { drop(Box::from_raw(events)); } // All actual controller workers and SDK captures are destroyed before assertions. - if let Some(code) = stale_result { assert_eq!(code, TB_CLOSED); } + if let Some(code) = stale_result { assert_eq!(code, TB_CLOSED, "retired successful context must not act on replacement mode {mode}"); } assert_eq!(seen.iter().filter(|(event, _)| *event == 3).count(), 1); assert_eq!(seen.iter().filter(|(event, value)| *event == 5 && *value == 1).count(), 1); assert_eq!(seen.iter().filter(|(event, value)| *event == 7 && *value == 1).count(), 1); @@ -279,7 +281,7 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal let finish = seen.iter().position(|(event, _)| *event == 5).unwrap(); assert!(drop < finish); } - old_context = Some(actual_context); + if mode == 1 { old_context = Some(actual_context); } } } From 1eefc670c736dec11131b65800a847fa45deb14e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:29:09 +0300 Subject: [PATCH 050/130] feat(tinybus): add early-finish and native query test hooks The terminal example now accepts test controls before the module factory runs, so a controller can be created already holding a finish callback and report a fault immediately. It also captures the host's native terminal state query during init and exposes it through a new test entry point, letting tests observe the real host flags instead of a fabricated value. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/examples/module_terminal.rs | 31 ++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index c8e613d7..75013a8d 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -3,6 +3,11 @@ use std::{ffi::c_void, future::Future, pin::Pin, sync::{Arc, Condvar, Mutex}, ta use tinybus::{Connection, Result}; use std::sync::atomic::{AtomicUsize, Ordering}; static DROPPED: AtomicUsize = AtomicUsize::new(0); +static EARLY: Mutex> = Mutex::new(None); +#[derive(Clone, Copy)] +struct NativeQuery { ctx: *mut c_void, query: unsafe extern "C" fn(*mut c_void) -> u32 } +unsafe impl Send for NativeQuery {} +static NATIVE_QUERY: Mutex> = Mutex::new(None); use tinybus::module::abi::*; use tinybus_module::{NativeInstanceLiveness, RuntimeHooks, TerminalCompletion}; #[derive(serde::Deserialize)] @@ -44,8 +49,11 @@ impl Drop for SetupFuture { fn drop(&mut self) { self.controller.emit(4, u32::from(std::thread::current().name() == Some("tinybus-terminal"))); } } fn factory(view: NativeInstanceLiveness) -> Result<(impl Fn(Connection, Config) -> Pin> + Send>> + Send + Sync, RuntimeHooks)> { - let controller = Arc::new(Controller { controls: Mutex::new(None), view, + let early = EARLY.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + let held_finish = early.is_some(); + let controller = Arc::new(Controller { controls: Mutex::new(early), view, wake: (Mutex::new(false), Condvar::new()), worker: Mutex::new(None) }); + if held_finish { controller.view.request_fault(); } let (request, finish) = (controller.clone(), controller.clone()); let setup = move |_: Connection, config: Config| { if config.test_controls.len() != size_of::() { return Box::pin(async { Err(tinybus::Error::failed("bad test controls")) }) as Pin> + Send>>; } @@ -70,9 +78,10 @@ fn factory(view: NativeInstanceLiveness) -> Result<(impl Fn(Connection, Config) *request.wake.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = true; request.wake.1.notify_all(); request.emit(3, 1); }, move || { + if held_finish { finish.emit(8, 0); } let worker = finish.worker.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); let outcome = worker.map(|worker| worker.join()); - let joined = matches!(outcome, Some(Ok(true))); + let joined = outcome.is_none() || matches!(outcome, Some(Ok(true))); finish.emit(5, u32::from(joined)); if joined { TerminalCompletion::PhysicallyStopped } else { TerminalCompletion::Uncertain } }); @@ -93,6 +102,11 @@ static LATEST: Mutex> = Mutex::new(None); /// Standard trusted module init contract plus live test controls in config. #[unsafe(no_mangle)] pub unsafe extern "C" fn tinybus_module_init_v1(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { + if let Some(host) = unsafe { TbHostVtable::read_compatible(host) } { + if let Some(query) = host.native_terminal_state { + *NATIVE_QUERY.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(NativeQuery { ctx: host.host_ctx, query }); + } + } let code = unsafe { tinybus_module::start_terminal_reconfigurable_module_with_factory::(host, out, 1, true, factory) }; if code == TB_OK { *LATEST.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Table(unsafe { *out })); } code @@ -112,3 +126,16 @@ pub extern "C" fn terminal_test_old_context(token: usize) -> i32 { /// Test-only acquire fence after the final controller event callback returned. #[unsafe(no_mangle)] pub extern "C" fn terminal_test_drops() -> usize { DROPPED.load(Ordering::Acquire) } + +/// # Safety +/// The test context and callback must remain live until actual controller Drop. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn terminal_test_early(ctx: *mut c_void, event: unsafe extern "C" fn(*mut c_void, u32, u32)) { + *EARLY.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Controls { ctx, event }); +} +/// Sample only the actual copied host query; no local or fabricated success bit. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_native_flags() -> u32 { + let query = *NATIVE_QUERY.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + query.map_or(0, |query| unsafe { (query.query)(query.ctx) }) +} From ba252050eec0f126705a85feeb089e20de194ed8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:34:35 +0300 Subject: [PATCH 051/130] test(tinybus-module): cover rejected native terminal admission Extend the terminal fixture symbol lookup to resolve the early-callback and native-flags entry points, then add a test asserting that a source fault before vtable publication rejects init while the SDK finish and resource drop still settle independently. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 85 +++++++++++++++++-- 1 file changed, 78 insertions(+), 7 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 11ee0fa8..878a82ba 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -206,7 +206,7 @@ unsafe extern "C" fn native_event(ctx: *mut c_void, event: u32, value: u32) { // Only fixture symbols are looked up; TinyBus itself verifies/adopts the pinned // private artifact through the actual ModuleHost loader first. It remains mapped. #[cfg(unix)] -unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32, extern "C" fn() -> usize) { +unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32, extern "C" fn() -> usize, unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), extern "C" fn() -> u32) { use std::os::unix::ffi::OsStrExt; #[cfg_attr(target_os = "linux", link(name = "dl"))] unsafe extern "C" { fn dlopen(path: *const std::ffi::c_char, flags: i32) -> *mut c_void; fn dlsym(handle: *mut c_void, name: *const std::ffi::c_char) -> *mut c_void; fn dlclose(handle: *mut c_void) -> i32; } @@ -216,12 +216,14 @@ unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() - let context = unsafe { dlsym(handle, c"terminal_test_context".as_ptr()) }; let probe = unsafe { dlsym(handle, c"terminal_test_old_context".as_ptr()) }; let drops = unsafe { dlsym(handle, c"terminal_test_drops".as_ptr()) }; - assert!(!context.is_null() && !probe.is_null() && !drops.is_null()); + let early = unsafe { dlsym(handle, c"terminal_test_early".as_ptr()) }; + let flags = unsafe { dlsym(handle, c"terminal_test_native_flags".as_ptr()) }; + assert!(!context.is_null() && !probe.is_null() && !drops.is_null() && !early.is_null() && !flags.is_null()); unsafe { dlclose(handle); } - unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops)) } + unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops), std::mem::transmute::<*mut c_void, unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32))>(early), std::mem::transmute::<*mut c_void, extern "C" fn() -> u32>(flags)) } } #[cfg(windows)] -unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32, extern "C" fn() -> usize) { +unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32, extern "C" fn() -> usize, unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), extern "C" fn() -> u32) { use std::os::windows::ffi::OsStrExt; #[link(name = "kernel32")] unsafe extern "system" { fn LoadLibraryW(path: *const u16) -> *mut c_void; fn GetProcAddress(handle: *mut c_void, name: *const u8) -> *mut c_void; fn FreeLibrary(handle: *mut c_void) -> i32; } @@ -231,9 +233,11 @@ unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() - let context = unsafe { GetProcAddress(handle, c"terminal_test_context".as_ptr().cast()) }; let probe = unsafe { GetProcAddress(handle, c"terminal_test_old_context".as_ptr().cast()) }; let drops = unsafe { GetProcAddress(handle, c"terminal_test_drops".as_ptr().cast()) }; - assert!(!context.is_null() && !probe.is_null() && !drops.is_null()); + let early = unsafe { GetProcAddress(handle, c"terminal_test_early".as_ptr().cast()) }; + let flags = unsafe { GetProcAddress(handle, c"terminal_test_native_flags".as_ptr().cast()) }; + assert!(!context.is_null() && !probe.is_null() && !drops.is_null() && !early.is_null() && !flags.is_null()); unsafe { FreeLibrary(handle); } - unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops)) } + unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops), std::mem::transmute::<*mut c_void, unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32))>(early), std::mem::transmute::<*mut c_void, extern "C" fn() -> u32>(flags)) } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -257,7 +261,7 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); host.load_file_with_config(&path, serde_json::json!({"mode":mode,"test_controls":encoded})).unwrap(); - let (context, old_probe, drops) = unsafe { terminal_fixture_symbols(&path) }; + let (context, old_probe, drops, _, _) = unsafe { terminal_fixture_symbols(&path) }; let dropped_before = dropped_total; let actual_context = context(); let first = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); @@ -283,6 +287,73 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal } if mode == 1 { old_context = Some(actual_context); } } + + // The image was admitted by the real loader above. Symbol lookup releases + // its temporary OS handle: only the canonical loader pins the code now. + let (_, _, drops, early, native_flags) = unsafe { terminal_fixture_symbols(&path) }; + let (sender, receiver) = std::sync::mpsc::channel(); + let controls = Box::into_raw(Box::new(HeldNativeControls { + sender, release: (std::sync::Mutex::new(false), std::sync::Condvar::new()), + })); + unsafe { early(controls.cast(), held_native_event); } + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = tinybus::module::ModuleHost::new(broker); + let result = host.load_file_with_config(&path, serde_json::json!({"mode":0,"test_controls":[]})); + let mut seen = Vec::new(); + let entered = loop { + match receiver.recv_timeout(Duration::from_secs(5)) { + Ok(event) => { seen.push(event); if event.0 == 8 { break true; } } + Err(_) => break false, + } + }; + let flags_before_drop = native_flags(); + drop(host); + let flags_after_host_drop = native_flags(); + // Release the actual physical callback before any assertion, even if + // rejected admission or host bookkeeping disagrees with the contract. + unsafe { + *(*controls).release.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = true; + (*controls).release.1.notify_all(); + } + while !seen.iter().any(|event| event.0 == 6) { + match receiver.recv_timeout(Duration::from_secs(5)) { + Ok(event) => seen.push(event), + Err(_) => break, + } + } + let settled = tokio::time::timeout(Duration::from_secs(5), async { + while drops() == dropped_total { tokio::task::yield_now().await; } + }).await.is_ok(); + let final_flags = native_flags(); + // On an unresolved controlled failure retain this one callback context. + // Otherwise the Acquire fence proves its final foreign use returned. + if settled { unsafe { drop(Box::from_raw(controls)); } } + assert!(result.is_err(), "source Fault before vtable publication must reject init"); + assert!(entered && settled, "actual SDK finish and resource Drop must settle independently"); + for flags in [flags_before_drop, flags_after_host_drop, final_flags] { + assert_ne!(flags & TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!(flags & TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0, + "no accepted vtable means no host physical-worker evidence"); + } + assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); + assert!(seen.iter().any(|event| *event == (5, 1))); + assert!(seen.iter().any(|event| *event == (6, 1))); +} + +struct HeldNativeControls { + sender: std::sync::mpsc::Sender<(u32, u32)>, + release: (std::sync::Mutex, std::sync::Condvar), +} +unsafe extern "C" fn held_native_event(ctx: *mut c_void, event: u32, value: u32) { + let controls = unsafe { &*ctx.cast::() }; + let _ = controls.sender.send((event, value)); + if event == 8 { + let mut release = controls.release.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + while !*release { + release = controls.release.1.wait(release).unwrap_or_else(std::sync::PoisonError::into_inner); + } + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From ec54077cbadd222d606f2ac550cddb6327caa2b2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:35:21 +0300 Subject: [PATCH 052/130] test(module): qualify terminal flag constants in assertions Reference the terminal fault and physically-stopped flags through the tinybus module ABI path so the test resolves them from the public interface rather than relying on local imports. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 878a82ba..bf371cef 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -332,8 +332,8 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal assert!(result.is_err(), "source Fault before vtable publication must reject init"); assert!(entered && settled, "actual SDK finish and resource Drop must settle independently"); for flags in [flags_before_drop, flags_after_host_drop, final_flags] { - assert_ne!(flags & TB_NATIVE_TERMINAL_FAULT, 0); - assert_eq!(flags & TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0, + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0, "no accepted vtable means no host physical-worker evidence"); } assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); From 5c4e51c206d4879ea7cf45083b6290b1b1e91805 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:36:21 +0300 Subject: [PATCH 053/130] feat(module): add reconfigurable terminal module support Adds a terminal module variant that supports live reconfiguration through a reinitialize callback, allowing the host to swap module configuration without restarting the instance. The new path validates native terminal authority before invoking the factory and retains state across reinitialization so in-flight work is not lost. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 462 ++++++++++++--- .../tinybus-module/src/lib_terminal_tests.rs | 539 +++++++++++++---- crates/tinybus-module/src/terminal.rs | 559 ++++++++++++++---- crates/tinybus-module/src/terminal_tests.rs | 185 ++++-- crates/tinybus/examples/module_terminal.rs | 270 +++++++-- 5 files changed, 1583 insertions(+), 432 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 4ce36f98..0dd3dfe4 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1271,10 +1271,14 @@ mod lifecycle_test; /// authority; unsupported hosts reject the factory before invoking it. #[doc(hidden)] pub unsafe fn start_terminal_reconfigurable_module_with_factory( - host: *const TbHostVtable, out: *mut TbModuleVtable, worker_threads: usize, - _detach_on_panic: bool, factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, + host: *const TbHostVtable, + out: *mut TbModuleVtable, + worker_threads: usize, + _detach_on_panic: bool, + factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, ) -> i32 -where C: serde::de::DeserializeOwned + Send + 'static, +where + C: serde::de::DeserializeOwned + Send + 'static, F: Fn(Connection, C) -> Fut + Send + Sync + 'static, Fut: Future> + Send + 'static, { @@ -1288,10 +1292,14 @@ where C: serde::de::DeserializeOwned + Send + 'static, /// Uphold [`start_terminal_reconfigurable_module_with_factory`]'s contracts. #[doc(hidden)] pub unsafe fn start_linked_terminal_reconfigurable_module_with_factory( - host: *const TbHostVtable, out: *mut TbModuleVtable, worker_threads: usize, - _detach_on_panic: bool, factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, + host: *const TbHostVtable, + out: *mut TbModuleVtable, + worker_threads: usize, + _detach_on_panic: bool, + factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, ) -> i32 -where C: serde::de::DeserializeOwned + Send + 'static, +where + C: serde::de::DeserializeOwned + Send + 'static, F: Fn(Connection, C) -> Fut + Send + Sync + 'static, Fut: Future> + Send + 'static, { @@ -1299,24 +1307,46 @@ where C: serde::de::DeserializeOwned + Send + 'static, } // Legacy transport remains unchanged; only opted-in Connections retain State. -struct TerminalTransport { inner: ModuleTransport, _owner: terminal::SdkOwner } +struct TerminalTransport { + inner: ModuleTransport, + _owner: terminal::SdkOwner, +} #[async_trait] impl Transport for TerminalTransport { - async fn send(&self, message: Message) -> Result<()> { self.inner.send(message).await } - async fn recv(&self) -> Result> { self.inner.recv().await } - async fn close(&self) -> Result<()> { self.inner.close().await } - fn describe(&self) -> String { self.inner.describe() } + async fn send(&self, message: Message) -> Result<()> { + self.inner.send(message).await + } + async fn recv(&self) -> Result> { + self.inner.recv().await + } + async fn close(&self) -> Result<()> { + self.inner.close().await + } + fn describe(&self) -> String { + self.inner.describe() + } } // The existing opaque module_ctx field is a bounded slot/generation token, // never a dereferenced pointer. Retirement denies old callbacks permanently. unsafe extern "C" fn terminal_deliver(ctx: *mut c_void, ptr: *const u8, len: usize) -> i32 { - if ctx.is_null() || ptr.is_null() || len > MAX_FRAME_LEN { return TB_BAD_ARGUMENT; } - let Some(state) = terminal::TerminalState::lookup(ctx) else { return TB_CLOSED; }; - if !state.is_live() { return TB_CLOSED; } + if ctx.is_null() || ptr.is_null() || len > MAX_FRAME_LEN { + return TB_BAD_ARGUMENT; + } + let Some(state) = terminal::TerminalState::lookup(ctx) else { + return TB_CLOSED; + }; + if !state.is_live() { + return TB_CLOSED; + } let bytes = unsafe { std::slice::from_raw_parts(ptr, len) }.to_vec(); let brokered = state.host.is_broker_routed(); - let sender = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).inbound.clone(); + let sender = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .inbound + .clone(); match sender.map(|sender| sender.try_send((bytes, brokered))) { Some(Ok(())) => TB_OK, Some(Err(mpsc::error::TrySendError::Full(_))) => TB_BACKPRESSURE, @@ -1324,8 +1354,12 @@ unsafe extern "C" fn terminal_deliver(ctx: *mut c_void, ptr: *const u8, len: usi } } unsafe extern "C" fn terminal_shutdown(ctx: *mut c_void, deadline_ms: u64) -> i32 { - if ctx.is_null() { return TB_BAD_ARGUMENT; } - let Some(state) = terminal::TerminalState::lookup(ctx) else { return TB_CLOSED; }; + if ctx.is_null() { + return TB_BAD_ARGUMENT; + } + let Some(state) = terminal::TerminalState::lookup(ctx) else { + return TB_CLOSED; + }; let _scope = state.owner.as_ref().map(ExecutionOwner::enter); state.observe_host(); // sample actual host Fault before selecting ordinary Stop state.signal(TerminalReason::Stop); @@ -1333,22 +1367,43 @@ unsafe extern "C" fn terminal_shutdown(ctx: *mut c_void, deadline_ms: u64) -> i3 // Genuine source/native Fault cannot earn accepted-vtable physical success. state.observe_host(); if completion != TerminalCompletion::PhysicallyStopped || state.has_fault() { - state.retain_uncertain_anchor(); return TB_PANICKED; + state.retain_uncertain_anchor(); + return TB_PANICKED; } TB_OK } unsafe extern "C" fn terminal_reinitialize(ctx: *mut c_void, ptr: *const u8, len: usize) -> i32 { - if ctx.is_null() || len > 1024 * 1024 || (ptr.is_null() && len != 0) { return TB_BAD_ARGUMENT; } - let Some(state) = terminal::TerminalState::lookup(ctx) else { return TB_CLOSED; }; - if !state.is_live() { return TB_CLOSED; } + if ctx.is_null() || len > 1024 * 1024 || (ptr.is_null() && len != 0) { + return TB_BAD_ARGUMENT; + } + let Some(state) = terminal::TerminalState::lookup(ctx) else { + return TB_CLOSED; + }; + if !state.is_live() { + return TB_CLOSED; + } let _scope = state.owner.as_ref().map(ExecutionOwner::enter); - let callback = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reinitialize.clone(); - let callback = callback.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if !state.is_live() { return TB_CLOSED; } - let Some(callback) = callback.as_ref() else { return TB_CLOSED; }; + let callback = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reinitialize + .clone(); + let callback = callback + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !state.is_live() { + return TB_CLOSED; + } + let Some(callback) = callback.as_ref() else { + return TB_CLOSED; + }; match catch_unwind(AssertUnwindSafe(|| callback(ptr, len))) { Ok(code) => code, - Err(payload) => { state.source_panic(terminal::PayloadKind::Reinit, payload); TB_PANICKED } + Err(payload) => { + state.source_panic(terminal::PayloadKind::Reinit, payload); + TB_PANICKED + } } } @@ -1356,19 +1411,35 @@ unsafe extern "C" fn terminal_reinitialize(ctx: *mut c_void, ptr: *const u8, len // Each poll takes/polls/reattaches without holding a resource mutex over user // code. Physical cleanup joins the sole task, then destroys this retained future. struct TerminalSetupPoll { - slot: terminal::SetupFutureSlot, state: Arc, kind: terminal::PayloadKind, + slot: terminal::SetupFutureSlot, + state: Arc, + kind: terminal::PayloadKind, } impl Future for TerminalSetupPoll { type Output = Result<()>; - fn poll(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll { - let mut future = self.slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + fn poll( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + let mut future = self + .slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); let outcome = catch_unwind(AssertUnwindSafe(|| { - future.as_mut().map_or(std::task::Poll::Ready(Err(Error::ConnectionClosed)), |future| future.as_mut().poll(cx)) + future.as_mut().map_or( + std::task::Poll::Ready(Err(Error::ConnectionClosed)), + |future| future.as_mut().poll(cx), + ) })); - *self.slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = future; + *self + .slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = future; match outcome { Ok(std::task::Poll::Ready(Err(error))) => { - self.state.signal(TerminalReason::SetupFailed); std::task::Poll::Ready(Err(error)) + self.state.signal(TerminalReason::SetupFailed); + std::task::Poll::Ready(Err(error)) } Ok(poll) => poll, Err(payload) => { @@ -1383,86 +1454,212 @@ async fn terminal_notification(state: &terminal::TerminalState) { let notification = state.terminal_notify.notified(); tokio::pin!(notification); notification.as_mut().enable(); - if state.is_terminal() { return; } + if state.is_terminal() { + return; + } notification.await; } } unsafe fn start_terminal_factory_mode( - host: *const TbHostVtable, out: *mut TbModuleVtable, worker_threads: usize, - factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, linked: bool, + host: *const TbHostVtable, + out: *mut TbModuleVtable, + worker_threads: usize, + factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, + linked: bool, ) -> i32 -where C: serde::de::DeserializeOwned + Send + 'static, +where + C: serde::de::DeserializeOwned + Send + 'static, F: Fn(Connection, C) -> Fut + Send + Sync + 'static, Fut: Future> + Send + 'static, { - let Some((capacity, compatible)) = (unsafe { validate_start_arguments(host, out, worker_threads) }) else { return TB_BAD_ARGUMENT; }; + let Some((capacity, compatible)) = + (unsafe { validate_start_arguments(host, out, worker_threads) }) + else { + return TB_BAD_ARGUMENT; + }; // Reject unavailable, unknown and terminal authority before any factory. - let Some(query) = compatible.native_terminal_state else { return TB_CLOSED; }; - if unsafe { query(compatible.host_ctx) } != tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE { return TB_CLOSED; } + let Some(query) = compatible.native_terminal_state else { + return TB_CLOSED; + }; + if unsafe { query(compatible.host_ctx) } != tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE { + return TB_CLOSED; + } let owner = (!linked).then(|| ExecutionOwner::new(HostCalls(compatible))); - let Some(state) = terminal::TerminalState::admit(HostCalls(compatible), owner) else { return TB_CLOSED; }; - if state.owner.is_some() { ExecutionOwner::install_panic_hook(); } + let Some(state) = terminal::TerminalState::admit(HostCalls(compatible), owner) else { + return TB_CLOSED; + }; + if state.owner.is_some() { + ExecutionOwner::install_panic_hook(); + } let _scope = state.owner.as_ref().map(ExecutionOwner::enter); let result = catch_unwind(AssertUnwindSafe(|| { - let config = match unsafe { parse_config::(host) } { Ok(config) => config, Err(code) => return code }; - state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).config = Some(Box::new(config)); + let config = match unsafe { parse_config::(host) } { + Ok(config) => config, + Err(code) => return code, + }; + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config = Some(Box::new(config)); let (setup, hooks) = match catch_unwind(AssertUnwindSafe(|| factory(state.view()))) { Ok(Ok(returned)) => returned, - Ok(Err(_error)) => { state.signal(TerminalReason::SetupFailed); return TB_CLOSED; } - Err(payload) => { state.source_panic(terminal::PayloadKind::Factory, payload); return TB_PANICKED; } + Ok(Err(_error)) => { + state.signal(TerminalReason::SetupFailed); + return TB_CLOSED; + } + Err(payload) => { + state.source_panic(terminal::PayloadKind::Factory, payload); + return TB_PANICKED; + } }; let setup = Arc::new(setup); - state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).retained_factory = Some(Box::new(setup.clone())); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retained_factory = Some(Box::new(setup.clone())); state.register(hooks); - if !state.is_live() { return TB_CLOSED; } - let config = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).config.take(); - let config = *config.expect("attached config").downcast::().unwrap_or_else(|_| unreachable!("private config type")); + if !state.is_live() { + return TB_CLOSED; + } + let config = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config + .take(); + let config = *config + .expect("attached config") + .downcast::() + .unwrap_or_else(|_| unreachable!("private config type")); let initial_setup = setup.clone(); - state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).setup = Some(Box::new(move |connection| Box::pin(initial_setup(connection, config)))); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .setup = Some(Box::new(move |connection| { + Box::pin(initial_setup(connection, config)) + })); let weak = Arc::downgrade(&state); let reinitializer: ReinitializeFactory = Box::new(move |handle, connection_slot| { Box::new(move |ptr, len| { - let Some(state) = weak.upgrade() else { return TB_CLOSED; }; - if !state.is_live() { return TB_CLOSED; } - let bytes = if len == 0 { b"{}".as_slice() } else { unsafe { std::slice::from_raw_parts(ptr, len) } }; - let config = match serde_json::from_slice::(bytes) { Ok(config) => config, Err(_) => return TB_BAD_ARGUMENT }; - let connection = connection_slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone(); - let Some(connection) = connection else { return TB_CLOSED; }; - let slot = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reinit_future.clone(); - let previous = slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + let Some(state) = weak.upgrade() else { + return TB_CLOSED; + }; + if !state.is_live() { + return TB_CLOSED; + } + let bytes = if len == 0 { + b"{}".as_slice() + } else { + unsafe { std::slice::from_raw_parts(ptr, len) } + }; + let config = match serde_json::from_slice::(bytes) { + Ok(config) => config, + Err(_) => return TB_BAD_ARGUMENT, + }; + let connection = connection_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let Some(connection) = connection else { + return TB_CLOSED; + }; + let slot = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reinit_future + .clone(); + let previous = slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); drop(previous); - if !state.is_live() { return TB_CLOSED; } + if !state.is_live() { + return TB_CLOSED; + } let future = Box::pin(setup(connection, config)); - *slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); + *slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); let result = handle.block_on(async { tokio::select! { biased; result = TerminalSetupPoll { slot, state: state.clone(), kind: terminal::PayloadKind::Reinit } => result, () = terminal_notification(&state) => Err(Error::ConnectionClosed), } }); - match result { Ok(()) if state.is_live() => TB_OK, Ok(()) | Err(_) => TB_CLOSED } + match result { + Ok(()) if state.is_live() => TB_OK, + Ok(()) | Err(_) => TB_CLOSED, + } }) }); - state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reinitializer_factory = Some(reinitializer); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reinitializer_factory = Some(reinitializer); let mut builder = tokio::runtime::Builder::new_multi_thread(); - if let Some(owner) = &state.owner { owner.configure(&mut builder); } + if let Some(owner) = &state.owner { + owner.configure(&mut builder); + } let runtime = match builder.worker_threads(worker_threads).enable_all().build() { - Ok(runtime) => runtime, Err(_) => { state.signal(TerminalReason::SetupFailed); return TB_CLOSED; } + Ok(runtime) => runtime, + Err(_) => { + state.signal(TerminalReason::SetupFailed); + return TB_CLOSED; + } }; // Immediate mechanical transfer: no later startup failure drops Runtime on ABI. - state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).runtime = Some(runtime); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .runtime = Some(runtime); let (inbound_tx, inbound_rx) = mpsc::channel(MODULE_QUEUE_CAPACITY); - state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).inbound = Some(inbound_tx); - let transport = Arc::new(TerminalTransport { inner: ModuleTransport { host: state.host, inbound: Mutex::new(inbound_rx), - provenance: Arc::new(AtomicBool::new(false)), detach_on_panic: false }, _owner: state.sdk_owner() }); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .inbound = Some(inbound_tx); + let transport = Arc::new(TerminalTransport { + inner: ModuleTransport { + host: state.host, + inbound: Mutex::new(inbound_rx), + provenance: Arc::new(AtomicBool::new(false)), + detach_on_panic: false, + }, + _owner: state.sdk_owner(), + }); let (handle, connection_slot, factory) = { - let mut resources = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - (resources.runtime.as_ref().expect("attached runtime").handle().clone(), resources.connection.clone(), resources.reinitializer_factory.take()) + let mut resources = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ( + resources + .runtime + .as_ref() + .expect("attached runtime") + .handle() + .clone(), + resources.connection.clone(), + resources.reinitializer_factory.take(), + ) }; let callback = factory.map(|factory| factory(handle.clone(), connection_slot.clone())); - let callback_slot = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).reinitialize.clone(); - *callback_slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = callback; + let callback_slot = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reinitialize + .clone(); + *callback_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = callback; let task_state = state.clone(); let task = handle.spawn(async move { let provenance = transport.inner.provenance.clone(); @@ -1470,55 +1667,124 @@ where C: serde::de::DeserializeOwned + Send + 'static, let connection = if brokered { // SAFETY: same private per-frame provenance contract as module_connection. unsafe { Connection::__attach_module_with_provenance(transport, provenance) } - } else { Connection::attach(transport) }; - if let Err(error) = connection.handshake().await { task_state.signal(TerminalReason::SetupFailed); return Err(error); } + } else { + Connection::attach(transport) + }; + if let Err(error) = connection.handshake().await { + task_state.signal(TerminalReason::SetupFailed); + return Err(error); + } connection.__set_panic_handler(Arc::new(|| Error::MethodFailed { - name: MODULE_PANIC_ERROR.to_owned(), message: "a module method panicked".to_owned(), + name: MODULE_PANIC_ERROR.to_owned(), + message: "a module method panicked".to_owned(), })); let observer_state = Arc::downgrade(&task_state); connection.__set_catch_observer(Arc::new(move || { if let Some(state) = observer_state.upgrade() { - state.mark_uncertain(); state.signal(TerminalReason::Fault); + state.mark_uncertain(); + state.signal(TerminalReason::Fault); } })); // Connection is persisted independently of initial task lifetime. - *connection_slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(connection.clone()); - if !task_state.is_live() { return Ok(()); } - let setup = task_state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).setup.take(); - let future = match catch_unwind(AssertUnwindSafe(|| setup.map(|setup| setup(connection)))) { - Ok(Some(future)) => future, - Ok(None) => return Err(Error::ConnectionClosed), - Err(payload) => { task_state.source_panic(terminal::PayloadKind::Initial, payload); return Err(Error::failed("module setup construction panicked")); } - }; - let slot = task_state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).initial_future.clone(); - *slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); - let result = TerminalSetupPoll { slot, state: task_state.clone(), kind: terminal::PayloadKind::Initial }.await; - if result.is_ok() && task_state.is_live() { task_state.host.ready(); } + *connection_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(connection.clone()); + if !task_state.is_live() { + return Ok(()); + } + let setup = task_state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .setup + .take(); + let future = + match catch_unwind(AssertUnwindSafe(|| setup.map(|setup| setup(connection)))) { + Ok(Some(future)) => future, + Ok(None) => return Err(Error::ConnectionClosed), + Err(payload) => { + task_state.source_panic(terminal::PayloadKind::Initial, payload); + return Err(Error::failed("module setup construction panicked")); + } + }; + let slot = task_state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .initial_future + .clone(); + *slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); + let result = TerminalSetupPoll { + slot, + state: task_state.clone(), + kind: terminal::PayloadKind::Initial, + } + .await; + if result.is_ok() && task_state.is_live() { + task_state.host.ready(); + } result }); - state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).initial = Some(task); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .initial = Some(task); let context = state.context(); state.mark_vtable_published(); if capacity >= size_of::() as u32 { - unsafe { out.write(TbModuleVtable { size: size_of::() as u32, _reserved: 0, - module_ctx: context, deliver: terminal_deliver, shutdown: terminal_shutdown, reinitialize: Some(terminal_reinitialize) }); } + unsafe { + out.write(TbModuleVtable { + size: size_of::() as u32, + _reserved: 0, + module_ctx: context, + deliver: terminal_deliver, + shutdown: terminal_shutdown, + reinitialize: Some(terminal_reinitialize), + }); + } } else { - unsafe { out.cast::().write(TbModuleVtableV1Prefix { - size: TB_MODULE_VTABLE_BASE_SIZE, _reserved: 0, module_ctx: context, deliver: terminal_deliver, shutdown: terminal_shutdown }); } + unsafe { + out.cast::() + .write(TbModuleVtableV1Prefix { + size: TB_MODULE_VTABLE_BASE_SIZE, + _reserved: 0, + module_ctx: context, + deliver: terminal_deliver, + shutdown: terminal_shutdown, + }); + } } TB_OK })); let code = match result { Ok(code) => code, - Err(payload) => { state.source_panic(terminal::PayloadKind::Startup, payload); TB_PANICKED } + Err(payload) => { + state.source_panic(terminal::PayloadKind::Startup, payload); + TB_PANICKED + } }; - if code == TB_BAD_ARGUMENT && state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).config.is_none() { + if code == TB_BAD_ARGUMENT + && state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config + .is_none() + { // Parsing failed before any factory/hook/config owner transfer. - state.empty_factory_failure(); return code; + state.empty_factory_failure(); + return code; } if code != TB_OK { - if let Some(owner) = &state.owner { owner.stop(); } - if !state.is_terminal() { state.signal(TerminalReason::SetupFailed); } + if let Some(owner) = &state.owner { + owner.stop(); + } + if !state.is_terminal() { + state.signal(TerminalReason::SetupFailed); + } } state.publish(); code diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index bf371cef..fad3446f 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -1,30 +1,64 @@ use super::*; use std::sync::atomic::AtomicUsize; use tinybus::module::abi::TbSlice; -unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { TB_OK } +unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { + TB_OK +} unsafe extern "C" fn notify(_: *mut c_void) {} unsafe extern "C" fn fault(_: *mut c_void, _: *const u8, _: usize) {} unsafe extern "C" fn log(_: *mut c_void, _: u32, _: *const u8, _: usize) {} fn host() -> TbHostVtable { - TbHostVtable { size: size_of::() as u32, _reserved: 0, - host_ctx: std::ptr::null_mut(), send, wake: notify, log, ready: notify, fault, - config: TbSlice { ptr: std::ptr::null(), len: 0 }, broker_routing: None, native_terminal_state: None } + TbHostVtable { + size: size_of::() as u32, + _reserved: 0, + host_ctx: std::ptr::null_mut(), + send, + wake: notify, + log, + ready: notify, + fault, + config: TbSlice { + ptr: std::ptr::null(), + len: 0, + }, + broker_routing: None, + native_terminal_state: None, + } } fn output() -> TbModuleVtable { - TbModuleVtable { size: size_of::() as u32, _reserved: 0, - module_ctx: std::ptr::null_mut(), deliver, shutdown, reinitialize: None } + TbModuleVtable { + size: size_of::() as u32, + _reserved: 0, + module_ctx: std::ptr::null_mut(), + deliver, + shutdown, + reinitialize: None, + } } #[test] fn native_query_is_required_before_terminal_factory_evaluation() { let calls = AtomicUsize::new(0); - let code = unsafe { start_terminal_reconfigurable_module_with_factory::( - &host(), &mut output(), 1, true, |_| { - calls.fetch_add(1, Ordering::SeqCst); - Ok((|_: Connection, _: serde_json::Value| async { Ok(()) }, - RuntimeHooks::new(|_| {}, || TerminalCompletion::PhysicallyStopped))) - }) }; + let code = unsafe { + start_terminal_reconfigurable_module_with_factory::( + &host(), + &mut output(), + 1, + true, + |_| { + calls.fetch_add(1, Ordering::SeqCst); + Ok(( + |_: Connection, _: serde_json::Value| async { Ok(()) }, + RuntimeHooks::new(|_| {}, || TerminalCompletion::PhysicallyStopped), + )) + }, + ) + }; assert_eq!(code, TB_CLOSED); - assert_eq!(calls.load(Ordering::SeqCst), 0, "missing native query must leave the factory untouched"); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "missing native query must leave the factory untouched" + ); } unsafe extern "C" fn native_query(ctx: *mut c_void) -> u32 { @@ -32,27 +66,46 @@ unsafe extern "C" fn native_query(ctx: *mut c_void) -> u32 { } #[test] fn shutdown_rechecks_actual_native_fault_after_local_stop_and_physical_finish() { - let flags = Arc::new(std::sync::atomic::AtomicU32::new(tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE)); + static FLAGS: std::sync::atomic::AtomicU32 = + std::sync::atomic::AtomicU32::new(tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE); + let flags = &FLAGS; let mut host = host(); - host.host_ctx = Arc::as_ptr(&flags).cast_mut().cast(); + host.host_ctx = (flags as *const std::sync::atomic::AtomicU32) + .cast_mut() + .cast(); host.native_terminal_state = Some(native_query); let state = terminal::TerminalState::admit(HostCalls(host), None).unwrap(); let (entered_tx, entered_rx) = std::sync::mpsc::channel(); let release = Arc::new(std::sync::Barrier::new(2)); let worker_release = release.clone(); - state.register(RuntimeHooks::new(|_| {}, move || { - entered_tx.send(()).unwrap(); worker_release.wait(); TerminalCompletion::PhysicallyStopped - })); - state.mark_vtable_published(); state.publish(); + state.register(RuntimeHooks::new( + |_| {}, + move || { + entered_tx.send(()).unwrap(); + worker_release.wait(); + TerminalCompletion::PhysicallyStopped + }, + )); + state.mark_vtable_published(); + state.publish(); let ptr = state.context().addr(); let caller = std::thread::spawn(move || unsafe { terminal_shutdown(ptr as *mut c_void, 3000) }); entered_rx.recv().unwrap(); - flags.store(tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, Ordering::Release); + flags.store( + tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, + Ordering::Release, + ); release.wait(); let code = caller.join().unwrap(); - assert_eq!(code, TB_PANICKED, "actual native Fault cannot disappear behind local Stop"); + assert_eq!( + code, TB_PANICKED, + "actual native Fault cannot disappear behind local Stop" + ); drop(state); - assert!(terminal::TerminalState::lookup(std::ptr::with_exposed_provenance_mut(ptr)).is_some(), "published Fault must retain the exact anchor instead of refunding its permit"); + assert!( + terminal::TerminalState::lookup(std::ptr::with_exposed_provenance_mut(ptr)).is_some(), + "published Fault must retain the exact anchor instead of refunding its permit" + ); } mod controlled { @@ -60,47 +113,87 @@ mod controlled { use std::pin::Pin; use std::task::{Context, Poll}; #[derive(serde::Deserialize)] - pub struct Config { pub mode: u8 } + pub struct Config { + pub mode: u8, + } #[derive(Debug)] - pub enum Event { Poll(u8), Request, FutureDrop(String), Finish, FactoryDrop(String) } + pub enum Event { + Poll(u8), + Request, + FutureDrop(String), + Finish, + FactoryDrop(String), + } pub struct Fixture { pub events: std::sync::mpsc::Sender, pub view: StdMutex>, wake: (StdMutex, std::sync::Condvar), worker: StdMutex>>, - pub requests: AtomicUsize, pub joins: AtomicUsize, + pub requests: AtomicUsize, + pub joins: AtomicUsize, } static NEXT: StdMutex>> = StdMutex::new(None); pub static TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); pub fn fixture() -> (Arc, std::sync::mpsc::Receiver) { let (events, receiver) = std::sync::mpsc::channel(); - let fixture = Arc::new(Fixture { events, view: StdMutex::new(None), + let fixture = Arc::new(Fixture { + events, + view: StdMutex::new(None), wake: (StdMutex::new(false), std::sync::Condvar::new()), - worker: StdMutex::new(None), requests: AtomicUsize::new(0), joins: AtomicUsize::new(0) }); + worker: StdMutex::new(None), + requests: AtomicUsize::new(0), + joins: AtomicUsize::new(0), + }); *NEXT.lock().unwrap() = Some(fixture.clone()); (fixture, receiver) } struct Capture(Arc); impl Drop for Capture { fn drop(&mut self) { - let _ = self.0.events.send(Event::FactoryDrop(std::thread::current().name().unwrap_or("unnamed").to_owned())); + let _ = self.0.events.send(Event::FactoryDrop( + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned(), + )); } } - struct SetupFuture { fixture: Arc, mode: u8, first: bool } + struct SetupFuture { + fixture: Arc, + mode: u8, + first: bool, + } impl Future for SetupFuture { type Output = Result<()>; fn poll(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - if self.first { self.first = false; self.fixture.events.send(Event::Poll(self.mode)).unwrap(); } - match self.mode { 0 => Poll::Ready(Ok(())), 2 => panic!("controlled setup poll"), - 3 => Poll::Ready(Err(Error::failed("controlled setup failure"))), _ => Poll::Pending } + if self.first { + self.first = false; + self.fixture.events.send(Event::Poll(self.mode)).unwrap(); + } + match self.mode { + 0 => Poll::Ready(Ok(())), + 2 => panic!("controlled setup poll"), + 3 => Poll::Ready(Err(Error::failed("controlled setup failure"))), + _ => Poll::Pending, + } } } impl Drop for SetupFuture { fn drop(&mut self) { - let _ = self.fixture.events.send(Event::FutureDrop(std::thread::current().name().unwrap_or("unnamed").to_owned())); + let _ = self.fixture.events.send(Event::FutureDrop( + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned(), + )); } } - fn factory(view: NativeInstanceLiveness) -> Result<(impl Fn(Connection, Config) -> Pin> + Send>> + Send + Sync, RuntimeHooks)> { + fn factory( + view: NativeInstanceLiveness, + ) -> Result<( + impl Fn(Connection, Config) -> Pin> + Send>> + Send + Sync, + RuntimeHooks, + )> { let fixture = NEXT.lock().unwrap().take().unwrap(); *fixture.view.lock().unwrap() = Some(view.clone()); let (request, finish) = (fixture.clone(), fixture.clone()); @@ -109,32 +202,52 @@ mod controlled { let _ = &capture; let mut worker = fixture.worker.lock().unwrap(); if worker.is_none() { - let work = fixture.clone(); let worker_view = view.clone(); - *worker = Some(std::thread::Builder::new().name("controlled-module-worker".into()).spawn(move || { - let mut cancelled = work.wake.0.lock().unwrap(); - while !*cancelled { cancelled = work.wake.1.wait(cancelled).unwrap(); } - drop(cancelled); - !worker_view.is_live() - }).unwrap()); + let work = fixture.clone(); + let worker_view = view.clone(); + *worker = Some( + std::thread::Builder::new() + .name("controlled-module-worker".into()) + .spawn(move || { + let mut cancelled = work.wake.0.lock().unwrap(); + while !*cancelled { + cancelled = work.wake.1.wait(cancelled).unwrap(); + } + drop(cancelled); + !worker_view.is_live() + }) + .unwrap(), + ); } drop(worker); - if config.mode == 4 { panic!("controlled setup construction"); } - Box::pin(SetupFuture { fixture: fixture.clone(), mode: config.mode, first: true }) as Pin> + Send>> - }; - let hooks = RuntimeHooks::new(move |_| { - request.requests.fetch_add(1, Ordering::SeqCst); - *request.wake.0.lock().unwrap() = true; request.wake.1.notify_all(); - request.events.send(Event::Request).unwrap(); - }, move || { - let worker = finish.worker.lock().unwrap().take(); - if let Some(worker) = worker { - let denied = worker.join().unwrap(); - finish.joins.fetch_add(1, Ordering::SeqCst); - if !denied { return TerminalCompletion::Uncertain; } + if config.mode == 4 { + panic!("controlled setup construction"); } - finish.events.send(Event::Finish).unwrap(); - TerminalCompletion::PhysicallyStopped - }); + Box::pin(SetupFuture { + fixture: fixture.clone(), + mode: config.mode, + first: true, + }) as Pin> + Send>> + }; + let hooks = RuntimeHooks::new( + move |_| { + request.requests.fetch_add(1, Ordering::SeqCst); + *request.wake.0.lock().unwrap() = true; + request.wake.1.notify_all(); + request.events.send(Event::Request).unwrap(); + }, + move || { + let worker = finish.worker.lock().unwrap().take(); + if let Some(worker) = worker { + let denied = worker.join().unwrap(); + finish.joins.fetch_add(1, Ordering::SeqCst); + if !denied { + return TerminalCompletion::Uncertain; + } + } + finish.events.send(Event::Finish).unwrap(); + TerminalCompletion::PhysicallyStopped + }, + ); Ok((setup, hooks)) } module_export_static! { @@ -149,9 +262,15 @@ async fn actual_linked_initial_cancellation_drops_future_and_joins_worker_on_phy let _serial = controlled::TEST_LOCK.lock().await; let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); - host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":1})).unwrap(); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":1}), + ) + .unwrap(); let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); let view = fixture.view.lock().unwrap().take().unwrap(); let shutdown = host.shutdown(Duration::from_secs(5)); @@ -161,12 +280,18 @@ async fn actual_linked_initial_cancellation_drops_future_and_joins_worker_on_phy let joins = fixture.joins.load(Ordering::SeqCst); // All actual workers/futures settled before asserting behavior. assert!(matches!(entered, controlled::Event::Poll(1))); - assert_eq!(request_count, 1); assert_eq!(joins, 1); + assert_eq!(request_count, 1); + assert_eq!(joins, 1); assert!(!view.is_live()); let future_drop = remainder.iter().position(|event| matches!(event, controlled::Event::FutureDrop(name) if name == "tinybus-terminal")).unwrap(); - let finish = remainder.iter().position(|event| matches!(event, controlled::Event::Finish)).unwrap(); + let finish = remainder + .iter() + .position(|event| matches!(event, controlled::Event::Finish)) + .unwrap(); assert!(future_drop < finish); - assert!(remainder.iter().any(|event| matches!(event, controlled::Event::FactoryDrop(name) if name == "tinybus-terminal"))); + assert!(remainder.iter().any( + |event| matches!(event, controlled::Event::FactoryDrop(name) if name == "tinybus-terminal") + )); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -175,15 +300,24 @@ async fn actual_linked_setup_construction_and_poll_panics_still_finish_real_work for mode in [2, 4] { let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); - host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":mode})).unwrap(); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":mode}), + ) + .unwrap(); // Consume through actual join notification before assertions or host teardown. let mut seen = Vec::new(); loop { let event = events.recv_timeout(Duration::from_secs(5)).unwrap(); - let finished = matches!(event, controlled::Event::Finish); seen.push(event); - if finished { break; } + let finished = matches!(event, controlled::Event::Finish); + seen.push(event); + if finished { + break; + } } host.shutdown(Duration::from_secs(5)).await; let view = fixture.view.lock().unwrap().take().unwrap(); @@ -198,7 +332,10 @@ async fn actual_linked_setup_construction_and_poll_panics_still_finish_real_work #[repr(C)] #[derive(Clone, Copy)] -struct NativeControls { ctx: *mut c_void, event: unsafe extern "C" fn(*mut c_void, u32, u32) } +struct NativeControls { + ctx: *mut c_void, + event: unsafe extern "C" fn(*mut c_void, u32, u32), +} unsafe extern "C" fn native_event(ctx: *mut c_void, event: u32, value: u32) { let sender = unsafe { &*ctx.cast::>() }; let _ = sender.send((event, value)); @@ -206,10 +343,22 @@ unsafe extern "C" fn native_event(ctx: *mut c_void, event: u32, value: u32) { // Only fixture symbols are looked up; TinyBus itself verifies/adopts the pinned // private artifact through the actual ModuleHost loader first. It remains mapped. #[cfg(unix)] -unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32, extern "C" fn() -> usize, unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), extern "C" fn() -> u32) { +unsafe fn terminal_fixture_symbols( + path: &std::path::Path, +) -> ( + extern "C" fn() -> usize, + extern "C" fn(usize) -> i32, + extern "C" fn() -> usize, + unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), + extern "C" fn() -> u32, +) { use std::os::unix::ffi::OsStrExt; #[cfg_attr(target_os = "linux", link(name = "dl"))] - unsafe extern "C" { fn dlopen(path: *const std::ffi::c_char, flags: i32) -> *mut c_void; fn dlsym(handle: *mut c_void, name: *const std::ffi::c_char) -> *mut c_void; fn dlclose(handle: *mut c_void) -> i32; } + unsafe extern "C" { + fn dlopen(path: *const std::ffi::c_char, flags: i32) -> *mut c_void; + fn dlsym(handle: *mut c_void, name: *const std::ffi::c_char) -> *mut c_void; + fn dlclose(handle: *mut c_void) -> i32; + } let path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); let handle = unsafe { dlopen(path.as_ptr(), 2) }; assert!(!handle.is_null()); @@ -218,15 +367,46 @@ unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() - let drops = unsafe { dlsym(handle, c"terminal_test_drops".as_ptr()) }; let early = unsafe { dlsym(handle, c"terminal_test_early".as_ptr()) }; let flags = unsafe { dlsym(handle, c"terminal_test_native_flags".as_ptr()) }; - assert!(!context.is_null() && !probe.is_null() && !drops.is_null() && !early.is_null() && !flags.is_null()); - unsafe { dlclose(handle); } - unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops), std::mem::transmute::<*mut c_void, unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32))>(early), std::mem::transmute::<*mut c_void, extern "C" fn() -> u32>(flags)) } + assert!( + !context.is_null() + && !probe.is_null() + && !drops.is_null() + && !early.is_null() + && !flags.is_null() + ); + unsafe { + dlclose(handle); + } + unsafe { + ( + std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), + std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), + std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops), + std::mem::transmute::< + *mut c_void, + unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), + >(early), + std::mem::transmute::<*mut c_void, extern "C" fn() -> u32>(flags), + ) + } } #[cfg(windows)] -unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() -> usize, extern "C" fn(usize) -> i32, extern "C" fn() -> usize, unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), extern "C" fn() -> u32) { +unsafe fn terminal_fixture_symbols( + path: &std::path::Path, +) -> ( + extern "C" fn() -> usize, + extern "C" fn(usize) -> i32, + extern "C" fn() -> usize, + unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), + extern "C" fn() -> u32, +) { use std::os::windows::ffi::OsStrExt; #[link(name = "kernel32")] - unsafe extern "system" { fn LoadLibraryW(path: *const u16) -> *mut c_void; fn GetProcAddress(handle: *mut c_void, name: *const u8) -> *mut c_void; fn FreeLibrary(handle: *mut c_void) -> i32; } + unsafe extern "system" { + fn LoadLibraryW(path: *const u16) -> *mut c_void; + fn GetProcAddress(handle: *mut c_void, name: *const u8) -> *mut c_void; + fn FreeLibrary(handle: *mut c_void) -> i32; + } let path: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); let handle = unsafe { LoadLibraryW(path.as_ptr()) }; assert!(!handle.is_null()); @@ -235,9 +415,28 @@ unsafe fn terminal_fixture_symbols(path: &std::path::Path) -> (extern "C" fn() - let drops = unsafe { GetProcAddress(handle, c"terminal_test_drops".as_ptr().cast()) }; let early = unsafe { GetProcAddress(handle, c"terminal_test_early".as_ptr().cast()) }; let flags = unsafe { GetProcAddress(handle, c"terminal_test_native_flags".as_ptr().cast()) }; - assert!(!context.is_null() && !probe.is_null() && !drops.is_null() && !early.is_null() && !flags.is_null()); - unsafe { FreeLibrary(handle); } - unsafe { (std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops), std::mem::transmute::<*mut c_void, unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32))>(early), std::mem::transmute::<*mut c_void, extern "C" fn() -> u32>(flags)) } + assert!( + !context.is_null() + && !probe.is_null() + && !drops.is_null() + && !early.is_null() + && !flags.is_null() + ); + unsafe { + FreeLibrary(handle); + } + unsafe { + ( + std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), + std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), + std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops), + std::mem::transmute::< + *mut c_void, + unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), + >(early), + std::mem::transmute::<*mut c_void, extern "C" fn() -> u32>(flags), + ) + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -255,12 +454,27 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal // If a controlled failure cannot prove teardown, conservatively retain // this one fixture callback owner; never free live foreign context. let events = Box::into_raw(Box::new(events)); - let controls = NativeControls { ctx: events.cast(), event: native_event }; - let encoded = unsafe { std::slice::from_raw_parts((&controls as *const NativeControls).cast::(), size_of::()) }.to_vec(); + let controls = NativeControls { + ctx: events.cast(), + event: native_event, + }; + let encoded = unsafe { + std::slice::from_raw_parts( + (&controls as *const NativeControls).cast::(), + size_of::(), + ) + } + .to_vec(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); - host.load_file_with_config(&path, serde_json::json!({"mode":mode,"test_controls":encoded})).unwrap(); + host.load_file_with_config( + &path, + serde_json::json!({"mode":mode,"test_controls":encoded}), + ) + .unwrap(); let (context, old_probe, drops, _, _) = unsafe { terminal_fixture_symbols(&path) }; let dropped_before = dropped_total; let actual_context = context(); @@ -268,24 +482,52 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal let stale_result = old_context.map(|token| old_probe(token)); host.shutdown(Duration::from_secs(5)).await; let mut seen = vec![first]; - while !seen.iter().any(|(event, _)| *event == 6) { seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); } + while !seen.iter().any(|(event, _)| *event == 6) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } tokio::time::timeout(Duration::from_secs(5), async { - while drops() == dropped_before { tokio::task::yield_now().await; } - }).await.unwrap(); + while drops() == dropped_before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); dropped_total = drops(); - unsafe { drop(Box::from_raw(events)); } + unsafe { + drop(Box::from_raw(events)); + } // All actual controller workers and SDK captures are destroyed before assertions. - if let Some(code) = stale_result { assert_eq!(code, TB_CLOSED, "retired successful context must not act on replacement mode {mode}"); } + if let Some(code) = stale_result { + assert_eq!( + code, TB_CLOSED, + "retired successful context must not act on replacement mode {mode}" + ); + } assert_eq!(seen.iter().filter(|(event, _)| *event == 3).count(), 1); - assert_eq!(seen.iter().filter(|(event, value)| *event == 5 && *value == 1).count(), 1); - assert_eq!(seen.iter().filter(|(event, value)| *event == 7 && *value == 1).count(), 1); + assert_eq!( + seen.iter() + .filter(|(event, value)| *event == 5 && *value == 1) + .count(), + 1 + ); + assert_eq!( + seen.iter() + .filter(|(event, value)| *event == 7 && *value == 1) + .count(), + 1 + ); assert!(seen.iter().any(|(event, value)| *event == 6 && *value == 1)); if mode != 4 { - let drop = seen.iter().position(|(event, value)| *event == 4 && *value == 1).unwrap(); + let drop = seen + .iter() + .position(|(event, value)| *event == 4 && *value == 1) + .unwrap(); let finish = seen.iter().position(|(event, _)| *event == 5).unwrap(); assert!(drop < finish); } - if mode == 1 { old_context = Some(actual_context); } + if mode == 1 { + old_context = Some(actual_context); + } } // The image was admitted by the real loader above. Symbol lookup releases @@ -293,17 +535,28 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal let (_, _, drops, early, native_flags) = unsafe { terminal_fixture_symbols(&path) }; let (sender, receiver) = std::sync::mpsc::channel(); let controls = Box::into_raw(Box::new(HeldNativeControls { - sender, release: (std::sync::Mutex::new(false), std::sync::Condvar::new()), + sender, + release: (std::sync::Mutex::new(false), std::sync::Condvar::new()), })); - unsafe { early(controls.cast(), held_native_event); } + unsafe { + early(controls.cast(), held_native_event); + } let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = tinybus::module::ModuleHost::new(broker); - let result = host.load_file_with_config(&path, serde_json::json!({"mode":0,"test_controls":[]})); + let result = + host.load_file_with_config(&path, serde_json::json!({"mode":0,"test_controls":[]})); let mut seen = Vec::new(); let entered = loop { match receiver.recv_timeout(Duration::from_secs(5)) { - Ok(event) => { seen.push(event); if event.0 == 8 { break true; } } + Ok(event) => { + seen.push(event); + if event.0 == 8 { + break true; + } + } Err(_) => break false, } }; @@ -313,7 +566,11 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal // Release the actual physical callback before any assertion, even if // rejected admission or host bookkeeping disagrees with the contract. unsafe { - *(*controls).release.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = true; + *(*controls) + .release + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = true; (*controls).release.1.notify_all(); } while !seen.iter().any(|event| event.0 == 6) { @@ -323,18 +580,35 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal } } let settled = tokio::time::timeout(Duration::from_secs(5), async { - while drops() == dropped_total { tokio::task::yield_now().await; } - }).await.is_ok(); + while drops() == dropped_total { + tokio::task::yield_now().await; + } + }) + .await + .is_ok(); let final_flags = native_flags(); // On an unresolved controlled failure retain this one callback context. // Otherwise the Acquire fence proves its final foreign use returned. - if settled { unsafe { drop(Box::from_raw(controls)); } } - assert!(result.is_err(), "source Fault before vtable publication must reject init"); - assert!(entered && settled, "actual SDK finish and resource Drop must settle independently"); + if settled { + unsafe { + drop(Box::from_raw(controls)); + } + } + assert!( + result.is_err(), + "source Fault before vtable publication must reject init" + ); + assert!( + entered && settled, + "actual SDK finish and resource Drop must settle independently" + ); for flags in [flags_before_drop, flags_after_host_drop, final_flags] { assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); - assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0, - "no accepted vtable means no host physical-worker evidence"); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0, + "no accepted vtable means no host physical-worker evidence" + ); } assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); assert!(seen.iter().any(|event| *event == (5, 1))); @@ -349,9 +623,17 @@ unsafe extern "C" fn held_native_event(ctx: *mut c_void, event: u32, value: u32) let controls = unsafe { &*ctx.cast::() }; let _ = controls.sender.send((event, value)); if event == 8 { - let mut release = controls.release.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut release = controls + .release + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); while !*release { - release = controls.release.1.wait(release).unwrap_or_else(std::sync::PoisonError::into_inner); + release = controls + .release + .1 + .wait(release) + .unwrap_or_else(std::sync::PoisonError::into_inner); } } } @@ -361,22 +643,43 @@ async fn actual_linked_reinit_terminal_cancellation_settles_before_worker_join() let _serial = controlled::TEST_LOCK.lock().await; let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); - host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); let initial = events.recv_timeout(Duration::from_secs(3)).unwrap(); tokio::time::timeout(Duration::from_secs(3), async { - while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { tokio::task::yield_now().await; } - }).await.unwrap(); + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); let reinit_host = host.clone(); - let reinit = tokio::spawn(async move { reinit_host.reinitialize("tinybus-module", serde_json::json!({"mode":1})).await }); + let reinit = tokio::spawn(async move { + reinit_host + .reinitialize("tinybus-module", serde_json::json!({"mode":1})) + .await + }); let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); let view = fixture.view.lock().unwrap().take().unwrap(); view.request_fault(); let reinit_result = reinit.await.unwrap(); host.shutdown(Duration::from_secs(5)).await; let mut remainder = Vec::new(); - while !remainder.iter().any(|event| matches!(event, controlled::Event::FactoryDrop(_))) { + while !remainder + .iter() + .any(|event| matches!(event, controlled::Event::FactoryDrop(_))) + { remainder.push(events.recv_timeout(Duration::from_secs(5)).unwrap()); } assert!(matches!(initial, controlled::Event::Poll(0))); @@ -385,8 +688,18 @@ async fn actual_linked_reinit_terminal_cancellation_settles_before_worker_join() assert_eq!(fixture.requests.load(Ordering::SeqCst), 1); assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); assert!(!view.is_live()); - let drops: Vec<_> = remainder.iter().enumerate().filter_map(|(index, event)| matches!(event, controlled::Event::FutureDrop(name) if name == "tinybus-terminal").then_some(index)).collect(); - let finish = remainder.iter().position(|event| matches!(event, controlled::Event::Finish)).unwrap(); + let drops: Vec<_> = remainder + .iter() + .enumerate() + .filter_map(|(index, event)| { + matches!(event, controlled::Event::FutureDrop(name) if name == "tinybus-terminal") + .then_some(index) + }) + .collect(); + let finish = remainder + .iter() + .position(|event| matches!(event, controlled::Event::Finish)) + .unwrap(); assert_eq!(drops.len(), 2); assert!(drops.into_iter().all(|index| index < finish)); } diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 41ddf1d2..93019adb 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -2,17 +2,27 @@ use super::*; use std::any::Any; use std::pin::Pin; -use std::sync::{Condvar, Weak}; use std::sync::atomic::{AtomicU8, AtomicUsize}; -use tinybus::module::abi::{TB_NATIVE_TERMINAL_LIVE, TB_NATIVE_TERMINAL_FAULT, TB_NATIVE_TERMINAL_STOP_INTENT}; +use std::sync::{Condvar, Weak}; +use tinybus::module::abi::{ + TB_NATIVE_TERMINAL_FAULT, TB_NATIVE_TERMINAL_LIVE, TB_NATIVE_TERMINAL_STOP_INTENT, +}; /// Cause of irreversible instance admission closure. Genuine fault takes /// precedence over ordinary stop or host disappearance. #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum TerminalReason { Stop, HostClosed, Fault, SetupFailed } +pub enum TerminalReason { + Stop, + HostClosed, + Fault, + SetupFailed, +} /// Actual controller cleanup outcome, independent of native admission. #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum TerminalCompletion { PhysicallyStopped, Uncertain } +pub enum TerminalCompletion { + PhysicallyStopped, + Uncertain, +} /// Controller callbacks supplied by the terminal factory. pub struct RuntimeHooks { request: Arc, @@ -25,76 +35,140 @@ impl RuntimeHooks { /// has settled; it must not depend on cancelled futures or host shutdown. /// Returning `PhysicallyStopped` asserts actual fence/join, not a timeout. pub fn new(request: R, finish: F) -> Self - where R: Fn(TerminalReason) + Send + Sync + 'static, - F: FnOnce() -> TerminalCompletion + Send + 'static { - Self { request: Arc::new(request), finish: Box::new(finish) } + where + R: Fn(TerminalReason) + Send + Sync + 'static, + F: FnOnce() -> TerminalCompletion + Send + 'static, + { + Self { + request: Arc::new(request), + finish: Box::new(finish), + } } } /// A retained view of one actual admitted native instance. Its lifetime retains /// the instance's admission permit even after successful physical cleanup. #[derive(Clone)] -pub struct NativeInstanceLiveness { pub(super) inner: Arc } +pub struct NativeInstanceLiveness { + pub(super) inner: Arc, +} impl NativeInstanceLiveness { /// Only a supported, exactly-live native query allows new admission. /// Observed terminal state never reopens. This is not a publication lease. - pub fn is_live(&self) -> bool { self.inner.is_live() } + pub fn is_live(&self) -> bool { + self.inner.is_live() + } /// Persist genuine fault and close independent native admission first. - pub fn request_fault(&self) { self.inner.signal(TerminalReason::Fault); } + pub fn request_fault(&self) { + self.inner.signal(TerminalReason::Fault); + } } const INSTANCE_LIMIT: usize = 256; #[derive(Default)] -struct Slot { reserved: bool, exhausted: bool, generation: usize, anchor: Option> } -struct Admission { slots: StdMutex<[Slot; INSTANCE_LIMIT]> } +struct Slot { + reserved: bool, + exhausted: bool, + generation: usize, + anchor: Option>, +} +struct Admission { + slots: StdMutex<[Slot; INSTANCE_LIMIT]>, +} impl Admission { fn new() -> Arc { - Arc::new(Self { slots: StdMutex::new(std::array::from_fn(|_| Slot::default())) }) + Arc::new(Self { + slots: StdMutex::new(std::array::from_fn(|_| Slot::default())), + }) } } static ADMISSION: OnceLock> = OnceLock::new(); -struct SlotLease { admission: Weak, index: usize } +struct SlotLease { + admission: Weak, + index: usize, +} impl Drop for SlotLease { fn drop(&mut self) { if let Some(admission) = self.admission.upgrade() { - let mut slots = admission.slots.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut slots = admission + .slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); // The permit ends only at actual State last-Drop, never at anchor retirement. slots[self.index].reserved = false; } } } #[derive(Clone, Copy, Eq, PartialEq)] -enum RequestState { Unregistered, Idle, Running, Done, Panicked } +enum RequestState { + Unregistered, + Idle, + Running, + Done, + Panicked, +} #[derive(Clone, Copy, Eq, PartialEq)] -enum FinishState { Idle, Submitted, Running, Complete, SpawnFailed } -struct FinishJob { state: Weak } +enum FinishState { + Idle, + Submitted, + Running, + Complete, + SpawnFailed, +} +struct FinishJob { + state: Weak, +} impl FinishJob { - fn run(self: Arc, state: Arc) { state.run_finish(); } + fn run(self: Arc, state: Arc) { + state.run_finish(); + } } type Payload = Box; #[derive(Default)] struct PrimaryPayloads { - request: Option, finish: Option, initial: Option, - reinit: Option, factory: Option, startup: Option, + request: Option, + finish: Option, + initial: Option, + reinit: Option, + factory: Option, + startup: Option, cleanup: Option, } #[derive(Clone, Copy)] -pub(super) enum PayloadKind { Request, Finish, Initial, Reinit, Factory, Startup, Cleanup } +pub(super) enum PayloadKind { + Request, + Finish, + Initial, + Reinit, + Factory, + Startup, + Cleanup, +} impl PrimaryPayloads { fn cell(&mut self, kind: PayloadKind) -> &mut Option { - match kind { PayloadKind::Request => &mut self.request, PayloadKind::Finish => &mut self.finish, - PayloadKind::Initial => &mut self.initial, PayloadKind::Reinit => &mut self.reinit, - PayloadKind::Factory => &mut self.factory, PayloadKind::Startup => &mut self.startup, - PayloadKind::Cleanup => &mut self.cleanup } + match kind { + PayloadKind::Request => &mut self.request, + PayloadKind::Finish => &mut self.finish, + PayloadKind::Initial => &mut self.initial, + PayloadKind::Reinit => &mut self.reinit, + PayloadKind::Factory => &mut self.factory, + PayloadKind::Startup => &mut self.startup, + PayloadKind::Cleanup => &mut self.cleanup, + } } } struct Control { request: Option>, finish: Option TerminalCompletion + Send>>, - request_state: RequestState, finish_state: FinishState, - published: bool, completion: Option, job: Option>, + request_state: RequestState, + finish_state: FinishState, + published: bool, + completion: Option, + job: Option>, payloads: PrimaryPayloads, } -pub(super) type Setup = Box Pin> + Send>> + Send>; -pub(super) type SetupFutureSlot = Arc> + Send>>>>>; +pub(super) type Setup = + Box Pin> + Send>> + Send>; +pub(super) type SetupFutureSlot = + Arc> + Send>>>>>; /// Resources are attached here before the next fallible startup boundary. /// Taking values is mechanical; all foreign destruction occurs off this lock. @@ -113,16 +187,28 @@ pub(super) struct StartupResources { } impl Default for StartupResources { fn default() -> Self { - Self { runtime: None, setup: None, retained_factory: None, config: None, - reinitializer_factory: None, reinitialize: Arc::new(StdMutex::new(None)), - connection: Arc::new(StdMutex::new(None)), inbound: None, initial: None, - initial_future: Arc::new(StdMutex::new(None)), reinit_future: Arc::new(StdMutex::new(None)) } + Self { + runtime: None, + setup: None, + retained_factory: None, + config: None, + reinitializer_factory: None, + reinitialize: Arc::new(StdMutex::new(None)), + connection: Arc::new(StdMutex::new(None)), + inbound: None, + initial: None, + initial_future: Arc::new(StdMutex::new(None)), + reinit_future: Arc::new(StdMutex::new(None)), + } } } pub(super) struct TerminalState { - pub host: HostCalls, pub owner: Option, - reason: AtomicU8, uncertain: AtomicBool, - control: StdMutex, changed: Condvar, + pub host: HostCalls, + pub owner: Option, + reason: AtomicU8, + uncertain: AtomicBool, + control: StdMutex, + changed: Condvar, pub terminal_notify: tokio::sync::Notify, pub resources: StdMutex, sdk_owners: AtomicUsize, @@ -132,118 +218,211 @@ pub(super) struct TerminalState { } /// Every surviving SDK transport/Connection/task retains this actual State. /// A pending runtime teardown may therefore never recycle its slot. -pub(super) struct SdkOwner { state: Arc } +pub(super) struct SdkOwner { + state: Arc, +} impl Drop for SdkOwner { - fn drop(&mut self) { self.state.sdk_owners.fetch_sub(1, Ordering::AcqRel); } + fn drop(&mut self) { + self.state.sdk_owners.fetch_sub(1, Ordering::AcqRel); + } } impl TerminalState { pub fn admit(host: HostCalls, owner: Option) -> Option> { Self::admit_in(ADMISSION.get_or_init(Admission::new), host, owner) } - fn admit_in(admission: &Arc, host: HostCalls, owner: Option) -> Option> { - let mut slots = admission.slots.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + fn admit_in( + admission: &Arc, + host: HostCalls, + owner: Option, + ) -> Option> { + let mut slots = admission + .slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let index = slots.iter_mut().enumerate().find_map(|(index, slot)| { - if slot.reserved || slot.exhausted { return None; } - let generation = slot.generation.checked_add(1).filter(|generation| *generation <= usize::MAX >> 8); + if slot.reserved || slot.exhausted { + return None; + } + let generation = slot + .generation + .checked_add(1) + .filter(|generation| *generation <= usize::MAX >> 8); match generation { - Some(generation) => { slot.generation = generation; Some(index) }, - None => { slot.exhausted = true; None }, + Some(generation) => { + slot.generation = generation; + Some(index) + } + None => { + slot.exhausted = true; + None + } } })?; slots[index].reserved = true; let token = (slots[index].generation << 8) | index; let state = Arc::new(Self { - host, owner, reason: AtomicU8::new(0), uncertain: AtomicBool::new(false), - control: StdMutex::new(Control { request: None, finish: None, - request_state: RequestState::Unregistered, finish_state: FinishState::Idle, - published: false, completion: None, job: None, payloads: PrimaryPayloads::default() }), - changed: Condvar::new(), terminal_notify: tokio::sync::Notify::new(), - resources: StdMutex::new(StartupResources::default()), sdk_owners: AtomicUsize::new(0), vtable_published: AtomicBool::new(false), - lease: SlotLease { admission: Arc::downgrade(admission), index }, token, + host, + owner, + reason: AtomicU8::new(0), + uncertain: AtomicBool::new(false), + control: StdMutex::new(Control { + request: None, + finish: None, + request_state: RequestState::Unregistered, + finish_state: FinishState::Idle, + published: false, + completion: None, + job: None, + payloads: PrimaryPayloads::default(), + }), + changed: Condvar::new(), + terminal_notify: tokio::sync::Notify::new(), + resources: StdMutex::new(StartupResources::default()), + sdk_owners: AtomicUsize::new(0), + vtable_published: AtomicBool::new(false), + lease: SlotLease { + admission: Arc::downgrade(admission), + index, + }, + token, }); slots[index].anchor = Some(state.clone()); Some(state) } /// Opaque address token only; no allocation or dereference occurs. - pub fn context(&self) -> *mut c_void { std::ptr::with_exposed_provenance_mut(self.token) } + pub fn context(&self) -> *mut c_void { + std::ptr::with_exposed_provenance_mut(self.token) + } pub fn lookup(context: *mut c_void) -> Option> { Self::lookup_in(ADMISSION.get()?, context.addr()) } fn lookup_in(admission: &Arc, token: usize) -> Option> { let generation = token >> 8; - if generation == 0 { return None; } - let slots = admission.slots.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if generation == 0 { + return None; + } + let slots = admission + .slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let slot = &slots[token & (INSTANCE_LIMIT - 1)]; - if slot.generation != generation { return None; } + if slot.generation != generation { + return None; + } // Strong exact State is acquired before releasing registry exclusion. - slot.anchor.as_ref().filter(|state| state.token == token).cloned() + slot.anchor + .as_ref() + .filter(|state| state.token == token) + .cloned() + } + pub fn mark_vtable_published(&self) { + self.vtable_published.store(true, Ordering::Release); } - pub fn mark_vtable_published(&self) { self.vtable_published.store(true, Ordering::Release); } pub fn retain_uncertain_anchor(self: &Arc) { self.uncertain.store(true, Ordering::Release); if let Some(admission) = self.lease.admission.upgrade() { - let mut slots = admission.slots.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut slots = admission + .slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let slot = &mut slots[self.lease.index]; // This State's still-live lease forbids reuse while a C callback or // retained view performs the final genuine-fault recheck. - if slot.generation == self.token >> 8 && slot.anchor.is_none() { slot.anchor = Some(self.clone()); } + if slot.generation == self.token >> 8 && slot.anchor.is_none() { + slot.anchor = Some(self.clone()); + } + } + } + pub fn view(self: &Arc) -> NativeInstanceLiveness { + NativeInstanceLiveness { + inner: self.clone(), } } - pub fn view(self: &Arc) -> NativeInstanceLiveness { NativeInstanceLiveness { inner: self.clone() } } pub fn sdk_owner(self: &Arc) -> SdkOwner { - self.sdk_owners.fetch_add(1, Ordering::AcqRel); SdkOwner { state: self.clone() } + self.sdk_owners.fetch_add(1, Ordering::AcqRel); + SdkOwner { + state: self.clone(), + } + } + pub fn is_terminal(&self) -> bool { + self.reason.load(Ordering::Acquire) != 0 } - pub fn is_terminal(&self) -> bool { self.reason.load(Ordering::Acquire) != 0 } pub fn is_live(self: &Arc) -> bool { - if self.is_terminal() { return false; } + if self.is_terminal() { + return false; + } self.observe_host(); !self.is_terminal() } // Cleanup must sample even when local Stop already closed admission: a real // later native Fault is orthogonal to that local admission predicate. pub fn observe_host(self: &Arc) { - let flags = self.host.0.native_terminal_state.map(|query| unsafe { query(self.host.0.host_ctx) }); + let flags = self + .host + .0 + .native_terminal_state + .map(|query| unsafe { query(self.host.0.host_ctx) }); if flags != Some(TB_NATIVE_TERMINAL_LIVE) { let reason = if flags.is_some_and(|flags| flags & TB_NATIVE_TERMINAL_FAULT != 0) { TerminalReason::Fault } else if flags.is_some_and(|flags| flags & TB_NATIVE_TERMINAL_STOP_INTENT != 0) { TerminalReason::Stop - } else { TerminalReason::HostClosed }; + } else { + TerminalReason::HostClosed + }; self.signal(reason); } } fn reason(&self) -> TerminalReason { - match self.reason.load(Ordering::Acquire) { 1 => TerminalReason::Stop, 2 => TerminalReason::HostClosed, - 4 => TerminalReason::SetupFailed, _ => TerminalReason::Fault } + match self.reason.load(Ordering::Acquire) { + 1 => TerminalReason::Stop, + 2 => TerminalReason::HostClosed, + 4 => TerminalReason::SetupFailed, + _ => TerminalReason::Fault, + } } fn commit(&self, reason: TerminalReason) { - let next = match reason { TerminalReason::Stop => 1, TerminalReason::HostClosed => 2, - TerminalReason::Fault => 3, TerminalReason::SetupFailed => 4 }; + let next = match reason { + TerminalReason::Stop => 1, + TerminalReason::HostClosed => 2, + TerminalReason::Fault => 3, + TerminalReason::SetupFailed => 4, + }; let mut old = self.reason.load(Ordering::Acquire); while old == 0 || (old < 3 && next >= 3) { - match self.reason.compare_exchange_weak(old, next, Ordering::AcqRel, Ordering::Acquire) { - Ok(_) => break, Err(actual) => old = actual, + match self + .reason + .compare_exchange_weak(old, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => break, + Err(actual) => old = actual, } } self.terminal_notify.notify_waiters(); } pub fn signal(self: &Arc, reason: TerminalReason) { - if matches!(reason, TerminalReason::Fault | TerminalReason::SetupFailed) { self.host.fault(); } + if matches!(reason, TerminalReason::Fault | TerminalReason::SetupFailed) { + self.host.fault(); + } self.commit(reason); self.run_request(); self.schedule(); } pub fn source_panic(self: &Arc, kind: PayloadKind, payload: Payload) { // Atomic authority fence must precede payload-cell locks and foreign Drop. - self.host.fault(); self.commit(TerminalReason::Fault); + self.host.fault(); + self.commit(TerminalReason::Fault); self.uncertain.store(true, Ordering::Release); self.keep_payload(kind, payload); - self.run_request(); self.schedule(); + self.run_request(); + self.schedule(); } fn lock(&self) -> std::sync::MutexGuard<'_, Control> { self.control.lock().unwrap_or_else(|poison| { - self.host.fault(); self.commit(TerminalReason::Fault); - self.uncertain.store(true, Ordering::Release); poison.into_inner() + self.host.fault(); + self.commit(TerminalReason::Fault); + self.uncertain.store(true, Ordering::Release); + poison.into_inner() }) } fn keep_payload(&self, kind: PayloadKind, payload: Payload) { @@ -256,27 +435,40 @@ impl TerminalState { *cell = Some(payload); } pub fn register(self: &Arc, hooks: RuntimeHooks) { - { let mut control = self.lock(); - control.request = Some(hooks.request); control.finish = Some(hooks.finish); + { + let mut control = self.lock(); + control.request = Some(hooks.request); + control.finish = Some(hooks.finish); control.request_state = RequestState::Idle; } - if self.is_terminal() { self.run_request(); self.schedule(); } + if self.is_terminal() { + self.run_request(); + self.schedule(); + } } fn run_request(self: &Arc) { let request = { let mut control = self.lock(); - if !self.is_terminal() || control.request_state != RequestState::Idle { return; } + if !self.is_terminal() || control.request_state != RequestState::Idle { + return; + } control.request_state = RequestState::Running; control.request.clone() }; let _scope = self.owner.as_ref().map(ExecutionOwner::enter); let outcome = catch_unwind(AssertUnwindSafe(|| { - if let Some(request) = request { request(self.reason()); } + if let Some(request) = request { + request(self.reason()); + } })); match outcome { - Ok(()) => { self.lock().request_state = RequestState::Done; self.changed.notify_all(); } + Ok(()) => { + self.lock().request_state = RequestState::Done; + self.changed.notify_all(); + } Err(payload) => { - self.host.fault(); self.commit(TerminalReason::Fault); + self.host.fault(); + self.commit(TerminalReason::Fault); self.uncertain.store(true, Ordering::Release); self.lock().request_state = RequestState::Panicked; self.changed.notify_all(); @@ -285,27 +477,47 @@ impl TerminalState { } } pub fn publish(self: &Arc) { - self.lock().published = true; self.changed.notify_all(); - if self.is_terminal() { self.run_request(); self.schedule(); } + self.lock().published = true; + self.changed.notify_all(); + if self.is_terminal() { + self.run_request(); + self.schedule(); + } } fn schedule(self: &Arc) { let job = { let mut control = self.lock(); - if control.finish_state != FinishState::Idle { return; } - let job = Arc::new(FinishJob { state: Arc::downgrade(self) }); - control.job = Some(job.clone()); control.finish_state = FinishState::Submitted; job + if control.finish_state != FinishState::Idle { + return; + } + let job = Arc::new(FinishJob { + state: Arc::downgrade(self), + }); + control.job = Some(job.clone()); + control.finish_state = FinishState::Submitted; + job }; let state = job.state.upgrade().expect("anchored terminal state"); - if std::thread::Builder::new().name("tinybus-terminal".into()).spawn(move || job.run(state)).is_err() { + if std::thread::Builder::new() + .name("tinybus-terminal".into()) + .spawn(move || job.run(state)) + .is_err() + { self.uncertain.store(true, Ordering::Release); - let mut control = self.lock(); control.finish_state = FinishState::SpawnFailed; - control.completion = Some(TerminalCompletion::Uncertain); drop(control); self.changed.notify_all(); + let mut control = self.lock(); + control.finish_state = FinishState::SpawnFailed; + control.completion = Some(TerminalCompletion::Uncertain); + drop(control); + self.changed.notify_all(); } } fn remove_anchor(&self) { if let Some(admission) = self.lease.admission.upgrade() { let anchor = { - let mut slots = admission.slots.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut slots = admission + .slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); slots[self.lease.index].anchor.take() }; drop(anchor); @@ -313,95 +525,188 @@ impl TerminalState { } pub fn empty_factory_failure(self: &Arc) { // No callable/controller was transferred. Retained views still hold the permit. - self.commit(TerminalReason::SetupFailed); self.remove_anchor(); + self.commit(TerminalReason::SetupFailed); + self.remove_anchor(); } fn run_finish(self: &Arc) { let _scope = self.owner.as_ref().map(ExecutionOwner::enter); let finish = { let mut control = self.lock(); - while !control.published || matches!(control.request_state, RequestState::Idle | RequestState::Running) { - control = self.changed.wait(control).unwrap_or_else(std::sync::PoisonError::into_inner); + while !control.published + || matches!( + control.request_state, + RequestState::Idle | RequestState::Running + ) + { + control = self + .changed + .wait(control) + .unwrap_or_else(std::sync::PoisonError::into_inner); } - control.finish_state = FinishState::Running; control.finish.take() + control.finish_state = FinishState::Running; + control.finish.take() }; let cleanup = catch_unwind(AssertUnwindSafe(|| self.settle_setup())); let setup_settled = match cleanup { Ok(settled) => settled, - Err(payload) => { self.source_panic(PayloadKind::Cleanup, payload); false } + Err(payload) => { + self.source_panic(PayloadKind::Cleanup, payload); + false + } }; let completed = if setup_settled { - if let Some(owner) = &self.owner { owner.stop(); } - match catch_unwind(AssertUnwindSafe(|| finish.map_or(TerminalCompletion::PhysicallyStopped, |finish| finish()))) { + if let Some(owner) = &self.owner { + owner.stop(); + } + match catch_unwind(AssertUnwindSafe(|| { + finish.map_or(TerminalCompletion::PhysicallyStopped, |finish| finish()) + })) { Ok(completion) => completion, - Err(payload) => { self.source_panic(PayloadKind::Finish, payload); TerminalCompletion::Uncertain } + Err(payload) => { + self.source_panic(PayloadKind::Finish, payload); + TerminalCompletion::Uncertain + } } } else { // If setup destruction could not be proved, keep the original uninvoked finish owner. - self.lock().finish = finish; TerminalCompletion::Uncertain + self.lock().finish = finish; + TerminalCompletion::Uncertain }; let released = if completed == TerminalCompletion::PhysicallyStopped { match catch_unwind(AssertUnwindSafe(|| self.release_resources())) { Ok(released) => released, - Err(payload) => { self.source_panic(PayloadKind::Cleanup, payload); false } + Err(payload) => { + self.source_panic(PayloadKind::Cleanup, payload); + false + } } - } else { false }; + } else { + false + }; self.observe_host(); if !released || (self.has_fault() && self.vtable_published.load(Ordering::Acquire)) { self.uncertain.store(true, Ordering::Release); } - let completion = if self.uncertain.load(Ordering::Acquire) { TerminalCompletion::Uncertain } else { completed }; - { let mut control = self.lock(); control.finish_state = FinishState::Complete; control.completion = Some(completion); } + let completion = if self.uncertain.load(Ordering::Acquire) { + TerminalCompletion::Uncertain + } else { + completed + }; + { + let mut control = self.lock(); + control.finish_state = FinishState::Complete; + control.completion = Some(completion); + } self.changed.notify_all(); - if completion == TerminalCompletion::PhysicallyStopped { self.remove_anchor(); } + if completion == TerminalCompletion::PhysicallyStopped { + self.remove_anchor(); + } } fn settle_setup(self: &Arc) -> bool { let (initial, handle, reinitialize, future, reinit_future) = { - let mut resources = self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - (resources.initial.take(), resources.runtime.as_ref().map(|runtime| runtime.handle().clone()), resources.reinitialize.clone(), resources.initial_future.clone(), resources.reinit_future.clone()) + let mut resources = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ( + resources.initial.take(), + resources + .runtime + .as_ref() + .map(|runtime| runtime.handle().clone()), + resources.reinitialize.clone(), + resources.initial_future.clone(), + resources.reinit_future.clone(), + ) }; if let Some(initial) = initial { initial.abort(); - let Some(handle) = handle else { self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).initial = Some(initial); return false; }; + let Some(handle) = handle else { + self.resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .initial = Some(initial); + return false; + }; match handle.block_on(initial) { - Ok(Ok(())) => {}, + Ok(Ok(())) => {} Ok(Err(_error)) => self.signal(TerminalReason::SetupFailed), - Err(error) if error.is_cancelled() => {}, + Err(error) if error.is_cancelled() => {} Err(error) => { - if error.is_panic() { self.source_panic(PayloadKind::Initial, error.into_panic()); } - else { self.signal(TerminalReason::SetupFailed); } + if error.is_panic() { + self.source_panic(PayloadKind::Initial, error.into_panic()); + } else { + self.signal(TerminalReason::SetupFailed); + } } } } - let callback = reinitialize.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); - if let Some(owner) = &self.owner { owner.stop(); } - let future = future.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + let callback = reinitialize + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + if let Some(owner) = &self.owner { + owner.stop(); + } + let future = future + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); drop(future); drop(callback); let request = self.lock().request.take(); drop(request); - let future = reinit_future.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + let future = reinit_future + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); drop(future); true } fn release_resources(&self) -> bool { let (setup, factory, config, reinit_factory, connection, inbound) = { - let mut resources = self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - (resources.setup.take(), resources.retained_factory.take(), resources.config.take(), - resources.reinitializer_factory.take(), resources.connection.clone(), resources.inbound.take()) + let mut resources = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ( + resources.setup.take(), + resources.retained_factory.take(), + resources.config.take(), + resources.reinitializer_factory.take(), + resources.connection.clone(), + resources.inbound.take(), + ) }; - let connection = connection.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + let connection = connection + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); drop((setup, factory, config, reinit_factory, connection, inbound)); - let runtime = self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).runtime.take(); - if let Some(runtime) = runtime { runtime.shutdown_timeout(Duration::from_secs(4)); } + let runtime = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .runtime + .take(); + if let Some(runtime) = runtime { + runtime.shutdown_timeout(Duration::from_secs(4)); + } // Timeout is not proof: any surviving SDK transport/task keeps State // and its resource count. Unknown surviving ownership stays anchored. self.sdk_owners.load(Ordering::Acquire) == 0 } - pub fn has_fault(&self) -> bool { self.reason.load(Ordering::Acquire) >= 3 } - pub fn mark_uncertain(&self) { self.uncertain.store(true, Ordering::Release); } + pub fn has_fault(&self) -> bool { + self.reason.load(Ordering::Acquire) >= 3 + } + pub fn mark_uncertain(&self) { + self.uncertain.store(true, Ordering::Release); + } pub fn wait(&self, deadline: Duration) -> TerminalCompletion { let control = self.lock(); - let (control, _) = self.changed.wait_timeout_while(control, deadline, |control| control.completion.is_none()) + let (control, _) = self + .changed + .wait_timeout_while(control, deadline, |control| control.completion.is_none()) .unwrap_or_else(std::sync::PoisonError::into_inner); control.completion.unwrap_or(TerminalCompletion::Uncertain) } diff --git a/crates/tinybus-module/src/terminal_tests.rs b/crates/tinybus-module/src/terminal_tests.rs index 9f713cfc..c04a450b 100644 --- a/crates/tinybus-module/src/terminal_tests.rs +++ b/crates/tinybus-module/src/terminal_tests.rs @@ -1,20 +1,43 @@ use super::*; use std::sync::atomic::AtomicU32; -use tinybus::module::abi::{TbSlice, TB_NATIVE_TERMINAL_STOP_INTENT}; +use tinybus::module::abi::{TB_NATIVE_TERMINAL_STOP_INTENT, TbSlice}; +static TEST_LOCK: StdMutex<()> = StdMutex::new(()); static FLAGS: AtomicU32 = AtomicU32::new(TB_NATIVE_TERMINAL_LIVE); -unsafe extern "C" fn query(_: *mut c_void) -> u32 { FLAGS.load(Ordering::Acquire) } -unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { TB_OK } +unsafe extern "C" fn query(_: *mut c_void) -> u32 { + FLAGS.load(Ordering::Acquire) +} +unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { + TB_OK +} unsafe extern "C" fn notify(_: *mut c_void) {} -unsafe extern "C" fn fault(_: *mut c_void, _: *const u8, _: usize) { FLAGS.store(4, Ordering::Release); } +unsafe extern "C" fn fault(_: *mut c_void, _: *const u8, _: usize) { + FLAGS.store(4, Ordering::Release); +} unsafe extern "C" fn log(_: *mut c_void, _: u32, _: *const u8, _: usize) {} fn host() -> HostCalls { - HostCalls(TbHostVtable { size: size_of::() as u32, _reserved: 0, - host_ctx: std::ptr::null_mut(), send, wake: notify, log, ready: notify, - fault, config: TbSlice { ptr: std::ptr::null(), len: 0 }, broker_routing: None, - native_terminal_state: Some(query) }) + HostCalls(TbHostVtable { + size: size_of::() as u32, + _reserved: 0, + host_ctx: std::ptr::null_mut(), + send, + wake: notify, + log, + ready: notify, + fault, + config: TbSlice { + ptr: std::ptr::null(), + len: 0, + }, + broker_routing: None, + native_terminal_state: Some(query), + }) } #[test] fn observed_native_terminal_never_reopens_a_retained_view() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); let admission = Admission::new(); let state = TerminalState::admit_in(&admission, host(), None).unwrap(); @@ -23,13 +46,23 @@ fn observed_native_terminal_never_reopens_a_retained_view() { FLAGS.store(TB_NATIVE_TERMINAL_STOP_INTENT, Ordering::Release); assert!(!view.is_live()); FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); - assert!(!view.is_live(), "an observed terminal tombstone must stay closed"); + assert!( + !view.is_live(), + "an observed terminal tombstone must stay closed" + ); state.publish(); - assert_eq!(state.wait(Duration::from_secs(3)), TerminalCompletion::PhysicallyStopped); + assert_eq!( + state.wait(Duration::from_secs(3)), + TerminalCompletion::PhysicallyStopped + ); } #[test] fn retired_anchor_cannot_refund_permit_while_old_view_survives() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); let admission = Admission::new(); let mut views = Vec::new(); for _ in 0..INSTANCE_LIMIT { @@ -43,40 +76,70 @@ fn retired_anchor_cannot_refund_permit_while_old_view_survives() { replacement.empty_factory_failure(); drop(replacement); drop(views); - assert!(admission.slots.lock().unwrap().iter().all(|slot| !slot.reserved)); + assert!( + admission + .slots + .lock() + .unwrap() + .iter() + .all(|slot| !slot.reserved) + ); } #[test] fn request_primary_payload_is_retained_and_finish_still_attempted_once() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); use std::sync::atomic::AtomicUsize; struct PayloadDrop(Arc); - impl Drop for PayloadDrop { fn drop(&mut self) { self.0.fetch_add(1, Ordering::SeqCst); } } + impl Drop for PayloadDrop { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } let admission = Admission::new(); let state = TerminalState::admit_in(&admission, host(), None).unwrap(); let dropped = Arc::new(AtomicUsize::new(0)); let requests = Arc::new(AtomicUsize::new(0)); let finishes = Arc::new(AtomicUsize::new(0)); - let (request_count, finish_count, payload) = (requests.clone(), finishes.clone(), dropped.clone()); - state.register(RuntimeHooks::new(move |_| { - request_count.fetch_add(1, Ordering::SeqCst); - std::panic::panic_any(PayloadDrop(payload.clone())); - }, move || { - finish_count.fetch_add(1, Ordering::SeqCst); - TerminalCompletion::PhysicallyStopped - })); + let (request_count, finish_count, payload) = + (requests.clone(), finishes.clone(), dropped.clone()); + state.register(RuntimeHooks::new( + move |_| { + request_count.fetch_add(1, Ordering::SeqCst); + std::panic::panic_any(PayloadDrop(payload.clone())); + }, + move || { + finish_count.fetch_add(1, Ordering::SeqCst); + TerminalCompletion::PhysicallyStopped + }, + )); state.publish(); state.signal(TerminalReason::Stop); - assert_eq!(state.wait(Duration::from_secs(3)), TerminalCompletion::Uncertain); + assert_eq!( + state.wait(Duration::from_secs(3)), + TerminalCompletion::Uncertain + ); state.signal(TerminalReason::Fault); assert_eq!(requests.load(Ordering::SeqCst), 1); assert_eq!(finishes.load(Ordering::SeqCst), 1); assert_eq!(dropped.load(Ordering::SeqCst), 0); assert!(state.lock().request_state == RequestState::Panicked); - assert!(admission.slots.lock().unwrap()[state.lease.index].anchor.is_some()); + assert!( + admission.slots.lock().unwrap()[state.lease.index] + .anchor + .is_some() + ); } #[test] fn physical_finish_waits_for_bounded_request_completion_and_startup_publication() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); let admission = Admission::new(); let state = TerminalState::admit_in(&admission, host(), None).unwrap(); let (entered_tx, entered_rx) = std::sync::mpsc::channel(); @@ -84,51 +147,94 @@ fn physical_finish_waits_for_bounded_request_completion_and_startup_publication( let request_release = release.clone(); let finished = Arc::new(AtomicBool::new(false)); let finish_seen = finished.clone(); - state.register(RuntimeHooks::new(move |_| { - entered_tx.send(()).unwrap(); request_release.wait(); - }, move || { finish_seen.store(true, Ordering::Release); TerminalCompletion::PhysicallyStopped })); + state.register(RuntimeHooks::new( + move |_| { + entered_tx.send(()).unwrap(); + request_release.wait(); + }, + move || { + finish_seen.store(true, Ordering::Release); + TerminalCompletion::PhysicallyStopped + }, + )); let caller_state = state.clone(); let caller = std::thread::spawn(move || caller_state.signal(TerminalReason::Stop)); entered_rx.recv().unwrap(); state.publish(); assert!(!finished.load(Ordering::Acquire)); - release.wait(); caller.join().unwrap(); - assert_eq!(state.wait(Duration::from_secs(3)), TerminalCompletion::PhysicallyStopped); + release.wait(); + caller.join().unwrap(); + assert_eq!( + state.wait(Duration::from_secs(3)), + TerminalCompletion::PhysicallyStopped + ); assert!(finished.load(Ordering::Acquire)); } #[test] fn surviving_sdk_owned_transport_keeps_cleanup_uncertain_and_permit_anchored() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); let admission = Admission::new(); let state = TerminalState::admit_in(&admission, host(), None).unwrap(); let sdk_owner = state.sdk_owner(); - state.register(RuntimeHooks::new(|_| {}, || TerminalCompletion::PhysicallyStopped)); - state.publish(); state.signal(TerminalReason::Stop); - assert_eq!(state.wait(Duration::from_secs(3)), TerminalCompletion::Uncertain); + state.register(RuntimeHooks::new( + |_| {}, + || TerminalCompletion::PhysicallyStopped, + )); + state.publish(); + state.signal(TerminalReason::Stop); + assert_eq!( + state.wait(Duration::from_secs(3)), + TerminalCompletion::Uncertain + ); drop(sdk_owner); - assert!(admission.slots.lock().unwrap()[state.lease.index].anchor.is_some()); + assert!( + admission.slots.lock().unwrap()[state.lease.index] + .anchor + .is_some() + ); } #[test] fn stale_context_never_selects_replacement_and_unknown_contexts_are_closed() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); let admission = Admission::new(); let state = TerminalState::admit_in(&admission, host(), None).unwrap(); let old = state.context().addr(); - state.empty_factory_failure(); drop(state); + state.empty_factory_failure(); + drop(state); let replacement = TerminalState::admit_in(&admission, host(), None).unwrap(); assert_eq!(replacement.lease.index, old & 255); assert_ne!(replacement.context().addr(), old); assert!(TerminalState::lookup_in(&admission, old).is_none()); - for invalid in [0, 1, 255, usize::MAX] { assert!(TerminalState::lookup_in(&admission, invalid).is_none()); } - assert!(Arc::ptr_eq(&TerminalState::lookup_in(&admission, replacement.token).unwrap(), &replacement)); + for invalid in [0, 1, 255, usize::MAX] { + assert!(TerminalState::lookup_in(&admission, invalid).is_none()); + } + assert!(Arc::ptr_eq( + &TerminalState::lookup_in(&admission, replacement.token).unwrap(), + &replacement + )); replacement.empty_factory_failure(); } #[test] fn exhausted_slot_generation_never_wraps_or_selects_old_context() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); let admission = Admission::new(); - { let mut slots = admission.slots.lock().unwrap(); + { + let mut slots = admission.slots.lock().unwrap(); slots[0].generation = usize::MAX >> 8; - for slot in &mut slots[1..] { slot.exhausted = true; } + for slot in &mut slots[1..] { + slot.exhausted = true; + } } assert!(TerminalState::admit_in(&admission, host(), None).is_none()); assert!(admission.slots.lock().unwrap()[0].exhausted); @@ -136,11 +242,16 @@ fn exhausted_slot_generation_never_wraps_or_selects_old_context() { } #[test] fn callback_lookup_holds_exact_state_across_retirement_until_actual_last_drop() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); let admission = Admission::new(); let state = TerminalState::admit_in(&admission, host(), None).unwrap(); let admitted = TerminalState::lookup_in(&admission, state.token).unwrap(); let index = state.lease.index; - state.empty_factory_failure(); drop(state); + state.empty_factory_failure(); + drop(state); assert!(admission.slots.lock().unwrap()[index].reserved); assert_eq!(admitted.lease.index, index); drop(admitted); diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 75013a8d..b65d58f6 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -1,98 +1,221 @@ //! Actual opted-in runtime fixture. Controls are trusted test-only C callbacks. -use std::{ffi::c_void, future::Future, pin::Pin, sync::{Arc, Condvar, Mutex}, task::{Context, Poll}}; -use tinybus::{Connection, Result}; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::{ + ffi::c_void, + future::Future, + pin::Pin, + sync::{Arc, Condvar, Mutex}, + task::{Context, Poll}, +}; +use tinybus::{Connection, Result}; static DROPPED: AtomicUsize = AtomicUsize::new(0); static EARLY: Mutex> = Mutex::new(None); #[derive(Clone, Copy)] -struct NativeQuery { ctx: *mut c_void, query: unsafe extern "C" fn(*mut c_void) -> u32 } +struct NativeQuery { + ctx: *mut c_void, + query: unsafe extern "C" fn(*mut c_void) -> u32, +} unsafe impl Send for NativeQuery {} static NATIVE_QUERY: Mutex> = Mutex::new(None); use tinybus::module::abi::*; use tinybus_module::{NativeInstanceLiveness, RuntimeHooks, TerminalCompletion}; #[derive(serde::Deserialize)] -struct Config { mode: u8, test_controls: Vec } +struct Config { + mode: u8, + test_controls: Vec, +} #[repr(C)] #[derive(Clone, Copy)] -struct Controls { ctx: *mut c_void, event: unsafe extern "C" fn(*mut c_void, u32, u32) } +struct Controls { + ctx: *mut c_void, + event: unsafe extern "C" fn(*mut c_void, u32, u32), +} unsafe impl Send for Controls {} unsafe impl Sync for Controls {} impl Controls { - fn emit(self, event: u32, value: u32) { unsafe { (self.event)(self.ctx, event, value) } } + fn emit(self, event: u32, value: u32) { + unsafe { (self.event)(self.ctx, event, value) } + } } struct Controller { - controls: Mutex>, view: NativeInstanceLiveness, - wake: (Mutex, Condvar), worker: Mutex>>, + controls: Mutex>, + view: NativeInstanceLiveness, + wake: (Mutex, Condvar), + worker: Mutex>>, } impl Controller { fn emit(&self, event: u32, value: u32) { - let controls = *self.controls.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if let Some(controls) = controls { controls.emit(event, value); } + let controls = *self + .controls + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(controls) = controls { + controls.emit(event, value); + } } } impl Drop for Controller { - fn drop(&mut self) { self.emit(6, u32::from(std::thread::current().name() == Some("tinybus-terminal"))); DROPPED.fetch_add(1, Ordering::Release); } + fn drop(&mut self) { + self.emit( + 6, + u32::from(std::thread::current().name() == Some("tinybus-terminal")), + ); + DROPPED.fetch_add(1, Ordering::Release); + } +} +struct SetupFuture { + controller: Arc, + mode: u8, + first: bool, } -struct SetupFuture { controller: Arc, mode: u8, first: bool } impl Future for SetupFuture { type Output = Result<()>; fn poll(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - if self.first { self.first = false; self.controller.emit(2, u32::from(self.mode)); } + if self.first { + self.first = false; + self.controller.emit(2, u32::from(self.mode)); + } match self.mode { - 0 => Poll::Ready(Ok(())), 2 => panic!("controlled native setup poll"), + 0 => Poll::Ready(Ok(())), + 2 => panic!("controlled native setup poll"), 5 => Poll::Ready(Err(tinybus::Error::failed("controlled native setup error"))), _ => Poll::Pending, } } } impl Drop for SetupFuture { - fn drop(&mut self) { self.controller.emit(4, u32::from(std::thread::current().name() == Some("tinybus-terminal"))); } + fn drop(&mut self) { + self.controller.emit( + 4, + u32::from(std::thread::current().name() == Some("tinybus-terminal")), + ); + } } -fn factory(view: NativeInstanceLiveness) -> Result<(impl Fn(Connection, Config) -> Pin> + Send>> + Send + Sync, RuntimeHooks)> { - let early = EARLY.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); +fn factory( + view: NativeInstanceLiveness, +) -> Result<( + impl Fn(Connection, Config) -> Pin> + Send>> + Send + Sync, + RuntimeHooks, +)> { + let early = EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); let held_finish = early.is_some(); - let controller = Arc::new(Controller { controls: Mutex::new(early), view, - wake: (Mutex::new(false), Condvar::new()), worker: Mutex::new(None) }); - if held_finish { controller.view.request_fault(); } + let controller = Arc::new(Controller { + controls: Mutex::new(early), + view, + wake: (Mutex::new(false), Condvar::new()), + worker: Mutex::new(None), + }); + if held_finish { + controller.view.request_fault(); + } let (request, finish) = (controller.clone(), controller.clone()); let setup = move |_: Connection, config: Config| { - if config.test_controls.len() != size_of::() { return Box::pin(async { Err(tinybus::Error::failed("bad test controls")) }) as Pin> + Send>>; } + if config.test_controls.len() != size_of::() { + return Box::pin(async { Err(tinybus::Error::failed("bad test controls")) }) + as Pin> + Send>>; + } // SAFETY: this explicit fixture accepts trusted in-process test pointers. - let controls = unsafe { config.test_controls.as_ptr().cast::().read_unaligned() }; - *controller.controls.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(controls); - let mut worker = controller.worker.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let controls = unsafe { + config + .test_controls + .as_ptr() + .cast::() + .read_unaligned() + }; + *controller + .controls + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(controls); + let mut worker = controller + .worker + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); if worker.is_none() { let work = controller.clone(); - *worker = Some(std::thread::Builder::new().name("native-terminal-controller".into()).spawn(move || { - let mut cancelled = work.wake.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - while !*cancelled { cancelled = work.wake.1.wait(cancelled).unwrap_or_else(std::sync::PoisonError::into_inner); } - drop(cancelled); - let denied = !work.view.is_live(); work.emit(7, u32::from(denied)); denied - }).expect("controlled worker spawn")); + *worker = Some( + std::thread::Builder::new() + .name("native-terminal-controller".into()) + .spawn(move || { + let mut cancelled = work + .wake + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + while !*cancelled { + cancelled = work + .wake + .1 + .wait(cancelled) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + drop(cancelled); + let denied = !work.view.is_live(); + work.emit(7, u32::from(denied)); + denied + }) + .expect("controlled worker spawn"), + ); } drop(worker); - if config.mode == 4 { panic!("controlled native setup construction"); } - Box::pin(SetupFuture { controller: controller.clone(), mode: config.mode, first: true }) as Pin> + Send>> + if config.mode == 4 { + panic!("controlled native setup construction"); + } + Box::pin(SetupFuture { + controller: controller.clone(), + mode: config.mode, + first: true, + }) as Pin> + Send>> }; - let hooks = RuntimeHooks::new(move |_| { - *request.wake.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = true; - request.wake.1.notify_all(); request.emit(3, 1); - }, move || { - if held_finish { finish.emit(8, 0); } - let worker = finish.worker.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); - let outcome = worker.map(|worker| worker.join()); - let joined = outcome.is_none() || matches!(outcome, Some(Ok(true))); - finish.emit(5, u32::from(joined)); - if joined { TerminalCompletion::PhysicallyStopped } else { TerminalCompletion::Uncertain } - }); + let hooks = RuntimeHooks::new( + move |_| { + *request + .wake + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = true; + request.wake.1.notify_all(); + request.emit(3, 1); + }, + move || { + if held_finish { + finish.emit(8, 0); + } + let worker = finish + .worker + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let outcome = worker.map(|worker| worker.join()); + let joined = outcome.is_none() || matches!(outcome, Some(Ok(true))); + finish.emit(5, u32::from(joined)); + if joined { + TerminalCompletion::PhysicallyStopped + } else { + TerminalCompletion::Uncertain + } + }, + ); Ok((setup, hooks)) } #[unsafe(no_mangle)] -pub static TINYBUS_MODULE_ABI_V1: TbAbiDescriptor = TbAbiDescriptor::current("module-terminal", env!("CARGO_PKG_VERSION")); +pub static TINYBUS_MODULE_ABI_V1: TbAbiDescriptor = + TbAbiDescriptor::current("module-terminal", env!("CARGO_PKG_VERSION")); #[unsafe(no_mangle)] pub extern "C" fn tinybus_module_manifest_v1() -> TbSlice { - tinybus_module::manifest_slice(tinybus_module::ManifestDeclaration { name: "module-terminal", version: env!("CARGO_PKG_VERSION"), - provides: &["ai.tinyhumans.TerminalFixture"], methods: &[], signals: &[], requires: &[], optional: &[], lazy: false, worker_threads: 1 }) + tinybus_module::manifest_slice(tinybus_module::ManifestDeclaration { + name: "module-terminal", + version: env!("CARGO_PKG_VERSION"), + provides: &["ai.tinyhumans.TerminalFixture"], + methods: &[], + signals: &[], + requires: &[], + optional: &[], + lazy: false, + worker_threads: 1, + }) } #[derive(Clone, Copy)] struct Table(TbModuleVtable); @@ -101,41 +224,74 @@ static LATEST: Mutex> = Mutex::new(None); /// # Safety /// Standard trusted module init contract plus live test controls in config. #[unsafe(no_mangle)] -pub unsafe extern "C" fn tinybus_module_init_v1(host: *const TbHostVtable, out: *mut TbModuleVtable) -> i32 { +pub unsafe extern "C" fn tinybus_module_init_v1( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { if let Some(host) = unsafe { TbHostVtable::read_compatible(host) } { if let Some(query) = host.native_terminal_state { - *NATIVE_QUERY.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(NativeQuery { ctx: host.host_ctx, query }); + *NATIVE_QUERY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(NativeQuery { + ctx: host.host_ctx, + query, + }); } } - let code = unsafe { tinybus_module::start_terminal_reconfigurable_module_with_factory::(host, out, 1, true, factory) }; - if code == TB_OK { *LATEST.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Table(unsafe { *out })); } + let code = unsafe { + tinybus_module::start_terminal_reconfigurable_module_with_factory::( + host, out, 1, true, factory, + ) + }; + if code == TB_OK { + *LATEST + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Table(unsafe { *out })); + } code } /// Test-only stale-context probe, using the same actual SDK callback. #[unsafe(no_mangle)] pub extern "C" fn terminal_test_context() -> usize { - LATEST.lock().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref().map_or(0, |table| table.0.module_ctx.addr()) + LATEST + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .map_or(0, |table| table.0.module_ctx.addr()) } /// Test-only copied callback probe; the token is never dereferenced. #[unsafe(no_mangle)] pub extern "C" fn terminal_test_old_context(token: usize) -> i32 { - let table = *LATEST.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - table.map_or(TB_CLOSED, |table| unsafe { (table.0.shutdown)(std::ptr::with_exposed_provenance_mut(token), 1000) }) + let table = *LATEST + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + table.map_or(TB_CLOSED, |table| unsafe { + (table.0.shutdown)(std::ptr::with_exposed_provenance_mut(token), 1000) + }) } /// Test-only acquire fence after the final controller event callback returned. #[unsafe(no_mangle)] -pub extern "C" fn terminal_test_drops() -> usize { DROPPED.load(Ordering::Acquire) } +pub extern "C" fn terminal_test_drops() -> usize { + DROPPED.load(Ordering::Acquire) +} /// # Safety /// The test context and callback must remain live until actual controller Drop. #[unsafe(no_mangle)] -pub unsafe extern "C" fn terminal_test_early(ctx: *mut c_void, event: unsafe extern "C" fn(*mut c_void, u32, u32)) { - *EARLY.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Controls { ctx, event }); +pub unsafe extern "C" fn terminal_test_early( + ctx: *mut c_void, + event: unsafe extern "C" fn(*mut c_void, u32, u32), +) { + *EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Controls { ctx, event }); } /// Sample only the actual copied host query; no local or fabricated success bit. #[unsafe(no_mangle)] pub extern "C" fn terminal_test_native_flags() -> u32 { - let query = *NATIVE_QUERY.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let query = *NATIVE_QUERY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); query.map_or(0, |query| unsafe { (query.query)(query.ctx) }) } From 92b9b33cecac9a48a148a2c292ec0bf980e8248d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:37:26 +0300 Subject: [PATCH 054/130] test(terminal): cover cleanup panic retaining later resource owners Add a test asserting that when the first cleanup drop panics, the terminal reports an uncertain completion and leaves the remaining foreign resource owners attached rather than dropping them. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/terminal_tests.rs | 34 +++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/tinybus-module/src/terminal_tests.rs b/crates/tinybus-module/src/terminal_tests.rs index c04a450b..331c8e34 100644 --- a/crates/tinybus-module/src/terminal_tests.rs +++ b/crates/tinybus-module/src/terminal_tests.rs @@ -257,3 +257,37 @@ fn callback_lookup_holds_exact_state_across_retirement_until_actual_last_drop() drop(admitted); assert!(!admission.slots.lock().unwrap()[index].reserved); } + +#[test] +fn first_cleanup_drop_panic_retains_all_unprocessed_resource_owners() { + let _serial = TEST_LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + struct FirstDrop; + impl Drop for FirstDrop { + fn drop(&mut self) { panic!("controlled first cleanup Drop"); } + } + struct RemainingDrop(Arc); + impl Drop for RemainingDrop { + fn drop(&mut self) { self.0.fetch_add(1, Ordering::SeqCst); } + } + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let remaining = Arc::new(AtomicUsize::new(0)); + { + let mut resources = state.resources.lock().unwrap(); + let first = FirstDrop; + resources.setup = Some(Box::new(move |_| { + drop(first); + Box::pin(async { Ok(()) }) + })); + resources.retained_factory = Some(Box::new(RemainingDrop(remaining.clone()))); + } + state.register(RuntimeHooks::new(|_| {}, || TerminalCompletion::PhysicallyStopped)); + state.publish(); + state.signal(TerminalReason::Stop); + let completion = state.wait(Duration::from_secs(3)); + let dropped = remaining.load(Ordering::SeqCst); + assert_eq!(completion, TerminalCompletion::Uncertain); + assert_eq!(dropped, 0, "first cleanup panic must leave later foreign owners attached"); + assert!(state.resources.lock().unwrap().retained_factory.is_some()); +} From 858a9f26691bef9a683190e9e0efe454ee2aaeeb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:38:15 +0300 Subject: [PATCH 055/130] fix(terminal): release resource owners one at a time during cleanup Cleanup now takes and drops each resource owner individually instead of draining them all into a single tuple, so a panic in one owner's foreign Drop no longer skips the remaining owners. The reinitialize callback is also fenced before the owner stops and taken only after the future is dropped, keeping it anchored until the synchronous callback is admitted. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/terminal.rs | 74 +++++++++++++++------ crates/tinybus-module/src/terminal_tests.rs | 22 ++++-- 2 files changed, 72 insertions(+), 24 deletions(-) diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 93019adb..10da366e 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -641,10 +641,13 @@ impl TerminalState { } } } - let callback = reinitialize - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .take(); + // Fence the admitted synchronous callback without taking its foreign + // captures yet. Terminal admission denies a waiter after this fence. + drop( + reinitialize + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ); if let Some(owner) = &self.owner { owner.stop(); } @@ -653,6 +656,10 @@ impl TerminalState { .unwrap_or_else(std::sync::PoisonError::into_inner) .take(); drop(future); + let callback = reinitialize + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); drop(callback); let request = self.lock().request.take(); drop(request); @@ -664,25 +671,54 @@ impl TerminalState { true } fn release_resources(&self) -> bool { - let (setup, factory, config, reinit_factory, connection, inbound) = { - let mut resources = self - .resources - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - ( - resources.setup.take(), - resources.retained_factory.take(), - resources.config.take(), - resources.reinitializer_factory.take(), - resources.connection.clone(), - resources.inbound.take(), - ) - }; + // Take one owner at a time. If its foreign Drop panics, the enclosing + // single cleanup catch stops here and every later owner stays anchored. + let setup = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .setup + .take(); + drop(setup); + let factory = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retained_factory + .take(); + drop(factory); + let config = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config + .take(); + drop(config); + let reinit_factory = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reinitializer_factory + .take(); + drop(reinit_factory); + let connection = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .connection + .clone(); let connection = connection .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .take(); - drop((setup, factory, config, reinit_factory, connection, inbound)); + drop(connection); + let inbound = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .inbound + .take(); + drop(inbound); let runtime = self .resources .lock() diff --git a/crates/tinybus-module/src/terminal_tests.rs b/crates/tinybus-module/src/terminal_tests.rs index 331c8e34..87c0448a 100644 --- a/crates/tinybus-module/src/terminal_tests.rs +++ b/crates/tinybus-module/src/terminal_tests.rs @@ -260,15 +260,21 @@ fn callback_lookup_holds_exact_state_across_retirement_until_actual_last_drop() #[test] fn first_cleanup_drop_panic_retains_all_unprocessed_resource_owners() { - let _serial = TEST_LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); struct FirstDrop; impl Drop for FirstDrop { - fn drop(&mut self) { panic!("controlled first cleanup Drop"); } + fn drop(&mut self) { + panic!("controlled first cleanup Drop"); + } } struct RemainingDrop(Arc); impl Drop for RemainingDrop { - fn drop(&mut self) { self.0.fetch_add(1, Ordering::SeqCst); } + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } } let admission = Admission::new(); let state = TerminalState::admit_in(&admission, host(), None).unwrap(); @@ -282,12 +288,18 @@ fn first_cleanup_drop_panic_retains_all_unprocessed_resource_owners() { })); resources.retained_factory = Some(Box::new(RemainingDrop(remaining.clone()))); } - state.register(RuntimeHooks::new(|_| {}, || TerminalCompletion::PhysicallyStopped)); + state.register(RuntimeHooks::new( + |_| {}, + || TerminalCompletion::PhysicallyStopped, + )); state.publish(); state.signal(TerminalReason::Stop); let completion = state.wait(Duration::from_secs(3)); let dropped = remaining.load(Ordering::SeqCst); assert_eq!(completion, TerminalCompletion::Uncertain); - assert_eq!(dropped, 0, "first cleanup panic must leave later foreign owners attached"); + assert_eq!( + dropped, 0, + "first cleanup panic must leave later foreign owners attached" + ); assert!(state.resources.lock().unwrap().retained_factory.is_some()); } From 30736626b9edb9772545bb929dc772fe0c6c3578 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:39:38 +0300 Subject: [PATCH 056/130] chore: files changed docs/modules/module/README.md,docs/modules/module/abi.md Auto-committed-on: dragonfly Co-authored-by: Medulla --- docs/modules/module/README.md | 50 +++++++++++++++++++++++++++++++++++ docs/modules/module/abi.md | 28 +++++++++++++++++--- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/docs/modules/module/README.md b/docs/modules/module/README.md index 164cd494..32cfe6de 100644 --- a/docs/modules/module/README.md +++ b/docs/modules/module/README.md @@ -44,6 +44,56 @@ last initialized instance. Linked modules keep the embedding host's tracing and panic hook. The existing instance-wide last panic location is diagnostic state, not a proof of exact attribution among concurrently panicking methods. +## Opted-in terminal factories + +Configured modules can opt in with `module_export!(terminal_setup_factory = +factory, config = Config, ...)`; the static export macro accepts the same +arguments. The factory receives one `NativeInstanceLiveness` and returns its +setup callback plus `RuntimeHooks`. An unsupported or non-live native query +rejects initialization before the factory runs. Linked modules require the +same explicit host query and retain the embedding process's panic hook. + +A retained liveness view refers to that exact instance. Only a fresh, exactly +Live native query admits work; any observed terminal intent permanently closes +the view. `request_fault()` first reports genuine native fault. The view grants +no publication lease and does not turn local cancellation into native success. + +The request hook runs at most once and must only close controller admission and +perform bounded cancel/wake. It must not join, do I/O, wait for a gate, or reenter +the SDK. Finish runs independently at most once on the physical cleanup thread, +after startup ownership is published, request has returned, the sole initial +setup task has settled, and any admitted reinitializer has returned. Finish must +fence and join the controller's actual workers. `PhysicallyStopped` asserts that +work; a timeout or host shutdown return does not prove it. + +The SDK retains setup futures across cancellation, destroys them on the +physical cleanup thread, and drops callback captures outside resource locks. +Reinitialization polls its result before terminal cancellation when both are +ready. The runtime is destroyed on the physical thread. Surviving SDK resources, +spawn failure, a hook panic, or unproved cleanup keep the owner uncertain and +anchored. Seven separate primary panic cells retain request, finish, initial +setup, reinitialize, factory, startup, and cleanup payloads. Native fault and +request completion precede any payload disposal. After the first cleanup +destructor panic, later resource owners remain attached rather than being +destroyed during cleanup unwinding. + +Each SDK image has 256 instance slots. The existing opaque `module_ctx` contains +a slot/generation address token, never a dereferenced pointer or extra ABI +allocation. Callback lookup acquires the exact State before releasing registry +exclusion. Successful teardown retires its registry anchor; only actual State +last-Drop releases the slot, so old views and surviving SDK resources count +against the limit. Generations never wrap: exhaustion permanently denies that +slot. Stale or unknown tokens return Closed and cannot select a replacement. +Tokens are private bookkeeping, not native authentication. + +This metadata bound does not bound existing host-context or mapped-library +retention. Libraries remain mapped until process exit, including a rejected +initialization whose independent SDK finish job is still running. A rejected +initialization without an accepted vtable cannot earn host PhysicalStopped +evidence. The native query remains authoritative after local Stop and before +any successful shutdown return or context retirement. Legacy exports retain +their existing lifecycle behavior. + ## Loading sequence 1. Check every directory component's ownership/mode, require a regular diff --git a/docs/modules/module/abi.md b/docs/modules/module/abi.md index 11927a5f..ca8d5ad1 100644 --- a/docs/modules/module/abi.md +++ b/docs/modules/module/abi.md @@ -18,7 +18,7 @@ support. The host initializes the output vtable's leading `size` field with its allocation capacity before calling init. A newer module reads only that frozen field first and writes no more than the advertised capacity, so the additive tail is safe in both compatibility directions. -The SDK bounds reinitialization setup to four seconds and returns `TB_TIMEOUT` +The legacy SDK bounds reinitialization setup to four seconds and returns `TB_TIMEOUT` after cancelling the setup future. The host serializes this callback with shutdown and retains the lifecycle lock even if its own outer deadline expires. The final host callback, `ready`, marks completion of asynchronous setup. This @@ -54,9 +54,29 @@ including independent workers after runtime shutdown or host destruction. Host destruction must leave a terminal tombstone; this context must never be recycled or reopened. Custom unsafe C hosts must truthfully uphold this contract, which cannot cryptographically authenticate an in-process host. -This tail defines an optional ABI capability; ordinary SDK hosts currently -supply `None`. It introduces no retain/release ABI or aggregate resource bound: -the existing process-lifetime host-context and library retention is unbounded. +This tail defines an optional ABI capability. Unsupported hosts supply `None`; +terminal factories reject them before factory evaluation. It introduces no +retain/release ABI or aggregate resource bound: the existing process-lifetime +host-context and library retention is unbounded. + +Opted-in SDK instances use the existing opaque `module_ctx` field for a private +slot/generation address token. No callback dereferences it and no ABI field or +context allocation is added. Each SDK image has 256 slots; callback lookup +clones the exact matching State under registry exclusion, then releases that +lock before instance operations. Successful resource teardown may retire the +anchor, but slot reuse waits for actual State last-Drop, including retained +liveness views and SDK resources. Generation exhaustion permanently denies the +slot instead of wrapping. Unknown or retired tokens return Closed; an old +instance token cannot act on a replacement. These tokens are not serialized or +logged and do not authenticate a native host. + +Terminal reinitialization uses result-first polling when completion and terminal +intent are simultaneously ready. Cancellation leaves its foreign future owned +by the instance until physical cleanup. Its runtime and terminal captures are +destroyed on the independent physical cleanup thread. Uncertainty remains +sticky and cannot be refunded by a timeout or shutdown return. Genuine native +Fault is sampled even after local Stop, and faulted published instances retain +their context anchor instead of earning successful retirement. ## Symbols From 09a4a1258d404f29855c7baa6868f66936de90a4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:52:33 +0300 Subject: [PATCH 057/130] test(module): cover connection teardown after runtime timeout Add a test that serves a real object whose Drop blocks, then shuts the host down past the SDK's shutdown timeout and checks that physical completion is denied and the terminal stays anchored. This exercises the case where a blocked whole-Connection teardown cannot refund admission, which the existing synthetic transport owner could not reach. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index fad3446f..272f09ba 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -703,3 +703,62 @@ async fn actual_linked_reinit_terminal_cancellation_settles_before_worker_join() assert_eq!(drops.len(), 2); assert!(drops.into_iter().all(|index| index < finish)); } + +// A real served object, rather than a synthetic transport owner: Inner destroys +// its transport before its ObjectTree, so the old count can reach zero here. +struct TeardownObject { + entered: std::sync::mpsc::Sender<()>, + release: Arc, + settled: std::sync::mpsc::Sender<()>, + panic: bool, +} +#[async_trait] +impl tinybus::Interface for TeardownObject { + fn name(&self) -> tinybus::InterfaceName { + "ai.tinyhumans.tinybus.Teardown".parse().unwrap() + } + fn members(&self) -> Vec { Vec::new() } + async fn call(&self, _: &tinybus::MemberName, _: serde_json::Value) -> Result { + Ok(serde_json::Value::Null) + } +} +impl Drop for TeardownObject { + fn drop(&mut self) { + self.entered.send(()).unwrap(); + self.release.wait(); + self.settled.send(()).unwrap(); + if self.panic { panic!("controlled ObjectTree teardown"); } + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_connection_object_drop_after_runtime_timeout_denies_physical_completion() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (settled_tx, settled_rx) = std::sync::mpsc::channel(); + let release = Arc::new(std::sync::Barrier::new(2)); + let connection = view.inner.resources.lock().unwrap().connection.lock().unwrap().clone().unwrap(); + connection.serve_at("/teardown".parse().unwrap(), TeardownObject { + entered: entered_tx, release: release.clone(), settled: settled_tx, panic: false, + }).await.unwrap(); + drop(connection); + let stopping_host = host.clone(); + let stopping = tokio::spawn(async move { stopping_host.shutdown(Duration::from_secs(8)).await; }); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + stopping.await.unwrap(); // The SDK's real shutdown_timeout has elapsed. + let completion = view.inner.wait(Duration::ZERO); + let flags = unsafe { (view.inner.host.0.native_terminal_state.unwrap())(view.inner.host.0.host_ctx) }; + let still_anchored = terminal::TerminalState::lookup(view.inner.context()).is_some(); + // Release and settle actual foreign work before reporting a failed proof. + release.wait(); + settled_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert_eq!(completion, TerminalCompletion::Uncertain); + assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); + assert!(still_anchored, "blocked whole-Connection teardown cannot refund admission"); +} From fdbeda185578303aa0348b8d1d697b04c34e1f91 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:53:24 +0300 Subject: [PATCH 058/130] test(module): cover panicking connection teardown fault fencing Add a terminal test asserting that a panicking Connection destructor during shutdown leaves the terminal uncertain and anchored while still setting the fault flag, complementing the existing owner-suppression case. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 272f09ba..1a2bb996 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -762,3 +762,37 @@ async fn actual_connection_object_drop_after_runtime_timeout_denies_physical_com assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); assert!(still_anchored, "blocked whole-Connection teardown cannot refund admission"); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_connection_object_drop_panic_fences_fault_after_owner_suppression() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (settled_tx, settled_rx) = std::sync::mpsc::channel(); + let release = Arc::new(std::sync::Barrier::new(2)); + let connection = view.inner.resources.lock().unwrap().connection.lock().unwrap().clone().unwrap(); + connection.serve_at("/teardown".parse().unwrap(), TeardownObject { + entered: entered_tx, release: release.clone(), settled: settled_tx, panic: true, + }).await.unwrap(); + drop(connection); + let stopping_host = host.clone(); + let stopping = tokio::spawn(async move { stopping_host.shutdown(Duration::from_secs(8)).await; }); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + release.wait(); + settled_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + stopping.await.unwrap(); // Runtime teardown settles the real panicking destructor. + let completion = view.inner.wait(Duration::ZERO); + let flags = unsafe { (view.inner.host.0.native_terminal_state.unwrap())(view.inner.host.0.host_ctx) }; + let still_anchored = terminal::TerminalState::lookup(view.inner.context()).is_some(); + + assert_eq!(completion, TerminalCompletion::Uncertain); + assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); + assert!(still_anchored, "panicking Connection teardown cannot refund admission"); + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); +} From 02d1f98d0c1b78a1f602ccdac0898bcce104e514 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:54:32 +0300 Subject: [PATCH 059/130] fix(module): report panics caught during sdk teardown The catch observer now routes through the SDK owner so a panic caught while unwinding marks the terminal state uncertain and signals a fault, instead of relying on a weak reference that may already be gone. The owner's drop also treats an in-flight panic as a fault, since a destructor unwinding is not clean disposal, and the connection's catch observer is moved last so it retains teardown ownership of the whole connection. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 7 ++----- crates/tinybus-module/src/terminal.rs | 15 ++++++++++++++- crates/tinybus/src/connection.rs | 4 +++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 0dd3dfe4..ef05d1bb 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1678,12 +1678,9 @@ where name: MODULE_PANIC_ERROR.to_owned(), message: "a module method panicked".to_owned(), })); - let observer_state = Arc::downgrade(&task_state); + let observer_owner = task_state.sdk_owner(); connection.__set_catch_observer(Arc::new(move || { - if let Some(state) = observer_state.upgrade() { - state.mark_uncertain(); - state.signal(TerminalReason::Fault); - } + observer_owner.caught_panic(); })); // Connection is persisted independently of initial task lifetime. *connection_slot diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 10da366e..e256723c 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -221,8 +221,21 @@ pub(super) struct TerminalState { pub(super) struct SdkOwner { state: Arc, } +impl SdkOwner { + pub fn caught_panic(&self) { + self.state.mark_uncertain(); + self.state.signal(TerminalReason::Fault); + } +} impl Drop for SdkOwner { fn drop(&mut self) { + // A last-field guard also runs when a preceding foreign destructor + // unwinds. Counting that as clean disposal would fabricate completion. + // The diagnostic owner may already be stopped; explicitly fence native + // authority and preserve the anchor before releasing the resource count. + if std::thread::panicking() { + self.caught_panic(); + } self.state.sdk_owners.fetch_sub(1, Ordering::AcqRel); } } @@ -728,7 +741,7 @@ impl TerminalState { if let Some(runtime) = runtime { runtime.shutdown_timeout(Duration::from_secs(4)); } - // Timeout is not proof: any surviving SDK transport/task keeps State + // Timeout is not proof: every surviving SDK transport/Connection/method keeps State // and its resource count. Unknown surviving ownership stays anchored. self.sdk_owners.load(Ordering::Acquire) == 0 } diff --git a/crates/tinybus/src/connection.rs b/crates/tinybus/src/connection.rs index 65238500..3c57b82a 100644 --- a/crates/tinybus/src/connection.rs +++ b/crates/tinybus/src/connection.rs @@ -192,11 +192,13 @@ struct Inner { signal_deliveries: broadcast::Sender, local_publisher: crate::LocalPublisherIdentity, panic_handler: std::sync::RwLock Error + Send + Sync>>>, - catch_observer: std::sync::RwLock>>, /// Bulk streams being received. On the connection rather than in the object /// tree because a chunk has to be checked against the header's stamped /// `sender`; interfaces receive only the verified subset in CallContext. streams: StreamRegistry, + // Last: opted-in observers retain whole-Connection teardown ownership, + // including ObjectTree, streams and cloned method observers. + catch_observer: std::sync::RwLock>>, } /// Closes the transport when the last [`Connection`] handle goes away. From bcd1bb526898e1fc7ba8dada62e54e28dba08478 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:57:57 +0300 Subject: [PATCH 060/130] test(module): cover reinit future disposal and fault paths off the lock Add two terminal tests that assert repeated reinitialization disposes prior futures on the same physical worker without holding the reinitializer mutex, and that source error and panic requests during reinit also run outside the callback exclusion. The fixture now records whether the drop and request hooks observed the mutex already held so the tests can verify the lock is not taken on those paths. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 1a2bb996..c2411ae9 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -131,6 +131,8 @@ mod controlled { worker: StdMutex>>, pub requests: AtomicUsize, pub joins: AtomicUsize, + pub drop_off_lock: AtomicBool, + pub request_off_lock: AtomicBool, } static NEXT: StdMutex>> = StdMutex::new(None); pub static TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); @@ -143,6 +145,8 @@ mod controlled { worker: StdMutex::new(None), requests: AtomicUsize::new(0), joins: AtomicUsize::new(0), + drop_off_lock: AtomicBool::new(true), + request_off_lock: AtomicBool::new(true), }); *NEXT.lock().unwrap() = Some(fixture.clone()); (fixture, receiver) @@ -180,6 +184,12 @@ mod controlled { } impl Drop for SetupFuture { fn drop(&mut self) { + if let Some(view) = self.fixture.view.lock().unwrap().as_ref() { + let slot = view.inner.resources.lock().unwrap().reinitialize.clone(); + if slot.try_lock().is_err() { + self.fixture.drop_off_lock.store(false, Ordering::Release); + } + } let _ = self.fixture.events.send(Event::FutureDrop( std::thread::current() .name() @@ -230,6 +240,12 @@ mod controlled { }; let hooks = RuntimeHooks::new( move |_| { + if let Some(view) = request.view.lock().unwrap().as_ref() { + let slot = view.inner.resources.lock().unwrap().reinitialize.clone(); + if slot.try_lock().is_err() { + request.request_off_lock.store(false, Ordering::Release); + } + } request.requests.fetch_add(1, Ordering::SeqCst); *request.wake.0.lock().unwrap() = true; request.wake.1.notify_all(); @@ -796,3 +812,53 @@ async fn actual_connection_object_drop_panic_fences_fault_after_owner_suppressio assert!(still_anchored, "panicking Connection teardown cannot refund admission"); assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn successful_repeated_reinit_disposes_prior_future_on_same_physical_worker_off_lock() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { + tokio::task::yield_now().await; + } + }).await.unwrap(); + let first = host.reinitialize("tinybus-module", serde_json::json!({"mode":0})).await; + let second = host.reinitialize("tinybus-module", serde_json::json!({"mode":0})).await; + host.shutdown(Duration::from_secs(8)).await; + let drops: Vec<_> = events.try_iter().filter_map(|event| { + if let controlled::Event::FutureDrop(name) = event { Some(name) } else { None } + }).collect(); + assert!(first.is_ok() && second.is_ok()); + assert_eq!(drops.len(), 3); + assert!(drops.iter().all(|name| name == "tinybus-terminal"), "all completed futures must be destroyed by the sole physical worker: {drops:?}"); + assert!(fixture.drop_off_lock.load(Ordering::Acquire)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_reinit_source_error_and_panic_request_outside_callback_exclusion() { + let _serial = controlled::TEST_LOCK.lock().await; + for mode in [3, 2, 4] { + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { + tokio::task::yield_now().await; + } + }).await.unwrap(); + let result = host.reinitialize("tinybus-module", serde_json::json!({"mode":mode})).await; + host.shutdown(Duration::from_secs(8)).await; + assert!(result.is_err()); + assert_eq!(fixture.requests.load(Ordering::SeqCst), 1); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + assert!(fixture.request_off_lock.load(Ordering::Acquire), "source fault request held reinitializer mutex in mode {mode}"); + } +} From 2224bb73dc7fa49aec2bba332f52692b45884338 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 17:59:52 +0300 Subject: [PATCH 061/130] fix(module): serialize terminal reinitialization with setup retirement Reinitialization now takes an exclusive slot so a second caller cannot start while one is in flight, and the previous setup future is retired through the worker before the next one is built. This prevents the reinitializer from racing the finish worker over the same future and leaves the authoritative Arc state-owned until retirement settles. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 36 +++++++---- crates/tinybus-module/src/terminal.rs | 86 ++++++++++++++++++++++----- 2 files changed, 93 insertions(+), 29 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index ef05d1bb..840e9b52 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1383,28 +1383,34 @@ unsafe extern "C" fn terminal_reinitialize(ctx: *mut c_void, ptr: *const u8, len return TB_CLOSED; } let _scope = state.owner.as_ref().map(ExecutionOwner::enter); - let callback = state + if !state.begin_reinit() { + return TB_CLOSED; + } + let callback_slot = state .resources .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .reinitialize .clone(); - let callback = callback + // The authoritative Arc remains State-owned until active_reinit settles. + // No callback exclusion is held across foreign invocation or source faults. + let callback = callback_slot .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if !state.is_live() { - return TB_CLOSED; - } - let Some(callback) = callback.as_ref() else { - return TB_CLOSED; - }; - match catch_unwind(AssertUnwindSafe(|| callback(ptr, len))) { + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let outcome = catch_unwind(AssertUnwindSafe(|| { + callback.as_ref().map_or(TB_CLOSED, |callback| callback(ptr, len)) + })); + drop(callback); // non-last: cleanup still waits active_reinit + let code = match outcome { Ok(code) => code, Err(payload) => { state.source_panic(terminal::PayloadKind::Reinit, payload); TB_PANICKED } - } + }; + state.end_reinit(); + code } // The task never owns the foreign future across a yield or on cancellation. @@ -1577,7 +1583,9 @@ where .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .take(); - drop(previous); + if !state.retire_reinit(previous) { + return TB_CLOSED; + } if !state.is_live() { return TB_CLOSED; } @@ -1650,7 +1658,9 @@ where resources.reinitializer_factory.take(), ) }; - let callback = factory.map(|factory| factory(handle.clone(), connection_slot.clone())); + let callback = factory.map(|factory| { + terminal::TerminalReinitialize::from(factory(handle.clone(), connection_slot.clone())) + }); let callback_slot = state .resources .lock() diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index e256723c..2521553e 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -164,11 +164,16 @@ struct Control { completion: Option, job: Option>, payloads: PrimaryPayloads, + active_reinit: bool, + prior_future: Option, + prior_pending: bool, + retirement_failed: bool, } pub(super) type Setup = Box Pin> + Send>> + Send>; -pub(super) type SetupFutureSlot = - Arc> + Send>>>>>; +pub(super) type SetupFuture = Pin> + Send>>; +pub(super) type SetupFutureSlot = Arc>>; +pub(super) type TerminalReinitialize = Arc i32 + Send + Sync>; /// Resources are attached here before the next fallible startup boundary. /// Taking values is mechanical; all foreign destruction occurs off this lock. @@ -178,7 +183,7 @@ pub(super) struct StartupResources { pub retained_factory: Option>, pub config: Option>, pub reinitializer_factory: Option, - pub reinitialize: Arc>>, + pub reinitialize: Arc>>, pub connection: Arc>>, pub inbound: Option>, pub initial: Option>>, @@ -287,6 +292,10 @@ impl TerminalState { completion: None, job: None, payloads: PrimaryPayloads::default(), + active_reinit: false, + prior_future: None, + prior_pending: false, + retirement_failed: false, }), changed: Condvar::new(), terminal_notify: tokio::sync::Notify::new(), @@ -497,6 +506,35 @@ impl TerminalState { self.schedule(); } } + pub fn begin_reinit(&self) -> bool { + let mut control = self.lock(); + if self.is_terminal() || control.active_reinit { + return false; + } + control.active_reinit = true; + true + } + pub fn end_reinit(&self) { + self.lock().active_reinit = false; + self.changed.notify_all(); + } + pub fn retire_reinit(self: &Arc, future: Option) -> bool { + let Some(future) = future else { return true; }; + { + let mut control = self.lock(); + // active_reinit admits one caller. It cannot construct the next + // future until the worker has acknowledged this exact disposal. + control.prior_future = Some(future); + control.prior_pending = true; + } + self.schedule(); + self.changed.notify_all(); + let mut control = self.lock(); + while control.prior_pending && control.finish_state != FinishState::SpawnFailed { + control = self.changed.wait(control).unwrap_or_else(std::sync::PoisonError::into_inner); + } + !control.retirement_failed && control.finish_state != FinishState::SpawnFailed + } fn schedule(self: &Arc) { let job = { let mut control = self.lock(); @@ -522,6 +560,9 @@ impl TerminalState { control.completion = Some(TerminalCompletion::Uncertain); drop(control); self.changed.notify_all(); + self.host.fault(); + self.commit(TerminalReason::Fault); + self.run_request(); } } fn remove_anchor(&self) { @@ -543,21 +584,34 @@ impl TerminalState { } fn run_finish(self: &Arc) { let _scope = self.owner.as_ref().map(ExecutionOwner::enter); - let finish = { + let finish = loop { let mut control = self.lock(); - while !control.published - || matches!( - control.request_state, - RequestState::Idle | RequestState::Running - ) - { - control = self - .changed - .wait(control) - .unwrap_or_else(std::sync::PoisonError::into_inner); + // Retirement takes precedence over waiting for active_reinit: + // that caller may itself be awaiting this worker's acknowledgment. + if let Some(future) = control.prior_future.take() { + drop(control); + let outcome = catch_unwind(AssertUnwindSafe(|| drop(future))); + let failed = match outcome { + Ok(()) => false, + Err(payload) => { + self.source_panic(PayloadKind::Cleanup, payload); + true + } + }; + let mut control = self.lock(); + control.retirement_failed |= failed; + control.prior_pending = false; + drop(control); + self.changed.notify_all(); + continue; + } + if control.published && !control.active_reinit && !matches!( + control.request_state, RequestState::Idle | RequestState::Running + ) { + control.finish_state = FinishState::Running; + break control.finish.take(); } - control.finish_state = FinishState::Running; - control.finish.take() + drop(self.changed.wait(control).unwrap_or_else(std::sync::PoisonError::into_inner)); }; let cleanup = catch_unwind(AssertUnwindSafe(|| self.settle_setup())); let setup_settled = match cleanup { From 022ab8b4eefc51ee59c0e8a4a9ea7d48c9732c55 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:01:26 +0300 Subject: [PATCH 062/130] test(tinybus-module): drain events before asserting reinit failure Wait for the controlled event stream to reach its Finish marker before checking the reinitialize result, so the assertions run against a fully settled host instead of racing the shutdown sequence. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index c2411ae9..9ec9e58b 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -856,6 +856,9 @@ async fn actual_reinit_source_error_and_panic_request_outside_callback_exclusion }).await.unwrap(); let result = host.reinitialize("tinybus-module", serde_json::json!({"mode":mode})).await; host.shutdown(Duration::from_secs(8)).await; + loop { + if matches!(events.recv_timeout(Duration::from_secs(5)).unwrap(), controlled::Event::Finish) { break; } + } assert!(result.is_err()); assert_eq!(fixture.requests.load(Ordering::SeqCst), 1); assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); From 2d4667cf6cc1be50c85c1aa1ca514f85313c3d87 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:03:09 +0300 Subject: [PATCH 063/130] refactor(terminal): split execution settlement from setup disposal Separate the worker's execution settlement from the destruction of foreign captures so a failed prior retirement no longer spends the controller's still-uninvoked physical finish callback. Settlement now reports whether execution completed, and setup disposal only runs when execution settled and retirement had not failed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/terminal.rs | 51 +++++++++++++++------------ 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 2521553e..dd50b2bc 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -584,7 +584,7 @@ impl TerminalState { } fn run_finish(self: &Arc) { let _scope = self.owner.as_ref().map(ExecutionOwner::enter); - let finish = loop { + loop { let mut control = self.lock(); // Retirement takes precedence over waiting for active_reinit: // that caller may itself be awaiting this worker's acknowledgment. @@ -609,18 +609,27 @@ impl TerminalState { control.request_state, RequestState::Idle | RequestState::Running ) { control.finish_state = FinishState::Running; - break control.finish.take(); + break; } drop(self.changed.wait(control).unwrap_or_else(std::sync::PoisonError::into_inner)); }; - let cleanup = catch_unwind(AssertUnwindSafe(|| self.settle_setup())); - let setup_settled = match cleanup { - Ok(settled) => settled, - Err(payload) => { - self.source_panic(PayloadKind::Cleanup, payload); - false + // Execution settlement is distinct from foreign capture destruction. + // A failed prior retirement retains all later captures, but does not + // spend the controller's still-uninvoked physical finish callback. + let execution_settled = self.settle_execution(); + let retirement_failed = self.lock().retirement_failed; + let setup_settled = if execution_settled && !retirement_failed { + match catch_unwind(AssertUnwindSafe(|| self.dispose_setup())) { + Ok(()) => true, + Err(payload) => { + self.source_panic(PayloadKind::Cleanup, payload); + false + } } + } else { + execution_settled }; + let finish = self.lock().finish.take(); let completed = if setup_settled { if let Some(owner) = &self.owner { owner.stop(); @@ -639,7 +648,7 @@ impl TerminalState { self.lock().finish = finish; TerminalCompletion::Uncertain }; - let released = if completed == TerminalCompletion::PhysicallyStopped { + let released = if completed == TerminalCompletion::PhysicallyStopped && !retirement_failed { match catch_unwind(AssertUnwindSafe(|| self.release_resources())) { Ok(released) => released, Err(payload) => { @@ -669,8 +678,8 @@ impl TerminalState { self.remove_anchor(); } } - fn settle_setup(self: &Arc) -> bool { - let (initial, handle, reinitialize, future, reinit_future) = { + fn settle_execution(self: &Arc) -> bool { + let (initial, handle) = { let mut resources = self .resources .lock() @@ -681,9 +690,6 @@ impl TerminalState { .runtime .as_ref() .map(|runtime| runtime.handle().clone()), - resources.reinitialize.clone(), - resources.initial_future.clone(), - resources.reinit_future.clone(), ) }; if let Some(initial) = initial { @@ -708,13 +714,15 @@ impl TerminalState { } } } - // Fence the admitted synchronous callback without taking its foreign - // captures yet. Terminal admission denies a waiter after this fence. - drop( - reinitialize - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner), - ); + true + } + fn dispose_setup(&self) { + // The worker already observed active_reinit=false. Every last capture + // is taken mechanically and destroyed after all bookkeeping unlocks. + let (reinitialize, future, reinit_future) = { + let resources = self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + (resources.reinitialize.clone(), resources.initial_future.clone(), resources.reinit_future.clone()) + }; if let Some(owner) = &self.owner { owner.stop(); } @@ -735,7 +743,6 @@ impl TerminalState { .unwrap_or_else(std::sync::PoisonError::into_inner) .take(); drop(future); - true } fn release_resources(&self) -> bool { // Take one owner at a time. If its foreign Drop panics, the enclosing From 1d402493b94873579f486d5b7f485d37d571d87b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:03:31 +0300 Subject: [PATCH 064/130] test(module): cover prior reinit future drop during reinitialization Add controlled tests that block the drop of a prior reinitialization future and verify the next setup does not start until it completes, with a stop settling the same job. A panicking drop is also covered to confirm resources stay anchored while finish is attempted once. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 9ec9e58b..408d79be 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -133,6 +133,7 @@ mod controlled { pub joins: AtomicUsize, pub drop_off_lock: AtomicBool, pub request_off_lock: AtomicBool, + pub future_drop_control: StdMutex, bool)>>, } static NEXT: StdMutex>> = StdMutex::new(None); pub static TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); @@ -147,6 +148,7 @@ mod controlled { joins: AtomicUsize::new(0), drop_off_lock: AtomicBool::new(true), request_off_lock: AtomicBool::new(true), + future_drop_control: StdMutex::new(None), }); *NEXT.lock().unwrap() = Some(fixture.clone()); (fixture, receiver) @@ -196,6 +198,11 @@ mod controlled { .unwrap_or("unnamed") .to_owned(), )); + let control = self.fixture.future_drop_control.lock().unwrap().take(); + if let Some((release, panic)) = control { + release.wait(); + if panic { panic!("controlled prior reinit future Drop"); } + } } } fn factory( @@ -865,3 +872,57 @@ async fn actual_reinit_source_error_and_panic_request_outside_callback_exclusion assert!(fixture.request_off_lock.load(Ordering::Acquire), "source fault request held reinitializer mutex in mode {mode}"); } } + +async fn controlled_prior_reinit_drop(panic: bool) { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { + tokio::task::yield_now().await; + } + }).await.unwrap(); + host.reinitialize("tinybus-module", serde_json::json!({"mode":0})).await.unwrap(); + let _ = events.try_iter().count(); + let release = Arc::new(std::sync::Barrier::new(2)); + *fixture.future_drop_control.lock().unwrap() = Some((release.clone(), panic)); + let caller_host = host.clone(); + let caller = tokio::spawn(async move { caller_host.reinitialize("tinybus-module", serde_json::json!({"mode":0})).await }); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().as_ref().unwrap().clone(); + view.inner.signal(TerminalReason::Stop); + let not_finished_while_blocked = fixture.joins.load(Ordering::SeqCst) == 0; + let no_next_setup = !events.try_iter().any(|event| matches!(event, controlled::Event::Poll(_))); + release.wait(); + let result = caller.await.unwrap(); + loop { + if matches!(events.recv_timeout(Duration::from_secs(5)).unwrap(), controlled::Event::Finish) { break; } + } + host.shutdown(Duration::from_secs(8)).await; + let completion = view.inner.wait(Duration::from_secs(5)); + assert!(matches!(entered, controlled::Event::FutureDrop(name) if name == "tinybus-terminal")); + assert!(not_finished_while_blocked && no_next_setup); + assert!(result.is_err()); + assert_eq!(fixture.requests.load(Ordering::SeqCst), 1); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + assert!(fixture.drop_off_lock.load(Ordering::Acquire)); + assert!(fixture.request_off_lock.load(Ordering::Acquire)); + if panic { + assert_eq!(completion, TerminalCompletion::Uncertain); + assert!(view.inner.has_fault()); + assert!(terminal::TerminalState::lookup(view.inner.context()).is_some()); + assert!(!events.try_iter().any(|event| matches!(event, controlled::Event::FactoryDrop(_))), "remaining foreign captures must stay anchored after first cleanup panic"); + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_prior_reinit_drop_blocks_next_setup_then_stop_settles_same_job() { + controlled_prior_reinit_drop(false).await; +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_prior_reinit_drop_panic_retains_resources_but_attempts_finish_once() { + controlled_prior_reinit_drop(true).await; +} From a726bb148599679050dcef5b916a8e7b7457bd6d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:05:05 +0300 Subject: [PATCH 065/130] test(module): cover terminal delivery caller restoration Add a test that drives terminal delivery while an outer caller is active and asserts the exact instance is entered and the nested caller is restored afterwards. A tracing native query is used so the delivery path can be observed without an inbound queue installed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/execution_context_tests.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/tinybus-module/src/execution_context_tests.rs b/crates/tinybus-module/src/execution_context_tests.rs index 3a3b40c7..0acdda77 100644 --- a/crates/tinybus-module/src/execution_context_tests.rs +++ b/crates/tinybus-module/src/execution_context_tests.rs @@ -223,3 +223,26 @@ fn shutdown_suppresses_the_exact_factory_and_configuration_caller_dispatch() { ); } } + +unsafe extern "C" fn tracing_native_query(_: *mut c_void) -> u32 { + tracing::info!("terminal delivery query"); + tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE +} +#[test] +fn terminal_deliver_enters_exact_instance_and_restores_nested_caller() { + let (caller, caller_count) = owner(); + let (instance, instance_count) = owner(); + let host = HostCalls(TbHostVtable { + native_terminal_state: Some(tracing_native_query), + ..instance.host.0 + }); + let state = super::super::terminal::TerminalState::admit(host, Some(instance)).unwrap(); + let _outer = caller.enter(); + let byte = 0; + let code = unsafe { super::super::terminal_deliver(state.context(), &byte, 1) }; + tracing::info!("restored caller"); + state.empty_factory_failure(); + assert_eq!(code, TB_CLOSED); // No inbound queue was installed. + assert_eq!(instance_count.load(Ordering::SeqCst), 1); + assert_eq!(caller_count.load(Ordering::SeqCst), 1); +} From 2823b5916455ae3e74d832414ecca89e260b6d37 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:05:44 +0300 Subject: [PATCH 066/130] fix(terminal): track disposal failure separately from execution settlement The completion path conflated setup disposal with task settlement, so a failed dispose_setup could be mistaken for a settled execution and the finish owner was retained on the wrong condition. Disposal failure is now tracked in its own flag, completion keys off execution_settled, and terminal_deliver enters the execution owner scope before checking liveness. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 1 + crates/tinybus-module/src/terminal.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 840e9b52..49c981fb 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1336,6 +1336,7 @@ unsafe extern "C" fn terminal_deliver(ctx: *mut c_void, ptr: *const u8, len: usi let Some(state) = terminal::TerminalState::lookup(ctx) else { return TB_CLOSED; }; + let _scope = state.owner.as_ref().map(ExecutionOwner::enter); if !state.is_live() { return TB_CLOSED; } diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index dd50b2bc..76cc89d3 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -618,19 +618,19 @@ impl TerminalState { // spend the controller's still-uninvoked physical finish callback. let execution_settled = self.settle_execution(); let retirement_failed = self.lock().retirement_failed; - let setup_settled = if execution_settled && !retirement_failed { + let disposal_failed = if execution_settled && !retirement_failed { match catch_unwind(AssertUnwindSafe(|| self.dispose_setup())) { - Ok(()) => true, + Ok(()) => false, Err(payload) => { self.source_panic(PayloadKind::Cleanup, payload); - false + true } } } else { - execution_settled + retirement_failed }; let finish = self.lock().finish.take(); - let completed = if setup_settled { + let completed = if execution_settled { if let Some(owner) = &self.owner { owner.stop(); } @@ -644,11 +644,11 @@ impl TerminalState { } } } else { - // If setup destruction could not be proved, keep the original uninvoked finish owner. + // If actual task settlement could not be proved, retain the original finish owner. self.lock().finish = finish; TerminalCompletion::Uncertain }; - let released = if completed == TerminalCompletion::PhysicallyStopped && !retirement_failed { + let released = if completed == TerminalCompletion::PhysicallyStopped && !disposal_failed { match catch_unwind(AssertUnwindSafe(|| self.release_resources())) { Ok(released) => released, Err(payload) => { From dd3bfda311a0ae893c10f4b945a19a77f194dbb6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:08:08 +0300 Subject: [PATCH 067/130] refactor(module): box terminal setup factory and expand teardown tests The controlled test fixture now returns a boxed setup factory so the terminal setup future type is named once, and the terminal tests were reformatted and extended to cover repeated reinitialization, connection teardown, and fault fencing. Documentation was updated to describe the bounded retirement cell, callback admission exclusion, and the counted observer that keeps the whole Connection anchored. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 4 +- .../tinybus-module/src/lib_terminal_tests.rs | 268 ++++++++++++++---- crates/tinybus-module/src/terminal.rs | 38 ++- docs/modules/module/README.md | 14 +- 4 files changed, 253 insertions(+), 71 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 49c981fb..9c0b7b19 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1400,7 +1400,9 @@ unsafe extern "C" fn terminal_reinitialize(ctx: *mut c_void, ptr: *const u8, len .unwrap_or_else(std::sync::PoisonError::into_inner) .clone(); let outcome = catch_unwind(AssertUnwindSafe(|| { - callback.as_ref().map_or(TB_CLOSED, |callback| callback(ptr, len)) + callback + .as_ref() + .map_or(TB_CLOSED, |callback| callback(ptr, len)) })); drop(callback); // non-last: cleanup still waits active_reinit let code = match outcome { diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 408d79be..de65344f 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -201,16 +201,14 @@ mod controlled { let control = self.fixture.future_drop_control.lock().unwrap().take(); if let Some((release, panic)) = control { release.wait(); - if panic { panic!("controlled prior reinit future Drop"); } + if panic { + panic!("controlled prior reinit future Drop"); + } } } } - fn factory( - view: NativeInstanceLiveness, - ) -> Result<( - impl Fn(Connection, Config) -> Pin> + Send>> + Send + Sync, - RuntimeHooks, - )> { + type FixtureSetup = Box terminal::SetupFuture + Send + Sync>; + fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> { let fixture = NEXT.lock().unwrap().take().unwrap(); *fixture.view.lock().unwrap() = Some(view.clone()); let (request, finish) = (fixture.clone(), fixture.clone()); @@ -271,7 +269,7 @@ mod controlled { TerminalCompletion::PhysicallyStopped }, ); - Ok((setup, hooks)) + Ok((Box::new(setup), hooks)) } module_export_static! { terminal_setup_factory = factory, config = Config, worker_threads = 1, @@ -634,8 +632,8 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal ); } assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); - assert!(seen.iter().any(|event| *event == (5, 1))); - assert!(seen.iter().any(|event| *event == (6, 1))); + assert!(seen.contains(&(5, 1))); + assert!(seen.contains(&(6, 1))); } struct HeldNativeControls { @@ -740,8 +738,14 @@ impl tinybus::Interface for TeardownObject { fn name(&self) -> tinybus::InterfaceName { "ai.tinyhumans.tinybus.Teardown".parse().unwrap() } - fn members(&self) -> Vec { Vec::new() } - async fn call(&self, _: &tinybus::MemberName, _: serde_json::Value) -> Result { + fn members(&self) -> Vec { + Vec::new() + } + async fn call( + &self, + _: &tinybus::MemberName, + _: serde_json::Value, + ) -> Result { Ok(serde_json::Value::Null) } } @@ -750,7 +754,9 @@ impl Drop for TeardownObject { self.entered.send(()).unwrap(); self.release.wait(); self.settled.send(()).unwrap(); - if self.panic { panic!("controlled ObjectTree teardown"); } + if self.panic { + panic!("controlled ObjectTree teardown"); + } } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -758,32 +764,65 @@ async fn actual_connection_object_drop_after_runtime_timeout_denies_physical_com let _serial = controlled::TEST_LOCK.lock().await; let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); - host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); events.recv_timeout(Duration::from_secs(3)).unwrap(); let view = fixture.view.lock().unwrap().take().unwrap(); let (entered_tx, entered_rx) = std::sync::mpsc::channel(); let (settled_tx, settled_rx) = std::sync::mpsc::channel(); let release = Arc::new(std::sync::Barrier::new(2)); - let connection = view.inner.resources.lock().unwrap().connection.lock().unwrap().clone().unwrap(); - connection.serve_at("/teardown".parse().unwrap(), TeardownObject { - entered: entered_tx, release: release.clone(), settled: settled_tx, panic: false, - }).await.unwrap(); + let connection = view + .inner + .resources + .lock() + .unwrap() + .connection + .lock() + .unwrap() + .clone() + .unwrap(); + connection + .serve_at( + "/teardown".parse().unwrap(), + TeardownObject { + entered: entered_tx, + release: release.clone(), + settled: settled_tx, + panic: false, + }, + ) + .await + .unwrap(); drop(connection); let stopping_host = host.clone(); - let stopping = tokio::spawn(async move { stopping_host.shutdown(Duration::from_secs(8)).await; }); + let stopping = tokio::spawn(async move { + stopping_host.shutdown(Duration::from_secs(8)).await; + }); entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); stopping.await.unwrap(); // The SDK's real shutdown_timeout has elapsed. let completion = view.inner.wait(Duration::ZERO); - let flags = unsafe { (view.inner.host.0.native_terminal_state.unwrap())(view.inner.host.0.host_ctx) }; + let flags = + unsafe { (view.inner.host.0.native_terminal_state.unwrap())(view.inner.host.0.host_ctx) }; let still_anchored = terminal::TerminalState::lookup(view.inner.context()).is_some(); // Release and settle actual foreign work before reporting a failed proof. release.wait(); settled_rx.recv_timeout(Duration::from_secs(3)).unwrap(); assert_eq!(completion, TerminalCompletion::Uncertain); - assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); - assert!(still_anchored, "blocked whole-Connection teardown cannot refund admission"); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert!( + still_anchored, + "blocked whole-Connection teardown cannot refund admission" + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -791,32 +830,65 @@ async fn actual_connection_object_drop_panic_fences_fault_after_owner_suppressio let _serial = controlled::TEST_LOCK.lock().await; let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); - host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); events.recv_timeout(Duration::from_secs(3)).unwrap(); let view = fixture.view.lock().unwrap().take().unwrap(); let (entered_tx, entered_rx) = std::sync::mpsc::channel(); let (settled_tx, settled_rx) = std::sync::mpsc::channel(); let release = Arc::new(std::sync::Barrier::new(2)); - let connection = view.inner.resources.lock().unwrap().connection.lock().unwrap().clone().unwrap(); - connection.serve_at("/teardown".parse().unwrap(), TeardownObject { - entered: entered_tx, release: release.clone(), settled: settled_tx, panic: true, - }).await.unwrap(); + let connection = view + .inner + .resources + .lock() + .unwrap() + .connection + .lock() + .unwrap() + .clone() + .unwrap(); + connection + .serve_at( + "/teardown".parse().unwrap(), + TeardownObject { + entered: entered_tx, + release: release.clone(), + settled: settled_tx, + panic: true, + }, + ) + .await + .unwrap(); drop(connection); let stopping_host = host.clone(); - let stopping = tokio::spawn(async move { stopping_host.shutdown(Duration::from_secs(8)).await; }); + let stopping = tokio::spawn(async move { + stopping_host.shutdown(Duration::from_secs(8)).await; + }); entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); release.wait(); settled_rx.recv_timeout(Duration::from_secs(3)).unwrap(); stopping.await.unwrap(); // Runtime teardown settles the real panicking destructor. let completion = view.inner.wait(Duration::ZERO); - let flags = unsafe { (view.inner.host.0.native_terminal_state.unwrap())(view.inner.host.0.host_ctx) }; + let flags = + unsafe { (view.inner.host.0.native_terminal_state.unwrap())(view.inner.host.0.host_ctx) }; let still_anchored = terminal::TerminalState::lookup(view.inner.context()).is_some(); assert_eq!(completion, TerminalCompletion::Uncertain); - assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); - assert!(still_anchored, "panicking Connection teardown cannot refund admission"); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert!( + still_anchored, + "panicking Connection teardown cannot refund admission" + ); assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); } @@ -825,24 +897,50 @@ async fn successful_repeated_reinit_disposes_prior_future_on_same_physical_worke let _serial = controlled::TEST_LOCK.lock().await; let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); - host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); events.recv_timeout(Duration::from_secs(3)).unwrap(); tokio::time::timeout(Duration::from_secs(3), async { - while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { tokio::task::yield_now().await; } - }).await.unwrap(); - let first = host.reinitialize("tinybus-module", serde_json::json!({"mode":0})).await; - let second = host.reinitialize("tinybus-module", serde_json::json!({"mode":0})).await; + }) + .await + .unwrap(); + let first = host + .reinitialize("tinybus-module", serde_json::json!({"mode":0})) + .await; + let second = host + .reinitialize("tinybus-module", serde_json::json!({"mode":0})) + .await; host.shutdown(Duration::from_secs(8)).await; - let drops: Vec<_> = events.try_iter().filter_map(|event| { - if let controlled::Event::FutureDrop(name) = event { Some(name) } else { None } - }).collect(); + let drops: Vec<_> = events + .try_iter() + .filter_map(|event| { + if let controlled::Event::FutureDrop(name) = event { + Some(name) + } else { + None + } + }) + .collect(); assert!(first.is_ok() && second.is_ok()); assert_eq!(drops.len(), 3); - assert!(drops.iter().all(|name| name == "tinybus-terminal"), "all completed futures must be destroyed by the sole physical worker: {drops:?}"); + assert!( + drops.iter().all(|name| name == "tinybus-terminal"), + "all completed futures must be destroyed by the sole physical worker: {drops:?}" + ); assert!(fixture.drop_off_lock.load(Ordering::Acquire)); } @@ -852,24 +950,46 @@ async fn actual_reinit_source_error_and_panic_request_outside_callback_exclusion for mode in [3, 2, 4] { let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); - host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); events.recv_timeout(Duration::from_secs(3)).unwrap(); tokio::time::timeout(Duration::from_secs(3), async { - while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { tokio::task::yield_now().await; } - }).await.unwrap(); - let result = host.reinitialize("tinybus-module", serde_json::json!({"mode":mode})).await; + }) + .await + .unwrap(); + let result = host + .reinitialize("tinybus-module", serde_json::json!({"mode":mode})) + .await; host.shutdown(Duration::from_secs(8)).await; loop { - if matches!(events.recv_timeout(Duration::from_secs(5)).unwrap(), controlled::Event::Finish) { break; } + if matches!( + events.recv_timeout(Duration::from_secs(5)).unwrap(), + controlled::Event::Finish + ) { + break; + } } assert!(result.is_err()); assert_eq!(fixture.requests.load(Ordering::SeqCst), 1); assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); - assert!(fixture.request_off_lock.load(Ordering::Acquire), "source fault request held reinitializer mutex in mode {mode}"); + assert!( + fixture.request_off_lock.load(Ordering::Acquire), + "source fault request held reinitializer mutex in mode {mode}" + ); } } @@ -877,30 +997,55 @@ async fn controlled_prior_reinit_drop(panic: bool) { let _serial = controlled::TEST_LOCK.lock().await; let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); - host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); events.recv_timeout(Duration::from_secs(3)).unwrap(); tokio::time::timeout(Duration::from_secs(3), async { - while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { tokio::task::yield_now().await; } - }).await.unwrap(); - host.reinitialize("tinybus-module", serde_json::json!({"mode":0})).await.unwrap(); + }) + .await + .unwrap(); + host.reinitialize("tinybus-module", serde_json::json!({"mode":0})) + .await + .unwrap(); let _ = events.try_iter().count(); let release = Arc::new(std::sync::Barrier::new(2)); *fixture.future_drop_control.lock().unwrap() = Some((release.clone(), panic)); let caller_host = host.clone(); - let caller = tokio::spawn(async move { caller_host.reinitialize("tinybus-module", serde_json::json!({"mode":0})).await }); + let caller = tokio::spawn(async move { + caller_host + .reinitialize("tinybus-module", serde_json::json!({"mode":0})) + .await + }); let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); let view = fixture.view.lock().unwrap().as_ref().unwrap().clone(); view.inner.signal(TerminalReason::Stop); let not_finished_while_blocked = fixture.joins.load(Ordering::SeqCst) == 0; - let no_next_setup = !events.try_iter().any(|event| matches!(event, controlled::Event::Poll(_))); + let no_next_setup = !events + .try_iter() + .any(|event| matches!(event, controlled::Event::Poll(_))); release.wait(); let result = caller.await.unwrap(); loop { - if matches!(events.recv_timeout(Duration::from_secs(5)).unwrap(), controlled::Event::Finish) { break; } + if matches!( + events.recv_timeout(Duration::from_secs(5)).unwrap(), + controlled::Event::Finish + ) { + break; + } } host.shutdown(Duration::from_secs(8)).await; let completion = view.inner.wait(Duration::from_secs(5)); @@ -915,7 +1060,12 @@ async fn controlled_prior_reinit_drop(panic: bool) { assert_eq!(completion, TerminalCompletion::Uncertain); assert!(view.inner.has_fault()); assert!(terminal::TerminalState::lookup(view.inner.context()).is_some()); - assert!(!events.try_iter().any(|event| matches!(event, controlled::Event::FactoryDrop(_))), "remaining foreign captures must stay anchored after first cleanup panic"); + assert!( + !events + .try_iter() + .any(|event| matches!(event, controlled::Event::FactoryDrop(_))), + "remaining foreign captures must stay anchored after first cleanup panic" + ); } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 76cc89d3..af60105d 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -519,7 +519,9 @@ impl TerminalState { self.changed.notify_all(); } pub fn retire_reinit(self: &Arc, future: Option) -> bool { - let Some(future) = future else { return true; }; + let Some(future) = future else { + return true; + }; { let mut control = self.lock(); // active_reinit admits one caller. It cannot construct the next @@ -531,7 +533,10 @@ impl TerminalState { self.changed.notify_all(); let mut control = self.lock(); while control.prior_pending && control.finish_state != FinishState::SpawnFailed { - control = self.changed.wait(control).unwrap_or_else(std::sync::PoisonError::into_inner); + control = self + .changed + .wait(control) + .unwrap_or_else(std::sync::PoisonError::into_inner); } !control.retirement_failed && control.finish_state != FinishState::SpawnFailed } @@ -605,14 +610,22 @@ impl TerminalState { self.changed.notify_all(); continue; } - if control.published && !control.active_reinit && !matches!( - control.request_state, RequestState::Idle | RequestState::Running - ) { + if control.published + && !control.active_reinit + && !matches!( + control.request_state, + RequestState::Idle | RequestState::Running + ) + { control.finish_state = FinishState::Running; break; } - drop(self.changed.wait(control).unwrap_or_else(std::sync::PoisonError::into_inner)); - }; + drop( + self.changed + .wait(control) + .unwrap_or_else(std::sync::PoisonError::into_inner), + ); + } // Execution settlement is distinct from foreign capture destruction. // A failed prior retirement retains all later captures, but does not // spend the controller's still-uninvoked physical finish callback. @@ -720,8 +733,15 @@ impl TerminalState { // The worker already observed active_reinit=false. Every last capture // is taken mechanically and destroyed after all bookkeeping unlocks. let (reinitialize, future, reinit_future) = { - let resources = self.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - (resources.reinitialize.clone(), resources.initial_future.clone(), resources.reinit_future.clone()) + let resources = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ( + resources.reinitialize.clone(), + resources.initial_future.clone(), + resources.reinit_future.clone(), + ) }; if let Some(owner) = &self.owner { owner.stop(); diff --git a/docs/modules/module/README.md b/docs/modules/module/README.md index 32cfe6de..5ba266b4 100644 --- a/docs/modules/module/README.md +++ b/docs/modules/module/README.md @@ -69,13 +69,23 @@ work; a timeout or host shutdown return does not prove it. The SDK retains setup futures across cancellation, destroys them on the physical cleanup thread, and drops callback captures outside resource locks. Reinitialization polls its result before terminal cancellation when both are -ready. The runtime is destroyed on the physical thread. Surviving SDK resources, +ready. Repeated successful reinitialization retires its previous future through +one bounded cell on that same physical worker; the next setup starts only after +the worker confirms disposal. Callback admission uses bookkeeping exclusion, +with foreign invocation, request and destruction outside every resource lock. +The runtime is destroyed on the physical thread. Surviving SDK resources, spawn failure, a hook panic, or unproved cleanup keep the owner uncertain and anchored. Seven separate primary panic cells retain request, finish, initial setup, reinitialize, factory, startup, and cleanup payloads. Native fault and request completion precede any payload disposal. After the first cleanup destructor panic, later resource owners remain attached rather than being -destroyed during cleanup unwinding. +destroyed during cleanup unwinding. Once actual setup execution has settled, +the controller finish still runs once; retained or panicking SDK teardown keeps +the result uncertain even if that controller finish succeeds. A counted observer +retains the entire Connection through its last field and all cloned method +observers. A destructor unwind explicitly reports native Fault even after +diagnostic callbacks have been suppressed; a transport count or runtime timeout +cannot certify whole-Connection destruction. Each SDK image has 256 instance slots. The existing opaque `module_ctx` contains a slot/generation address token, never a dereferenced pointer or extra ABI From 931aa3689d4f02b3168c64b65ea98f5ddaf26049 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:09:10 +0300 Subject: [PATCH 068/130] refactor(tinybus): name terminal example fixture types Extract the factory return types into FixtureFuture and FixtureSetup aliases and box the setup closure, so the signature is readable and the closure can be returned without leaking its concrete type. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/examples/module_terminal.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index b65d58f6..9f97d9dd 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -91,12 +91,9 @@ impl Drop for SetupFuture { ); } } -fn factory( - view: NativeInstanceLiveness, -) -> Result<( - impl Fn(Connection, Config) -> Pin> + Send>> + Send + Sync, - RuntimeHooks, -)> { +type FixtureFuture = Pin> + Send>>; +type FixtureSetup = Box FixtureFuture + Send + Sync>; +fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> { let early = EARLY .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -198,7 +195,7 @@ fn factory( } }, ); - Ok((setup, hooks)) + Ok((Box::new(setup), hooks)) } #[unsafe(no_mangle)] pub static TINYBUS_MODULE_ABI_V1: TbAbiDescriptor = From adc2c9d25c4f54108196b6739cc95679bdfc2b3a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:11:13 +0300 Subject: [PATCH 069/130] test(tinybus-module): wait for inner settlement in terminal drop tests Two terminal drop tests now wait until only the retained registry anchor and the test's own view hold the inner state before asserting, so the checks no longer treat the object's last event as whole-Inner settlement. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index de65344f..aafde124 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -814,6 +814,16 @@ async fn actual_connection_object_drop_after_runtime_timeout_denies_physical_com // Release and settle actual foreign work before reporting a failed proof. release.wait(); settled_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + // Only the retained registry anchor and this test's view may remain. + // This waits for actual SDK Connection/task owners to finish dropping, + // rather than treating the object's last event as whole-Inner settlement. + tokio::time::timeout(Duration::from_secs(3), async { + while Arc::strong_count(&view.inner) > 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); assert_eq!(completion, TerminalCompletion::Uncertain); assert_eq!( flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, @@ -880,6 +890,16 @@ async fn actual_connection_object_drop_panic_fences_fault_after_owner_suppressio unsafe { (view.inner.host.0.native_terminal_state.unwrap())(view.inner.host.0.host_ctx) }; let still_anchored = terminal::TerminalState::lookup(view.inner.context()).is_some(); + // Only the retained registry anchor and this test's view may remain. + // This waits for actual SDK Connection/task owners to finish dropping, + // rather than treating the object's last event as whole-Inner settlement. + tokio::time::timeout(Duration::from_secs(3), async { + while Arc::strong_count(&view.inner) > 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); assert_eq!(completion, TerminalCompletion::Uncertain); assert_eq!( flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, From cc104293656090ed979e89ee592cde952fbba840 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:20:27 +0300 Subject: [PATCH 070/130] test(module): assert config drop runs on the physical worker Extend the controlled terminal fixture to count config drops and emit a ConfigDrop event, then assert that a rejected reinit never receives a config and that an accepted reinit destroys its config on the single physical worker even when the constructor panics. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index aafde124..72d1278a 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -115,12 +115,24 @@ mod controlled { #[derive(serde::Deserialize)] pub struct Config { pub mode: u8, + #[serde(default)] + record_drop: bool, + } + impl Drop for Config { + fn drop(&mut self) { + if self.record_drop { + let fixture = NEXT.lock().unwrap().as_ref().unwrap().clone(); + fixture.config_drops.fetch_add(1, Ordering::SeqCst); + let _ = fixture.events.send(Event::ConfigDrop(std::thread::current().name().unwrap_or("unnamed").to_owned())); + } + } } #[derive(Debug)] pub enum Event { Poll(u8), Request, FutureDrop(String), + ConfigDrop(String), Finish, FactoryDrop(String), } @@ -131,6 +143,7 @@ mod controlled { worker: StdMutex>>, pub requests: AtomicUsize, pub joins: AtomicUsize, + pub config_drops: AtomicUsize, pub drop_off_lock: AtomicBool, pub request_off_lock: AtomicBool, pub future_drop_control: StdMutex, bool)>>, @@ -146,6 +159,7 @@ mod controlled { worker: StdMutex::new(None), requests: AtomicUsize::new(0), joins: AtomicUsize::new(0), + config_drops: AtomicUsize::new(0), drop_off_lock: AtomicBool::new(true), request_off_lock: AtomicBool::new(true), future_drop_control: StdMutex::new(None), @@ -1047,7 +1061,7 @@ async fn controlled_prior_reinit_drop(panic: bool) { let caller_host = host.clone(); let caller = tokio::spawn(async move { caller_host - .reinitialize("tinybus-module", serde_json::json!({"mode":0})) + .reinitialize("tinybus-module", serde_json::json!({"mode":0,"record_drop":true})) .await }); let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); @@ -1069,6 +1083,7 @@ async fn controlled_prior_reinit_drop(panic: bool) { } host.shutdown(Duration::from_secs(8)).await; let completion = view.inner.wait(Duration::from_secs(5)); + assert_eq!(fixture.config_drops.load(Ordering::SeqCst), 0, "rejected reinit must not receive a config before prior retirement and LIVE admission"); assert!(matches!(entered, controlled::Event::FutureDrop(name) if name == "tinybus-terminal")); assert!(not_finished_while_blocked && no_next_setup); assert!(result.is_err()); @@ -1096,3 +1111,35 @@ async fn actual_prior_reinit_drop_blocks_next_setup_then_stop_settles_same_job() async fn actual_prior_reinit_drop_panic_retains_resources_but_attempts_finish_once() { controlled_prior_reinit_drop(true).await; } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn accepted_reinit_config_is_destroyed_on_physical_worker_including_constructor_panic() { + let _serial = controlled::TEST_LOCK.lock().await; + for mode in [0, 4] { + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { tokio::task::yield_now().await; } + }).await.unwrap(); + let result = host.reinitialize("tinybus-module", serde_json::json!({"mode":mode,"record_drop":true})).await; + host.shutdown(Duration::from_secs(8)).await; + let mut names = Vec::new(); + loop { + match events.recv_timeout(Duration::from_secs(5)).unwrap() { + controlled::Event::ConfigDrop(name) => names.push(name), + controlled::Event::Finish => break, + _ => {} + } + } + let view = fixture.view.lock().unwrap().as_ref().unwrap().clone(); + let _completion = view.inner.wait(Duration::from_secs(5)); + assert_eq!(result.is_ok(), mode == 0); + assert_eq!(fixture.config_drops.load(Ordering::SeqCst), 1); + assert_eq!(names, ["tinybus-terminal"], "accepted generic config Drop must run on the one physical worker"); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + } +} From aab34bb0835ff5e9a7cbb261e0bd43bdcea8c708 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:21:38 +0300 Subject: [PATCH 071/130] test(tinybus-module): hold config fixture via weak reference The controlled test Config now keeps a weak reference to its fixture instead of reaching for the global NEXT slot on drop, so drop recording no longer panics when the fixture has already been torn down. The fixture is registered in a static weak slot when it is created. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 72d1278a..612607d1 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -117,11 +117,15 @@ mod controlled { pub mode: u8, #[serde(default)] record_drop: bool, + #[serde(skip, default = "config_fixture")] + fixture: std::sync::Weak, } + static CONFIG_FIXTURE: StdMutex> = StdMutex::new(std::sync::Weak::new()); + fn config_fixture() -> std::sync::Weak { CONFIG_FIXTURE.lock().unwrap().clone() } impl Drop for Config { fn drop(&mut self) { if self.record_drop { - let fixture = NEXT.lock().unwrap().as_ref().unwrap().clone(); + let Some(fixture) = self.fixture.upgrade() else { return; }; fixture.config_drops.fetch_add(1, Ordering::SeqCst); let _ = fixture.events.send(Event::ConfigDrop(std::thread::current().name().unwrap_or("unnamed").to_owned())); } @@ -164,6 +168,7 @@ mod controlled { request_off_lock: AtomicBool::new(true), future_drop_control: StdMutex::new(None), }); + *CONFIG_FIXTURE.lock().unwrap() = Arc::downgrade(&fixture); *NEXT.lock().unwrap() = Some(fixture.clone()); (fixture, receiver) } From 364e32429bdaac83f95601646ee7987073b0a5b6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:22:56 +0300 Subject: [PATCH 072/130] fix(module): defer reinit config parsing to the worker thread Reinit now copies the ABI input and hands a construction closure to the existing worker instead of parsing the config on the caller's thread, so the config is owned and parsed by the same worker that runs setup. This keeps the received config attached until that worker disposes its terminal resources, so a Stop arriving mid-parse no longer loses it. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 39 ++++++++++++------- crates/tinybus-module/src/terminal.rs | 55 ++++++++++++++++++++++----- 2 files changed, 71 insertions(+), 23 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 9c0b7b19..248de0ee 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1560,15 +1560,6 @@ where if !state.is_live() { return TB_CLOSED; } - let bytes = if len == 0 { - b"{}".as_slice() - } else { - unsafe { std::slice::from_raw_parts(ptr, len) } - }; - let config = match serde_json::from_slice::(bytes) { - Ok(config) => config, - Err(_) => return TB_BAD_ARGUMENT, - }; let connection = connection_slot .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -1592,10 +1583,32 @@ where if !state.is_live() { return TB_CLOSED; } - let future = Box::pin(setup(connection, config)); - *slot - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); + // Copy ABI input before transferring the single construction operation. + // Only the existing physical worker parses/owns C and calls setup. + let bytes = if len == 0 { b"{}".to_vec() } else { + unsafe { std::slice::from_raw_parts(ptr, len) }.to_vec() + }; + let construct_state = Arc::downgrade(&state); + let construct_setup = setup.clone(); + let construct_slot = slot.clone(); + let code = state.construct_reinit(Box::new(move || { + let Some(state) = construct_state.upgrade() else { return TB_CLOSED; }; + if !state.is_live() { return TB_CLOSED; } + let config = match serde_json::from_slice::(&bytes) { + Ok(config) => config, + Err(_) => return TB_BAD_ARGUMENT, + }; + // If Stop arrived during parsing, keep the received config attached + // until this same worker's terminal resource disposal. + state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).config = Some(Box::new(config)); + if !state.is_live() { return TB_CLOSED; } + let config = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).config.take().expect("attached reinit config"); + let config = *config.downcast::().unwrap_or_else(|_| unreachable!("private config type")); + let future = Box::pin(construct_setup(connection, config)); + *construct_slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); + TB_OK + })); + if code != TB_OK { return code; } let result = handle.block_on(async { tokio::select! { biased; result = TerminalSetupPoll { slot, state: state.clone(), kind: terminal::PayloadKind::Reinit } => result, diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index af60105d..19703d63 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -155,6 +155,10 @@ impl PrimaryPayloads { } } } +enum ReinitOperation { + Retire(SetupFuture), + Construct(Box i32 + Send>), +} struct Control { request: Option>, finish: Option TerminalCompletion + Send>>, @@ -165,7 +169,8 @@ struct Control { job: Option>, payloads: PrimaryPayloads, active_reinit: bool, - prior_future: Option, + reinit_operation: Option, + construction_code: i32, prior_pending: bool, retirement_failed: bool, } @@ -293,7 +298,8 @@ impl TerminalState { job: None, payloads: PrimaryPayloads::default(), active_reinit: false, - prior_future: None, + reinit_operation: None, + construction_code: TB_CLOSED, prior_pending: false, retirement_failed: false, }), @@ -526,7 +532,7 @@ impl TerminalState { let mut control = self.lock(); // active_reinit admits one caller. It cannot construct the next // future until the worker has acknowledged this exact disposal. - control.prior_future = Some(future); + control.reinit_operation = Some(ReinitOperation::Retire(future)); control.prior_pending = true; } self.schedule(); @@ -540,6 +546,22 @@ impl TerminalState { } !control.retirement_failed && control.finish_state != FinishState::SpawnFailed } + pub fn construct_reinit(self: &Arc, construct: Box i32 + Send>) -> i32 { + { + let mut control = self.lock(); + // The same sole cell owns construction captures before scheduling. + // Spawn failure retains them; no ABI caller receives generic C. + control.reinit_operation = Some(ReinitOperation::Construct(construct)); + control.prior_pending = true; + } + self.schedule(); + self.changed.notify_all(); + let mut control = self.lock(); + while control.prior_pending && control.finish_state != FinishState::SpawnFailed { + control = self.changed.wait(control).unwrap_or_else(std::sync::PoisonError::into_inner); + } + if control.finish_state == FinishState::SpawnFailed { TB_CLOSED } else { control.construction_code } + } fn schedule(self: &Arc) { let job = { let mut control = self.lock(); @@ -593,18 +615,31 @@ impl TerminalState { let mut control = self.lock(); // Retirement takes precedence over waiting for active_reinit: // that caller may itself be awaiting this worker's acknowledgment. - if let Some(future) = control.prior_future.take() { + if let Some(operation) = control.reinit_operation.take() { drop(control); - let outcome = catch_unwind(AssertUnwindSafe(|| drop(future))); - let failed = match outcome { - Ok(()) => false, - Err(payload) => { - self.source_panic(PayloadKind::Cleanup, payload); - true + let (failed, code) = match operation { + ReinitOperation::Retire(future) => { + match catch_unwind(AssertUnwindSafe(|| drop(future))) { + Ok(()) => (false, TB_OK), + Err(payload) => { + self.source_panic(PayloadKind::Cleanup, payload); + (true, TB_CLOSED) + } + } + } + ReinitOperation::Construct(construct) => { + match catch_unwind(AssertUnwindSafe(construct)) { + Ok(code) => (false, code), + Err(payload) => { + self.source_panic(PayloadKind::Reinit, payload); + (false, TB_PANICKED) + } + } } }; let mut control = self.lock(); control.retirement_failed |= failed; + control.construction_code = code; control.prior_pending = false; drop(control); self.changed.notify_all(); From e359a8766be2be0f24bb6c82eb57806ad34162bf Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:24:29 +0300 Subject: [PATCH 073/130] style: reformat long lines and expand single-line if/else blocks Reformatted the module, terminal, and terminal test sources to break long expressions and single-line conditional bodies across multiple lines. The change is purely cosmetic and leaves behaviour unchanged. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 42 +++++++++--- .../tinybus-module/src/lib_terminal_tests.rs | 64 +++++++++++++++---- crates/tinybus-module/src/terminal.rs | 11 +++- 3 files changed, 94 insertions(+), 23 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 248de0ee..92f198fa 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1585,30 +1585,54 @@ where } // Copy ABI input before transferring the single construction operation. // Only the existing physical worker parses/owns C and calls setup. - let bytes = if len == 0 { b"{}".to_vec() } else { + let bytes = if len == 0 { + b"{}".to_vec() + } else { unsafe { std::slice::from_raw_parts(ptr, len) }.to_vec() }; let construct_state = Arc::downgrade(&state); let construct_setup = setup.clone(); let construct_slot = slot.clone(); let code = state.construct_reinit(Box::new(move || { - let Some(state) = construct_state.upgrade() else { return TB_CLOSED; }; - if !state.is_live() { return TB_CLOSED; } + let Some(state) = construct_state.upgrade() else { + return TB_CLOSED; + }; + if !state.is_live() { + return TB_CLOSED; + } let config = match serde_json::from_slice::(&bytes) { Ok(config) => config, Err(_) => return TB_BAD_ARGUMENT, }; // If Stop arrived during parsing, keep the received config attached // until this same worker's terminal resource disposal. - state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).config = Some(Box::new(config)); - if !state.is_live() { return TB_CLOSED; } - let config = state.resources.lock().unwrap_or_else(std::sync::PoisonError::into_inner).config.take().expect("attached reinit config"); - let config = *config.downcast::().unwrap_or_else(|_| unreachable!("private config type")); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config = Some(Box::new(config)); + if !state.is_live() { + return TB_CLOSED; + } + let config = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config + .take() + .expect("attached reinit config"); + let config = *config + .downcast::() + .unwrap_or_else(|_| unreachable!("private config type")); let future = Box::pin(construct_setup(connection, config)); - *construct_slot.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); + *construct_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); TB_OK })); - if code != TB_OK { return code; } + if code != TB_OK { + return code; + } let result = handle.block_on(async { tokio::select! { biased; result = TerminalSetupPoll { slot, state: state.clone(), kind: terminal::PayloadKind::Reinit } => result, diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 612607d1..06b6aa28 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -120,14 +120,24 @@ mod controlled { #[serde(skip, default = "config_fixture")] fixture: std::sync::Weak, } - static CONFIG_FIXTURE: StdMutex> = StdMutex::new(std::sync::Weak::new()); - fn config_fixture() -> std::sync::Weak { CONFIG_FIXTURE.lock().unwrap().clone() } + static CONFIG_FIXTURE: StdMutex> = + StdMutex::new(std::sync::Weak::new()); + fn config_fixture() -> std::sync::Weak { + CONFIG_FIXTURE.lock().unwrap().clone() + } impl Drop for Config { fn drop(&mut self) { if self.record_drop { - let Some(fixture) = self.fixture.upgrade() else { return; }; + let Some(fixture) = self.fixture.upgrade() else { + return; + }; fixture.config_drops.fetch_add(1, Ordering::SeqCst); - let _ = fixture.events.send(Event::ConfigDrop(std::thread::current().name().unwrap_or("unnamed").to_owned())); + let _ = fixture.events.send(Event::ConfigDrop( + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned(), + )); } } } @@ -1066,7 +1076,10 @@ async fn controlled_prior_reinit_drop(panic: bool) { let caller_host = host.clone(); let caller = tokio::spawn(async move { caller_host - .reinitialize("tinybus-module", serde_json::json!({"mode":0,"record_drop":true})) + .reinitialize( + "tinybus-module", + serde_json::json!({"mode":0,"record_drop":true}), + ) .await }); let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); @@ -1088,7 +1101,11 @@ async fn controlled_prior_reinit_drop(panic: bool) { } host.shutdown(Duration::from_secs(8)).await; let completion = view.inner.wait(Duration::from_secs(5)); - assert_eq!(fixture.config_drops.load(Ordering::SeqCst), 0, "rejected reinit must not receive a config before prior retirement and LIVE admission"); + assert_eq!( + fixture.config_drops.load(Ordering::SeqCst), + 0, + "rejected reinit must not receive a config before prior retirement and LIVE admission" + ); assert!(matches!(entered, controlled::Event::FutureDrop(name) if name == "tinybus-terminal")); assert!(not_finished_while_blocked && no_next_setup); assert!(result.is_err()); @@ -1123,14 +1140,33 @@ async fn accepted_reinit_config_is_destroyed_on_physical_worker_including_constr for mode in [0, 4] { let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); - host.attach_linked_with_config(controlled::linked_module().unwrap(), serde_json::json!({"mode":0})).unwrap(); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); events.recv_timeout(Duration::from_secs(3)).unwrap(); tokio::time::timeout(Duration::from_secs(3), async { - while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { tokio::task::yield_now().await; } - }).await.unwrap(); - let result = host.reinitialize("tinybus-module", serde_json::json!({"mode":mode,"record_drop":true})).await; + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let result = host + .reinitialize( + "tinybus-module", + serde_json::json!({"mode":mode,"record_drop":true}), + ) + .await; host.shutdown(Duration::from_secs(8)).await; let mut names = Vec::new(); loop { @@ -1144,7 +1180,11 @@ async fn accepted_reinit_config_is_destroyed_on_physical_worker_including_constr let _completion = view.inner.wait(Duration::from_secs(5)); assert_eq!(result.is_ok(), mode == 0); assert_eq!(fixture.config_drops.load(Ordering::SeqCst), 1); - assert_eq!(names, ["tinybus-terminal"], "accepted generic config Drop must run on the one physical worker"); + assert_eq!( + names, + ["tinybus-terminal"], + "accepted generic config Drop must run on the one physical worker" + ); assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); } } diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 19703d63..45fe7ae6 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -558,9 +558,16 @@ impl TerminalState { self.changed.notify_all(); let mut control = self.lock(); while control.prior_pending && control.finish_state != FinishState::SpawnFailed { - control = self.changed.wait(control).unwrap_or_else(std::sync::PoisonError::into_inner); + control = self + .changed + .wait(control) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + if control.finish_state == FinishState::SpawnFailed { + TB_CLOSED + } else { + control.construction_code } - if control.finish_state == FinishState::SpawnFailed { TB_CLOSED } else { control.construction_code } } fn schedule(self: &Arc) { let job = { From 79bdc75ea40cd4be4184c8af85b341b88e991d9b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:25:05 +0300 Subject: [PATCH 074/130] test(module): cover stop during reinit config parse Add a test that blocks config deserialization mid-reinitialize, signals stop while the parse is gated, and asserts the received config is not dropped or joined until physical cleanup completes. The controlled fixture gains a parse gate hook and a ConfigParsing event so the test can observe the parse window deterministically. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 06b6aa28..c09882fd 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -115,6 +115,9 @@ mod controlled { #[derive(serde::Deserialize)] pub struct Config { pub mode: u8, + #[serde(default, deserialize_with = "parse_gate")] + #[serde(rename = "parse_gate")] + _parse_gate: bool, #[serde(default)] record_drop: bool, #[serde(skip, default = "config_fixture")] @@ -125,6 +128,18 @@ mod controlled { fn config_fixture() -> std::sync::Weak { CONFIG_FIXTURE.lock().unwrap().clone() } + fn parse_gate<'de, D: serde::Deserializer<'de>>( + deserializer: D, + ) -> std::result::Result { + let enabled = ::deserialize(deserializer)?; + if enabled { + let fixture = config_fixture().upgrade().unwrap(); + let barrier = fixture.config_parse_control.lock().unwrap().take().unwrap(); + fixture.events.send(Event::ConfigParsing).unwrap(); + barrier.wait(); + } + Ok(enabled) + } impl Drop for Config { fn drop(&mut self) { if self.record_drop { @@ -147,6 +162,7 @@ mod controlled { Request, FutureDrop(String), ConfigDrop(String), + ConfigParsing, Finish, FactoryDrop(String), } @@ -158,6 +174,7 @@ mod controlled { pub requests: AtomicUsize, pub joins: AtomicUsize, pub config_drops: AtomicUsize, + pub config_parse_control: StdMutex>>, pub drop_off_lock: AtomicBool, pub request_off_lock: AtomicBool, pub future_drop_control: StdMutex, bool)>>, @@ -174,6 +191,7 @@ mod controlled { requests: AtomicUsize::new(0), joins: AtomicUsize::new(0), config_drops: AtomicUsize::new(0), + config_parse_control: StdMutex::new(None), drop_off_lock: AtomicBool::new(true), request_off_lock: AtomicBool::new(true), future_drop_control: StdMutex::new(None), @@ -1188,3 +1206,75 @@ async fn accepted_reinit_config_is_destroyed_on_physical_worker_including_constr assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); } } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stop_during_reinit_parse_retains_received_config_until_physical_cleanup() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let release = Arc::new(std::sync::Barrier::new(2)); + *fixture.config_parse_control.lock().unwrap() = Some(release.clone()); + let caller_host = host.clone(); + let caller = tokio::spawn(async move { + caller_host + .reinitialize( + "tinybus-module", + serde_json::json!({"mode":0,"record_drop":true,"parse_gate":true}), + ) + .await + }); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().as_ref().unwrap().clone(); + view.inner.signal(TerminalReason::Stop); + let drops_while_blocked = fixture.config_drops.load(Ordering::SeqCst); + let joins_while_blocked = fixture.joins.load(Ordering::SeqCst); + release.wait(); + let result = caller.await.unwrap(); + loop { + if matches!( + events.recv_timeout(Duration::from_secs(5)).unwrap(), + controlled::Event::Finish + ) { + break; + } + } + host.shutdown(Duration::from_secs(8)).await; + let completion = view.inner.wait(Duration::from_secs(5)); + let remainder: Vec<_> = events.try_iter().collect(); + assert!(matches!(entered, controlled::Event::ConfigParsing)); + assert_eq!(drops_while_blocked, 0); + assert_eq!(joins_while_blocked, 0); + assert!(result.is_err()); + assert_eq!(completion, TerminalCompletion::PhysicallyStopped); + assert_eq!(fixture.config_drops.load(Ordering::SeqCst), 1); + assert!(remainder.iter().any( + |event| matches!(event, controlled::Event::ConfigDrop(name) if name == "tinybus-terminal") + )); + assert!( + !remainder + .iter() + .any(|event| matches!(event, controlled::Event::Poll(_))) + ); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); +} From 779e98af5211112853816d735b4ab20bf793ff91 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:33:04 +0300 Subject: [PATCH 075/130] chore: files changed crates/tinybus-module/src/lib_terminal_tests.rs Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index c09882fd..69d03a18 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -163,6 +163,7 @@ mod controlled { FutureDrop(String), ConfigDrop(String), ConfigParsing, + TimerConstruct(std::thread::ThreadId, String, tokio::runtime::Id), Finish, FactoryDrop(String), } @@ -215,16 +216,18 @@ mod controlled { fixture: Arc, mode: u8, first: bool, + timer: Option>>, } impl Future for SetupFuture { type Output = Result<()>; - fn poll(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { if self.first { self.first = false; self.fixture.events.send(Event::Poll(self.mode)).unwrap(); } match self.mode { 0 => Poll::Ready(Ok(())), + 5 => self.timer.as_mut().unwrap().as_mut().poll(cx).map(|()| Ok(())), 2 => panic!("controlled setup poll"), 3 => Poll::Ready(Err(Error::failed("controlled setup failure"))), _ => Poll::Pending, @@ -284,10 +287,21 @@ mod controlled { if config.mode == 4 { panic!("controlled setup construction"); } + let timer = (config.mode == 5).then(|| { + // Construct synchronously in F, before returning its future. + let timer = Box::pin(tokio::time::sleep(Duration::from_millis(1))); + fixture.events.send(Event::TimerConstruct( + std::thread::current().id(), + std::thread::current().name().unwrap_or("unnamed").to_owned(), + tokio::runtime::Handle::current().id(), + )).unwrap(); + timer + }); Box::pin(SetupFuture { fixture: fixture.clone(), mode: config.mode, first: true, + timer, }) as Pin> + Send>> }; let hooks = RuntimeHooks::new( @@ -1278,3 +1292,48 @@ async fn stop_during_reinit_parse_retains_received_config_until_physical_cleanup ); assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn linked_reinit_synchronous_timer_uses_retained_runtime_on_same_worker() { + let _serial = controlled::TEST_LOCK.lock().await; + let caller_runtime = tokio::runtime::Handle::current().id(); + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), serde_json::json!({"mode":5}), + ).unwrap(); + let mut observed = vec![events.recv_timeout(Duration::from_secs(3)).unwrap()]; + tokio::time::timeout(Duration::from_secs(3), async { + while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { + tokio::task::yield_now().await; + } + }).await.unwrap(); + let first = host.reinitialize("tinybus-module", serde_json::json!({"mode":5})).await; + let second = host.reinitialize("tinybus-module", serde_json::json!({"mode":5})).await; + let caller_restored = tokio::runtime::Handle::current().id() == caller_runtime; + let view = fixture.view.lock().unwrap().as_ref().unwrap().clone(); + host.shutdown(Duration::from_secs(8)).await; + let completion = view.inner.wait(Duration::from_secs(5)); + let after_stop = host.reinitialize("tinybus-module", serde_json::json!({"mode":5})).await; + observed.extend(events.try_iter()); + let constructors: Vec<_> = observed.iter().filter_map(|event| match event { + controlled::Event::TimerConstruct(thread, name, runtime) => Some((*thread, name.as_str(), *runtime)), + _ => None, + }).collect(); + // Actual caller, controller worker and SDK cleanup have settled before assertions, + // including the unfixed constructor-panic path. + assert!(first.is_ok() && second.is_ok(), "synchronous timer reinit failed: {first:?}, {second:?}"); + assert_eq!(completion, TerminalCompletion::PhysicallyStopped); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + assert_eq!(constructors.len(), 3); + assert_eq!(constructors[1].1, "tinybus-terminal"); + assert_eq!(constructors[2].1, "tinybus-terminal"); + assert_eq!(constructors[1].0, constructors[2].0); + assert!(constructors.iter().all(|entry| entry.2 == constructors[0].2)); + assert_ne!(constructors[0].2, caller_runtime); + assert!(caller_restored); + assert!(after_stop.is_err()); + assert_eq!(observed.iter().filter(|event| matches!(event, controlled::Event::Poll(5))).count(), 3); +} From e6bb76dc939709fed0d3eea0c15f0f7b0eb3d594 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:34:15 +0300 Subject: [PATCH 076/130] fix(tinybus-module): enter retained runtime during synchronous reinit The reinit worker now enters the retained runtime's context before running synchronous construction, so timers and other runtime-bound resources created by a module's constructor are bound to the same runtime that owns the module rather than the caller's. The terminal test was extended to assert the constructor runs on the retained runtime's worker. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 6 ++ .../tinybus-module/src/lib_terminal_tests.rs | 88 ++++++++++++++----- 2 files changed, 73 insertions(+), 21 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 92f198fa..e4bbea7e 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1593,6 +1593,7 @@ where let construct_state = Arc::downgrade(&state); let construct_setup = setup.clone(); let construct_slot = slot.clone(); + let construct_handle = handle.clone(); let code = state.construct_reinit(Box::new(move || { let Some(state) = construct_state.upgrade() else { return TB_CLOSED; @@ -1600,6 +1601,11 @@ where if !state.is_live() { return TB_CLOSED; } + // active_reinit keeps the actual State-owned Runtime alive until + // this worker acknowledges construction and the ABI caller settles. + // Enter its context locally, outside all locks, for synchronous C/F + // construction; the guard never leaves this physical worker. + let _runtime_scope = construct_handle.enter(); let config = match serde_json::from_slice::(&bytes) { Ok(config) => config, Err(_) => return TB_BAD_ARGUMENT, diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 69d03a18..2e0fd9d0 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -227,7 +227,13 @@ mod controlled { } match self.mode { 0 => Poll::Ready(Ok(())), - 5 => self.timer.as_mut().unwrap().as_mut().poll(cx).map(|()| Ok(())), + 5 => self + .timer + .as_mut() + .unwrap() + .as_mut() + .poll(cx) + .map(|()| Ok(())), 2 => panic!("controlled setup poll"), 3 => Poll::Ready(Err(Error::failed("controlled setup failure"))), _ => Poll::Pending, @@ -290,11 +296,17 @@ mod controlled { let timer = (config.mode == 5).then(|| { // Construct synchronously in F, before returning its future. let timer = Box::pin(tokio::time::sleep(Duration::from_millis(1))); - fixture.events.send(Event::TimerConstruct( - std::thread::current().id(), - std::thread::current().name().unwrap_or("unnamed").to_owned(), - tokio::runtime::Handle::current().id(), - )).unwrap(); + fixture + .events + .send(Event::TimerConstruct( + std::thread::current().id(), + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned(), + tokio::runtime::Handle::current().id(), + )) + .unwrap(); timer }); Box::pin(SetupFuture { @@ -1299,41 +1311,75 @@ async fn linked_reinit_synchronous_timer_uses_retained_runtime_on_same_worker() let caller_runtime = tokio::runtime::Handle::current().id(); let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); host.attach_linked_with_config( - controlled::linked_module().unwrap(), serde_json::json!({"mode":5}), - ).unwrap(); + controlled::linked_module().unwrap(), + serde_json::json!({"mode":5}), + ) + .unwrap(); let mut observed = vec![events.recv_timeout(Duration::from_secs(3)).unwrap()]; tokio::time::timeout(Duration::from_secs(3), async { - while !host.list().iter().any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { tokio::task::yield_now().await; } - }).await.unwrap(); - let first = host.reinitialize("tinybus-module", serde_json::json!({"mode":5})).await; - let second = host.reinitialize("tinybus-module", serde_json::json!({"mode":5})).await; + }) + .await + .unwrap(); + let first = host + .reinitialize("tinybus-module", serde_json::json!({"mode":5})) + .await; + let second = host + .reinitialize("tinybus-module", serde_json::json!({"mode":5})) + .await; let caller_restored = tokio::runtime::Handle::current().id() == caller_runtime; let view = fixture.view.lock().unwrap().as_ref().unwrap().clone(); host.shutdown(Duration::from_secs(8)).await; let completion = view.inner.wait(Duration::from_secs(5)); - let after_stop = host.reinitialize("tinybus-module", serde_json::json!({"mode":5})).await; + let after_stop = host + .reinitialize("tinybus-module", serde_json::json!({"mode":5})) + .await; observed.extend(events.try_iter()); - let constructors: Vec<_> = observed.iter().filter_map(|event| match event { - controlled::Event::TimerConstruct(thread, name, runtime) => Some((*thread, name.as_str(), *runtime)), - _ => None, - }).collect(); + let constructors: Vec<_> = observed + .iter() + .filter_map(|event| match event { + controlled::Event::TimerConstruct(thread, name, runtime) => { + Some((*thread, name.as_str(), *runtime)) + } + _ => None, + }) + .collect(); // Actual caller, controller worker and SDK cleanup have settled before assertions, // including the unfixed constructor-panic path. - assert!(first.is_ok() && second.is_ok(), "synchronous timer reinit failed: {first:?}, {second:?}"); + assert!( + first.is_ok() && second.is_ok(), + "synchronous timer reinit failed: {first:?}, {second:?}" + ); assert_eq!(completion, TerminalCompletion::PhysicallyStopped); assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); assert_eq!(constructors.len(), 3); assert_eq!(constructors[1].1, "tinybus-terminal"); assert_eq!(constructors[2].1, "tinybus-terminal"); assert_eq!(constructors[1].0, constructors[2].0); - assert!(constructors.iter().all(|entry| entry.2 == constructors[0].2)); + assert!( + constructors + .iter() + .all(|entry| entry.2 == constructors[0].2) + ); assert_ne!(constructors[0].2, caller_runtime); assert!(caller_restored); assert!(after_stop.is_err()); - assert_eq!(observed.iter().filter(|event| matches!(event, controlled::Event::Poll(5))).count(), 3); + assert_eq!( + observed + .iter() + .filter(|event| matches!(event, controlled::Event::Poll(5))) + .count(), + 3 + ); } From b7923e905c3aafbf3032c001793b7fc082bf49d4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:34:48 +0300 Subject: [PATCH 077/130] test(tinybus-module): assert timer cleanup runs outside a runtime Extend the synchronous-timer reinit test to record the runtime context observed when each timer is dropped, and assert that all three drops happen with no runtime entered. This guards against the constructor runtime guard leaking into later physical-worker cleanup. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 2e0fd9d0..022eb99d 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -164,6 +164,7 @@ mod controlled { ConfigDrop(String), ConfigParsing, TimerConstruct(std::thread::ThreadId, String, tokio::runtime::Id), + TimerDrop(Option), Finish, FactoryDrop(String), } @@ -242,6 +243,13 @@ mod controlled { } impl Drop for SetupFuture { fn drop(&mut self) { + if self.mode == 5 { + let _ = self.fixture.events.send(Event::TimerDrop( + tokio::runtime::Handle::try_current() + .ok() + .map(|handle| handle.id()), + )); + } if let Some(view) = self.fixture.view.lock().unwrap().as_ref() { let slot = view.inner.resources.lock().unwrap().reinitialize.clone(); if slot.try_lock().is_err() { @@ -1375,6 +1383,18 @@ async fn linked_reinit_synchronous_timer_uses_retained_runtime_on_same_worker() assert_ne!(constructors[0].2, caller_runtime); assert!(caller_restored); assert!(after_stop.is_err()); + let cleanup_contexts: Vec<_> = observed + .iter() + .filter_map(|event| match event { + controlled::Event::TimerDrop(context) => Some(context), + _ => None, + }) + .collect(); + assert_eq!(cleanup_contexts.len(), 3); + assert!( + cleanup_contexts.iter().all(|context| context.is_none()), + "constructor runtime guard must be restored before later physical-worker cleanup" + ); assert_eq!( observed .iter() From 14ec15f9c61153b2866ec93380f73a28f0db6396 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:44:57 +0300 Subject: [PATCH 078/130] test(module): cover native query flag gating and invalid config rejection The native query test now exercises every non-LIVE flag combination, including missing, empty, terminal, combined and unknown values, asserting that the factory is never evaluated and no output is published. A new test verifies that malformed initial config is rejected with TB_BAD_ARGUMENT before the factory runs or module context is published. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 64 ++++++++++++++++--- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 022eb99d..ff4b3783 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -37,13 +37,60 @@ fn output() -> TbModuleVtable { } #[test] fn native_query_is_required_before_terminal_factory_evaluation() { + use tinybus::module::abi::*; + // Exactly LIVE is the sole admitted value. Missing, empty, terminal, + // combined and unknown flags must not evaluate arbitrary factory code. + for flags in [ + None, + Some(0), + Some(TB_NATIVE_TERMINAL_STOP_INTENT), + Some(TB_NATIVE_TERMINAL_FAULT), + Some(TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED), + Some(TB_NATIVE_TERMINAL_LIVE | TB_NATIVE_TERMINAL_STOP_INTENT), + Some(TB_NATIVE_TERMINAL_LIVE | TB_NATIVE_TERMINAL_FAULT), + Some(TB_NATIVE_TERMINAL_LIVE | (1 << 31)), + Some(u32::MAX), + ] { + let native = std::sync::atomic::AtomicU32::new(flags.unwrap_or(0)); + let mut host = host(); + host.host_ctx = (&native as *const std::sync::atomic::AtomicU32).cast_mut().cast(); + host.native_terminal_state = flags.map(|_| native_query as unsafe extern "C" fn(*mut c_void) -> u32); + let calls = AtomicUsize::new(0); + let mut out = output(); + let code = unsafe { + start_terminal_reconfigurable_module_with_factory::( + &host, + &mut out, + 1, + true, + |_| { + calls.fetch_add(1, Ordering::SeqCst); + Ok(( + |_: Connection, _: serde_json::Value| async { Ok(()) }, + RuntimeHooks::new(|_| {}, || TerminalCompletion::PhysicallyStopped), + )) + }, + ) + }; + assert_eq!(code, TB_CLOSED, "query {flags:?}"); + assert_eq!(calls.load(Ordering::SeqCst), 0, "query {flags:?}"); + assert!(out.module_ctx.is_null(), "denied init cannot publish output"); + } +} + +#[test] +fn invalid_initial_config_rejects_before_factory_or_output_publication() { + let native = std::sync::atomic::AtomicU32::new(tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE); + let bytes = b"{"; + let mut host = host(); + host.host_ctx = (&native as *const std::sync::atomic::AtomicU32).cast_mut().cast(); + host.native_terminal_state = Some(native_query); + host.config = TbSlice { ptr: bytes.as_ptr(), len: bytes.len() }; let calls = AtomicUsize::new(0); + let mut out = output(); let code = unsafe { start_terminal_reconfigurable_module_with_factory::( - &host(), - &mut output(), - 1, - true, + &host, &mut out, 1, true, |_| { calls.fetch_add(1, Ordering::SeqCst); Ok(( @@ -53,12 +100,9 @@ fn native_query_is_required_before_terminal_factory_evaluation() { }, ) }; - assert_eq!(code, TB_CLOSED); - assert_eq!( - calls.load(Ordering::SeqCst), - 0, - "missing native query must leave the factory untouched" - ); + assert_eq!(code, TB_BAD_ARGUMENT); + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert!(out.module_ctx.is_null()); } unsafe extern "C" fn native_query(ctx: *mut c_void) -> u32 { From 6f63af77dfbf86bd6b69a4fab45045600c290cbd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:46:24 +0300 Subject: [PATCH 079/130] test(terminal): cover worker spawn failure and hook panic paths Adds test-only controls to force terminal worker spawn rejection and to panic from the setup and finish hooks, so the submitted-job ownership path and its cleanup can be exercised without touching process-wide thread creation. The fixture now tracks finishes, hook failures, and payload drops to assert that rejected or panicking workers are joined and their payloads released. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 32 ++++++++++++++++++- crates/tinybus-module/src/terminal.rs | 31 +++++++++++++++--- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index ff4b3783..e517bb10 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -219,6 +219,9 @@ mod controlled { worker: StdMutex>>, pub requests: AtomicUsize, pub joins: AtomicUsize, + pub finishes: AtomicUsize, + pub hook_failure: AtomicUsize, + pub payload_drops: AtomicUsize, pub config_drops: AtomicUsize, pub config_parse_control: StdMutex>>, pub drop_off_lock: AtomicBool, @@ -236,6 +239,9 @@ mod controlled { worker: StdMutex::new(None), requests: AtomicUsize::new(0), joins: AtomicUsize::new(0), + finishes: AtomicUsize::new(0), + hook_failure: AtomicUsize::new(0), + payload_drops: AtomicUsize::new(0), config_drops: AtomicUsize::new(0), config_parse_control: StdMutex::new(None), drop_off_lock: AtomicBool::new(true), @@ -246,6 +252,22 @@ mod controlled { *NEXT.lock().unwrap() = Some(fixture.clone()); (fixture, receiver) } + struct HookPayload(Arc); + impl Drop for HookPayload { + fn drop(&mut self) { + self.0.payload_drops.fetch_add(1, Ordering::SeqCst); + } + } + impl Fixture { + pub fn settle_rejected_worker(&self) { + *self.wake.0.lock().unwrap() = true; + self.wake.1.notify_all(); + if let Some(worker) = self.worker.lock().unwrap().take() { + let _ = worker.join().unwrap(); + self.joins.fetch_add(1, Ordering::SeqCst); + } + } + } struct Capture(Arc); impl Drop for Capture { fn drop(&mut self) { @@ -380,6 +402,9 @@ mod controlled { *request.wake.0.lock().unwrap() = true; request.wake.1.notify_all(); request.events.send(Event::Request).unwrap(); + if request.hook_failure.load(Ordering::Acquire) == 1 { + std::panic::panic_any(HookPayload(request.clone())); + } }, move || { let worker = finish.worker.lock().unwrap().take(); @@ -390,8 +415,13 @@ mod controlled { return TerminalCompletion::Uncertain; } } + finish.finishes.fetch_add(1, Ordering::SeqCst); finish.events.send(Event::Finish).unwrap(); - TerminalCompletion::PhysicallyStopped + match finish.hook_failure.load(Ordering::Acquire) { + 2 => std::panic::panic_any(HookPayload(finish.clone())), + 3 => TerminalCompletion::Uncertain, + _ => TerminalCompletion::PhysicallyStopped, + } }, ); Ok((Box::new(setup), hooks)) diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 45fe7ae6..f18e0e9c 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -223,6 +223,8 @@ pub(super) struct TerminalState { pub resources: StdMutex, sdk_owners: AtomicUsize, vtable_published: AtomicBool, + #[cfg(test)] + reject_worker_spawn: AtomicBool, lease: SlotLease, token: usize, } @@ -308,6 +310,8 @@ impl TerminalState { resources: StdMutex::new(StartupResources::default()), sdk_owners: AtomicUsize::new(0), vtable_published: AtomicBool::new(false), + #[cfg(test)] + reject_worker_spawn: AtomicBool::new(false), lease: SlotLease { admission: Arc::downgrade(admission), index, @@ -569,6 +573,14 @@ impl TerminalState { control.construction_code } } + #[cfg(test)] + pub(super) fn reject_worker_spawn_for_test(&self) { + self.reject_worker_spawn.store(true, Ordering::Release); + } + #[cfg(test)] + pub(super) fn request_done_for_test(&self) -> bool { + matches!(self.lock().request_state, RequestState::Done | RequestState::Panicked) + } fn schedule(self: &Arc) { let job = { let mut control = self.lock(); @@ -583,11 +595,20 @@ impl TerminalState { job }; let state = job.state.upgrade().expect("anchored terminal state"); - if std::thread::Builder::new() - .name("tinybus-terminal".into()) - .spawn(move || job.run(state)) - .is_err() - { + // Private per-instance rejection exercises the real submitted-job ownership + // path without changing process-wide thread creation or public authority. + #[cfg(test)] + let rejected = self.reject_worker_spawn.load(Ordering::Acquire); + #[cfg(not(test))] + let rejected = false; + let submitted = if rejected { + Err(std::io::Error::other("controlled terminal worker spawn failure")) + } else { + std::thread::Builder::new() + .name("tinybus-terminal".into()) + .spawn(move || job.run(state)) + }; + if submitted.is_err() { self.uncertain.store(true, Ordering::Release); let mut control = self.lock(); control.finish_state = FinishState::SpawnFailed; From 963072af4cde8f1f02938c6afa329cddd51424f3 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:47:35 +0300 Subject: [PATCH 080/130] test(module): cover linked hook panic and rejected spawn terminal paths Add integration tests exercising the real linked terminal lifecycle when a hook panics and when physical worker spawn is rejected. They assert that uncertain completion, once-only request/finish/join accounting, and anchored resource retention hold across shutdown retries and later fault upgrades. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index e517bb10..eef419d8 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -433,6 +433,104 @@ mod controlled { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_hook_panics_and_uncertain_finish_preserve_once_join_and_anchor() { + let _serial = controlled::TEST_LOCK.lock().await; + for mode in [1, 2, 3] { + let (fixture, events) = controlled::fixture(); + fixture.hook_failure.store(mode, Ordering::Release); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), serde_json::json!({"mode":1}), + ).unwrap(); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let context = view.inner.context(); + // Real setup has launched the independent controller worker. Actual + // request closes/wakes it, and original finish really joins it even + // when request panic has already made uncertainty sticky. + view.inner.signal(TerminalReason::Stop); + host.shutdown(Duration::from_secs(5)).await; + let completed = view.inner.wait(Duration::from_secs(5)); + let observations: Vec<_> = events.try_iter().collect(); + let native = view.inner.host.0; + let flags = unsafe { native.native_terminal_state.unwrap()(native.host_ctx) }; + // Controlled work and actual original finish/join have settled before + // assertions. Uncertain primary payload/resource ownership stays anchored. + assert!(matches!(entered, controlled::Event::Poll(1))); + assert_eq!(completed, TerminalCompletion::Uncertain, "hook mode {mode}"); + assert!(view.inner.request_done_for_test()); + assert_eq!(fixture.requests.load(Ordering::Acquire), 1); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 1); + assert_eq!(fixture.joins.load(Ordering::Acquire), 1); + assert_eq!(fixture.payload_drops.load(Ordering::Acquire), 0); + assert!(!view.is_live()); + assert!(terminal::TerminalState::lookup(context).is_some()); + if mode != 3 { + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + } + let future_drop = observations.iter().position(|event| matches!(event, + controlled::Event::FutureDrop(name) if name == "tinybus-terminal")).unwrap(); + let finish = observations.iter().position(|event| matches!(event, controlled::Event::Finish)).unwrap(); + assert!(future_drop < finish); + // Neither a later cause upgrade nor shutdown retry spends hooks again. + view.request_fault(); + host.shutdown(Duration::from_millis(20)).await; + assert_eq!(fixture.requests.load(Ordering::Acquire), 1); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 1); + assert_eq!(fixture.payload_drops.load(Ordering::Acquire), 0); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_rejected_physical_spawn_retains_received_resources_without_retry() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), serde_json::json!({"mode":1}), + ).unwrap(); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let context = view.inner.context(); + view.inner.reject_worker_spawn_for_test(); + view.request_fault(); + host.shutdown(Duration::from_secs(3)).await; + // Submission failure cannot execute original finish. The fixture alone + // releases/joins its independent worker; this is not SDK finish evidence. + fixture.settle_rejected_worker(); + let completed = view.inner.wait(Duration::from_secs(1)); + let observations: Vec<_> = events.try_iter().collect(); + let native = view.inner.host.0; + let flags = unsafe { native.native_terminal_state.unwrap()(native.host_ctx) }; + assert!(matches!(entered, controlled::Event::Poll(1))); + assert_eq!(completed, TerminalCompletion::Uncertain); + assert!(view.inner.request_done_for_test()); + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!(fixture.requests.load(Ordering::Acquire), 1); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 0); + assert_eq!(fixture.joins.load(Ordering::Acquire), 1); + assert!(!observations.iter().any(|event| matches!(event, + controlled::Event::FactoryDrop(_) | controlled::Event::FutureDrop(_) | controlled::Event::Finish))); + assert!(!view.is_live()); + drop(host); + view.request_fault(); + assert_eq!(view.inner.wait(Duration::from_millis(20)), TerminalCompletion::Uncertain); + assert_eq!(fixture.requests.load(Ordering::Acquire), 1); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 0); + let anchored = terminal::TerminalState::lookup(context).unwrap(); + let resources = anchored.resources.lock().unwrap(); + assert!(resources.retained_factory.is_some()); + assert!(resources.runtime.is_some()); + assert!(resources.initial.is_some()); + assert!(resources.initial_future.lock().unwrap().is_some()); + // One deliberately retained, bounded original State/job/Runtime remains. +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn actual_linked_initial_cancellation_drops_future_and_joins_worker_on_physical_thread() { let _serial = controlled::TEST_LOCK.lock().await; From aecd95c268a762198f62f06d8e1fd998a846e1d4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:48:40 +0300 Subject: [PATCH 081/130] test(module): cover linked factory failure config disposal Add a controlled factory failure mode to the terminal test fixture and a test asserting that a config received before a factory error or panic is dropped on the original physical worker, with no hooks transferred and no setup task launched. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index eef419d8..e36037ea 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -221,6 +221,7 @@ mod controlled { pub joins: AtomicUsize, pub finishes: AtomicUsize, pub hook_failure: AtomicUsize, + pub factory_failure: AtomicUsize, pub payload_drops: AtomicUsize, pub config_drops: AtomicUsize, pub config_parse_control: StdMutex>>, @@ -241,6 +242,7 @@ mod controlled { joins: AtomicUsize::new(0), finishes: AtomicUsize::new(0), hook_failure: AtomicUsize::new(0), + factory_failure: AtomicUsize::new(0), payload_drops: AtomicUsize::new(0), config_drops: AtomicUsize::new(0), config_parse_control: StdMutex::new(None), @@ -341,6 +343,11 @@ mod controlled { fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> { let fixture = NEXT.lock().unwrap().take().unwrap(); *fixture.view.lock().unwrap() = Some(view.clone()); + match fixture.factory_failure.load(Ordering::Acquire) { + 1 => return Err(Error::failed("controlled factory failure before hook transfer")), + 2 => std::panic::panic_any(HookPayload(fixture.clone())), + _ => {} + } let (request, finish) = (fixture.clone(), fixture.clone()); let capture = Capture(fixture.clone()); let setup = move |_: Connection, config: Config| { @@ -433,6 +440,54 @@ mod controlled { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_factory_failure_disposes_received_config_on_physical_worker() { + let _serial = controlled::TEST_LOCK.lock().await; + for mode in [1, 2] { + let (fixture, events) = controlled::fixture(); + fixture.factory_failure.store(mode, Ordering::Release); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + let rejected = host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":1,"record_drop":true}), + ); + let view = fixture.view.lock().unwrap().take().unwrap(); + let completed = view.inner.wait(Duration::from_secs(5)); + host.shutdown(Duration::from_secs(3)).await; + let observations: Vec<_> = events.try_iter().collect(); + let native = view.inner.host.0; + let flags = unsafe { native.native_terminal_state.unwrap()(native.host_ctx) }; + // No controller hooks have transferred, and no setup task was launched. + // Parsed C already belongs to SDK: its actual Drop must occur on the + // original physical job even though no output vtable was returned. + assert!(rejected.is_err()); + assert_eq!(completed, if mode == 1 { + TerminalCompletion::PhysicallyStopped + } else { + TerminalCompletion::Uncertain + }); + assert_eq!(fixture.config_drops.load(Ordering::Acquire), 1); + assert_eq!(fixture.requests.load(Ordering::Acquire), 0); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 0); + assert_eq!(fixture.joins.load(Ordering::Acquire), 0); + assert_eq!(fixture.payload_drops.load(Ordering::Acquire), 0); + assert!(observations.iter().all(|event| matches!(event, + controlled::Event::ConfigDrop(name) if name == "tinybus-terminal"))); + assert_eq!(observations.len(), 1); + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0, + "SDK failed-init cleanup cannot fabricate host physical completion"); + assert!(!view.is_live()); + if mode == 1 { + assert!(terminal::TerminalState::lookup(view.inner.context()).is_none()); + } else { + assert!(terminal::TerminalState::lookup(view.inner.context()).is_some()); + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn actual_linked_hook_panics_and_uncertain_finish_preserve_once_join_and_anchor() { let _serial = controlled::TEST_LOCK.lock().await; From 799c1be9ee32155a14d8d2370479c492ff58937c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:51:07 +0300 Subject: [PATCH 082/130] test(terminal): cover native fault fencing before blocked payload drop Add a test that drives a real served method into a foreign payload Drop while the module shuts down, asserting the native fault flag, request completion and SDK owner retention are observed before the payload is released. Expose the SDK owner count behind cfg(test) so the test can check that the method connection keeps the state alive. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 103 ++++++++++++++++++ crates/tinybus-module/src/terminal.rs | 4 + 2 files changed, 107 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index e36037ea..7d0f9af9 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -1033,6 +1033,109 @@ async fn actual_linked_reinit_terminal_cancellation_settles_before_worker_join() assert!(drops.into_iter().all(|index| index < finish)); } +struct MethodPayloadGate { + state: std::sync::Weak, + entered: std::sync::mpsc::Sender<(u32, bool, usize, tokio::runtime::Id)>, + release: Arc, + settled: std::sync::mpsc::Sender<()>, +} +struct BlockingMethodPayload(Arc); +impl Drop for BlockingMethodPayload { + fn drop(&mut self) { + let state = self.0.state.upgrade().unwrap(); + let host = state.host.0; + let flags = unsafe { host.native_terminal_state.unwrap()(host.host_ctx) }; + // This sample is the first foreign payload-disposal action. It reads + // actual copied native authority and the real registered request state. + self.0.entered.send(( + flags, state.request_done_for_test(), state.sdk_owner_count_for_test(), + tokio::runtime::Handle::current().id(), + )).unwrap(); + drop(state); + self.0.release.wait(); + self.0.settled.send(()).unwrap(); + } +} +struct MethodPayloadObject(Arc); +#[async_trait] +impl tinybus::Interface for MethodPayloadObject { + fn name(&self) -> tinybus::InterfaceName { + "ai.tinyhumans.tinybus.Payload".parse().unwrap() + } + fn members(&self) -> Vec { + vec!["Panic".parse().unwrap()] + } + async fn call(&self, _: &tinybus::MemberName, _: serde_json::Value) -> Result { + std::panic::panic_any(BlockingMethodPayload(self.0.clone())); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_method_observer_fences_native_fault_and_request_done_before_blocked_payload_drop() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + let bus = tinybus::transport::memory::MemoryBus::new(); + broker.clone().spawn(bus.clone()); + let client = Connection::connect(bus.connect().await.unwrap()).await.unwrap(); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), serde_json::json!({"mode":0}), + ).unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (settled_tx, settled_rx) = std::sync::mpsc::channel(); + let release = Arc::new(std::sync::Barrier::new(2)); + let gate = Arc::new(MethodPayloadGate { + state: Arc::downgrade(&view.inner), entered: entered_tx, + release: release.clone(), settled: settled_tx, + }); + let (connection, module_runtime) = { + let resources = view.inner.resources.lock().unwrap(); + let connection = resources.connection.lock().unwrap().clone().unwrap(); + (connection, resources.runtime.as_ref().unwrap().handle().id()) + }; + let destination = connection.unique_name().unwrap().to_string(); + connection.serve_at("/payload".parse().unwrap(), MethodPayloadObject(gate)).await.unwrap(); + drop(connection); + let call = tokio::spawn(async move { + client.proxy(&destination, "/payload", "ai.tinyhumans.tinybus.Payload") + .unwrap().call::<_, serde_json::Value>("Panic", ()).await + }); + let sample = entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + // Actual method task is held in foreign payload Drop. Its Connection owner + // survives runtime shutdown_timeout and therefore denies SDK completion. + host.shutdown(Duration::from_secs(8)).await; + let completion = view.inner.wait(Duration::ZERO); + let anchored = terminal::TerminalState::lookup(view.inner.context()).is_some(); + let retained_sdk = view.inner.sdk_owner_count_for_test(); + let native = view.inner.host.0; + let flags = unsafe { native.native_terminal_state.unwrap()(native.host_ctx) }; + release.wait(); + settled_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + let call_result = call.await.unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while view.inner.sdk_owner_count_for_test() != 0 { + tokio::task::yield_now().await; + } + }).await.unwrap(); + // All controlled payload work, actual method caller and controller worker + // have settled before assertions; the conservative original anchor remains. + assert_ne!(sample.0 & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert!(sample.1, "installed observer must finish bounded request before foreign Drop"); + assert!(sample.2 > 0, "actual method Connection/task must retain SDK State owner"); + assert_eq!(sample.3, module_runtime); + assert_eq!(completion, TerminalCompletion::Uncertain); + assert!(anchored && retained_sdk > 0); + assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); + assert_eq!(fixture.requests.load(Ordering::Acquire), 1); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 1); + assert_eq!(fixture.joins.load(Ordering::Acquire), 1); + assert!(call_result.is_err()); + assert!(terminal::TerminalState::lookup(view.inner.context()).is_some()); +} + // A real served object, rather than a synthetic transport owner: Inner destroys // its transport before its ObjectTree, so the old count can reach zero here. struct TeardownObject { diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index f18e0e9c..8c7d28b2 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -578,6 +578,10 @@ impl TerminalState { self.reject_worker_spawn.store(true, Ordering::Release); } #[cfg(test)] + pub(super) fn sdk_owner_count_for_test(&self) -> usize { + self.sdk_owners.load(Ordering::Acquire) + } + #[cfg(test)] pub(super) fn request_done_for_test(&self) -> bool { matches!(self.lock().request_state, RequestState::Done | RequestState::Panicked) } From f973d743d70eeb08f8a95d9cfba216d76891654d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:51:39 +0300 Subject: [PATCH 083/130] test(tinybus-module): drop unused generic from proxy call The proxy call in the native fault fencing test now specifies only the response type, matching the current call signature. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 7d0f9af9..6c7fc2e8 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -1101,7 +1101,7 @@ async fn actual_method_observer_fences_native_fault_and_request_done_before_bloc drop(connection); let call = tokio::spawn(async move { client.proxy(&destination, "/payload", "ai.tinyhumans.tinybus.Payload") - .unwrap().call::<_, serde_json::Value>("Panic", ()).await + .unwrap().call::("Panic", ()).await }); let sample = entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); // Actual method task is held in foreign payload Drop. Its Connection owner From 684aeb88190ccb9a6dba276b56a7167d2dbc9bb2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:54:47 +0300 Subject: [PATCH 084/130] chore: files changed crates/tinybus-module/src/lib.rs,crates/tinybus-module/src/lib_terminal_tests.r Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 16 ++++++ .../tinybus-module/src/lib_terminal_tests.rs | 54 ++++++++++++++++++- crates/tinybus-module/src/terminal.rs | 14 +++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index e4bbea7e..3a6b5a44 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1530,6 +1530,8 @@ where .unwrap_or_else(std::sync::PoisonError::into_inner) .retained_factory = Some(Box::new(setup.clone())); state.register(hooks); + #[cfg(test)] + state.startup_checkpoint_for_test(1); if !state.is_live() { return TB_CLOSED; } @@ -1551,6 +1553,8 @@ where .setup = Some(Box::new(move |connection| { Box::pin(initial_setup(connection, config)) })); + #[cfg(test)] + state.startup_checkpoint_for_test(2); let weak = Arc::downgrade(&state); let reinitializer: ReinitializeFactory = Box::new(move |handle, connection_slot| { Box::new(move |ptr, len| { @@ -1656,6 +1660,8 @@ where .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .reinitializer_factory = Some(reinitializer); + #[cfg(test)] + state.startup_checkpoint_for_test(3); let mut builder = tokio::runtime::Builder::new_multi_thread(); if let Some(owner) = &state.owner { owner.configure(&mut builder); @@ -1673,6 +1679,8 @@ where .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .runtime = Some(runtime); + #[cfg(test)] + state.startup_checkpoint_for_test(4); let (inbound_tx, inbound_rx) = mpsc::channel(MODULE_QUEUE_CAPACITY); state .resources @@ -1688,6 +1696,8 @@ where }, _owner: state.sdk_owner(), }); + #[cfg(test)] + state.startup_checkpoint_for_test(5); let (handle, connection_slot, factory) = { let mut resources = state .resources @@ -1716,6 +1726,8 @@ where *callback_slot .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) = callback; + #[cfg(test)] + state.startup_checkpoint_for_test(6); let task_state = state.clone(); let task = handle.spawn(async move { let provenance = transport.inner.provenance.clone(); @@ -1785,7 +1797,11 @@ where .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .initial = Some(task); + #[cfg(test)] + state.startup_checkpoint_for_test(7); let context = state.context(); + #[cfg(test)] + state.startup_checkpoint_for_test(8); state.mark_vtable_published(); if capacity >= size_of::() as u32 { unsafe { diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 6c7fc2e8..7bcda2db 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -222,6 +222,7 @@ mod controlled { pub finishes: AtomicUsize, pub hook_failure: AtomicUsize, pub factory_failure: AtomicUsize, + pub startup_failure: AtomicUsize, pub payload_drops: AtomicUsize, pub config_drops: AtomicUsize, pub config_parse_control: StdMutex>>, @@ -243,6 +244,7 @@ mod controlled { finishes: AtomicUsize::new(0), hook_failure: AtomicUsize::new(0), factory_failure: AtomicUsize::new(0), + startup_failure: AtomicUsize::new(0), payload_drops: AtomicUsize::new(0), config_drops: AtomicUsize::new(0), config_parse_control: StdMutex::new(None), @@ -343,6 +345,7 @@ mod controlled { fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> { let fixture = NEXT.lock().unwrap().take().unwrap(); *fixture.view.lock().unwrap() = Some(view.clone()); + view.inner.reject_startup_for_test(fixture.startup_failure.load(Ordering::Acquire) as u8); match fixture.factory_failure.load(Ordering::Acquire) { 1 => return Err(Error::failed("controlled factory failure before hook transfer")), 2 => std::panic::panic_any(HookPayload(fixture.clone())), @@ -440,6 +443,55 @@ mod controlled { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_posttransfer_startup_panics_keep_received_captures_on_physical_worker() { + let _serial = controlled::TEST_LOCK.lock().await; + // Stages correspond to real attach boundaries in start_terminal_factory_mode: + // returned F/hooks, initial setup, reinit factory, Runtime, inbound/transport, + // installed callback, sole JoinHandle, immediately before output publication. + for stage in 1..=8 { + let (fixture, events) = controlled::fixture(); + fixture.startup_failure.store(stage, Ordering::Release); + let broker = tinybus::broker::Broker::new(); + broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + let rejected = host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":1,"record_drop":true}), + ); + let view = fixture.view.lock().unwrap().take().unwrap(); + let completion = view.inner.wait(Duration::from_secs(5)); + host.shutdown(Duration::from_secs(3)).await; + let observations: Vec<_> = events.try_iter().collect(); + let native = view.inner.host.0; + let flags = unsafe { native.native_terminal_state.unwrap()(native.host_ctx) }; + // Actual setup execution is aborted/joined before original finish and + // before received F/C disposal; early stages legitimately launch no work. + assert!(rejected.is_err(), "stage {stage}"); + assert_eq!(completion, TerminalCompletion::Uncertain, "stage {stage}"); + assert!(view.inner.request_done_for_test()); + assert_eq!(fixture.requests.load(Ordering::Acquire), 1, "stage {stage}"); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 1, "stage {stage}"); + assert_eq!(fixture.config_drops.load(Ordering::Acquire), 1, "stage {stage}"); + assert!(observations.iter().any(|event| matches!(event, + controlled::Event::FactoryDrop(name) if name == "tinybus-terminal")), "stage {stage}"); + assert!(observations.iter().filter_map(|event| match event { + controlled::Event::ConfigDrop(name) | controlled::Event::FutureDrop(name) => Some(name), + _ => None, + }).all(|name| name == "tinybus-terminal"), "stage {stage}"); + let launched = observations.iter().any(|event| matches!(event, controlled::Event::Poll(_))); + assert_eq!(fixture.joins.load(Ordering::Acquire), usize::from(launched), "stage {stage}"); + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); + assert!(terminal::TerminalState::lookup(view.inner.context()).is_some()); + let resources = view.inner.resources.lock().unwrap(); + assert!(resources.runtime.is_none()); + assert!(resources.initial.is_none()); + assert!(resources.retained_factory.is_none()); + // Sticky Startup primary keeps exactly this bounded State anchored. + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn actual_linked_factory_failure_disposes_received_config_on_physical_worker() { let _serial = controlled::TEST_LOCK.lock().await; @@ -1101,7 +1153,7 @@ async fn actual_method_observer_fences_native_fault_and_request_done_before_bloc drop(connection); let call = tokio::spawn(async move { client.proxy(&destination, "/payload", "ai.tinyhumans.tinybus.Payload") - .unwrap().call::("Panic", ()).await + .unwrap().with_timeout(Duration::from_secs(8)).call::("Panic", ()).await }); let sample = entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); // Actual method task is held in foreign payload Drop. Its Connection owner diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 8c7d28b2..160e589a 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -225,6 +225,8 @@ pub(super) struct TerminalState { vtable_published: AtomicBool, #[cfg(test)] reject_worker_spawn: AtomicBool, + #[cfg(test)] + startup_failure_stage: AtomicU8, lease: SlotLease, token: usize, } @@ -312,6 +314,8 @@ impl TerminalState { vtable_published: AtomicBool::new(false), #[cfg(test)] reject_worker_spawn: AtomicBool::new(false), + #[cfg(test)] + startup_failure_stage: AtomicU8::new(0), lease: SlotLease { admission: Arc::downgrade(admission), index, @@ -574,6 +578,16 @@ impl TerminalState { } } #[cfg(test)] + pub(super) fn reject_startup_for_test(&self, stage: u8) { + self.startup_failure_stage.store(stage, Ordering::Release); + } + #[cfg(test)] + pub(super) fn startup_checkpoint_for_test(&self, stage: u8) { + if self.startup_failure_stage.load(Ordering::Acquire) == stage { + panic!("controlled startup failure after transfer stage {stage}"); + } + } + #[cfg(test)] pub(super) fn reject_worker_spawn_for_test(&self) { self.reject_worker_spawn.store(true, Ordering::Release); } From e4cb25ddecc353757c029b438537a139362e7057 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:55:59 +0300 Subject: [PATCH 085/130] test(terminal): assert transport drops on the terminal thread Add test-only instrumentation that records the thread on which the terminal transport is dropped, and assert in the startup panic tests that stages 5 and 6 drop it on the tinybus-terminal thread. This pins down the ownership guarantee that the transport is released on the terminal thread rather than the caller's. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 6 ++++++ crates/tinybus-module/src/lib_terminal_tests.rs | 4 ++++ crates/tinybus-module/src/terminal.rs | 14 ++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 3a6b5a44..e4951d1c 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1311,6 +1311,12 @@ struct TerminalTransport { inner: ModuleTransport, _owner: terminal::SdkOwner, } +#[cfg(test)] +impl Drop for TerminalTransport { + fn drop(&mut self) { + self._owner.record_transport_drop_for_test(); + } +} #[async_trait] impl Transport for TerminalTransport { async fn send(&self, message: Message) -> Result<()> { diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 7bcda2db..e4e8fb0d 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -484,6 +484,10 @@ async fn actual_linked_posttransfer_startup_panics_keep_received_captures_on_phy assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); assert!(terminal::TerminalState::lookup(view.inner.context()).is_some()); + if stage == 5 || stage == 6 { + assert_eq!(view.inner.transport_drop_thread_for_test().as_deref(), + Some("tinybus-terminal"), "actual startup transport stage {stage}"); + } let resources = view.inner.resources.lock().unwrap(); assert!(resources.runtime.is_none()); assert!(resources.initial.is_none()); diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 160e589a..70992da7 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -227,6 +227,8 @@ pub(super) struct TerminalState { reject_worker_spawn: AtomicBool, #[cfg(test)] startup_failure_stage: AtomicU8, + #[cfg(test)] + transport_drop_thread: StdMutex>, lease: SlotLease, token: usize, } @@ -236,6 +238,12 @@ pub(super) struct SdkOwner { state: Arc, } impl SdkOwner { + #[cfg(test)] + pub(super) fn record_transport_drop_for_test(&self) { + *self.state.transport_drop_thread.lock().unwrap() = Some( + std::thread::current().name().unwrap_or("unnamed").to_owned(), + ); + } pub fn caught_panic(&self) { self.state.mark_uncertain(); self.state.signal(TerminalReason::Fault); @@ -316,6 +324,8 @@ impl TerminalState { reject_worker_spawn: AtomicBool::new(false), #[cfg(test)] startup_failure_stage: AtomicU8::new(0), + #[cfg(test)] + transport_drop_thread: StdMutex::new(None), lease: SlotLease { admission: Arc::downgrade(admission), index, @@ -578,6 +588,10 @@ impl TerminalState { } } #[cfg(test)] + pub(super) fn transport_drop_thread_for_test(&self) -> Option { + self.transport_drop_thread.lock().unwrap().clone() + } + #[cfg(test)] pub(super) fn reject_startup_for_test(&self, stage: u8) { self.startup_failure_stage.store(stage, Ordering::Release); } From 4b2e0915e33b2c4b0b23b71c21141a108aeb68db Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:57:37 +0300 Subject: [PATCH 086/130] fix(module): retain sdk transport across startup failures Store the received SDK transport in the startup resources so it is attached before any later fallible startup boundary, and release it during terminal teardown. Previously the ABI/task local could be the sole owner of the transport, so a failure after receipt could drop it prematurely. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 7 +++++++ crates/tinybus-module/src/terminal.rs | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index e4951d1c..329c935f 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1702,6 +1702,13 @@ where }, _owner: state.sdk_owner(), }); + // Attach before any later fallible startup boundary. The ABI/task local + // is never the sole owner of the received SDK transport. + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .transport = Some(transport.clone()); #[cfg(test)] state.startup_checkpoint_for_test(5); let (handle, connection_slot, factory) = { diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 70992da7..a3717aae 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -191,6 +191,7 @@ pub(super) struct StartupResources { pub reinitialize: Arc>>, pub connection: Arc>>, pub inbound: Option>, + pub transport: Option>, pub initial: Option>>, pub initial_future: SetupFutureSlot, pub reinit_future: SetupFutureSlot, @@ -206,6 +207,7 @@ impl Default for StartupResources { reinitialize: Arc::new(StdMutex::new(None)), connection: Arc::new(StdMutex::new(None)), inbound: None, + transport: None, initial: None, initial_future: Arc::new(StdMutex::new(None)), reinit_future: Arc::new(StdMutex::new(None)), @@ -908,6 +910,13 @@ impl TerminalState { .inbound .take(); drop(inbound); + let transport = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .transport + .take(); + drop(transport); let runtime = self .resources .lock() From 96fa5b1521ad48daa12e533b27873f7972d7cd19 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 18:58:09 +0300 Subject: [PATCH 087/130] chore: files changed crates/tinybus-module/src/lib_terminal_tests.rs,crates/tinybus-module/src/termi Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 232 +++++++++++++----- crates/tinybus-module/src/terminal.rs | 14 +- 2 files changed, 184 insertions(+), 62 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index e4e8fb0d..d9e7d8b6 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -53,8 +53,11 @@ fn native_query_is_required_before_terminal_factory_evaluation() { ] { let native = std::sync::atomic::AtomicU32::new(flags.unwrap_or(0)); let mut host = host(); - host.host_ctx = (&native as *const std::sync::atomic::AtomicU32).cast_mut().cast(); - host.native_terminal_state = flags.map(|_| native_query as unsafe extern "C" fn(*mut c_void) -> u32); + host.host_ctx = (&native as *const std::sync::atomic::AtomicU32) + .cast_mut() + .cast(); + host.native_terminal_state = + flags.map(|_| native_query as unsafe extern "C" fn(*mut c_void) -> u32); let calls = AtomicUsize::new(0); let mut out = output(); let code = unsafe { @@ -74,7 +77,10 @@ fn native_query_is_required_before_terminal_factory_evaluation() { }; assert_eq!(code, TB_CLOSED, "query {flags:?}"); assert_eq!(calls.load(Ordering::SeqCst), 0, "query {flags:?}"); - assert!(out.module_ctx.is_null(), "denied init cannot publish output"); + assert!( + out.module_ctx.is_null(), + "denied init cannot publish output" + ); } } @@ -83,14 +89,22 @@ fn invalid_initial_config_rejects_before_factory_or_output_publication() { let native = std::sync::atomic::AtomicU32::new(tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE); let bytes = b"{"; let mut host = host(); - host.host_ctx = (&native as *const std::sync::atomic::AtomicU32).cast_mut().cast(); + host.host_ctx = (&native as *const std::sync::atomic::AtomicU32) + .cast_mut() + .cast(); host.native_terminal_state = Some(native_query); - host.config = TbSlice { ptr: bytes.as_ptr(), len: bytes.len() }; + host.config = TbSlice { + ptr: bytes.as_ptr(), + len: bytes.len(), + }; let calls = AtomicUsize::new(0); let mut out = output(); let code = unsafe { start_terminal_reconfigurable_module_with_factory::( - &host, &mut out, 1, true, + &host, + &mut out, + 1, + true, |_| { calls.fetch_add(1, Ordering::SeqCst); Ok(( @@ -219,6 +233,7 @@ mod controlled { worker: StdMutex>>, pub requests: AtomicUsize, pub joins: AtomicUsize, + pub workers_started: AtomicUsize, pub finishes: AtomicUsize, pub hook_failure: AtomicUsize, pub factory_failure: AtomicUsize, @@ -241,6 +256,7 @@ mod controlled { worker: StdMutex::new(None), requests: AtomicUsize::new(0), joins: AtomicUsize::new(0), + workers_started: AtomicUsize::new(0), finishes: AtomicUsize::new(0), hook_failure: AtomicUsize::new(0), factory_failure: AtomicUsize::new(0), @@ -345,9 +361,14 @@ mod controlled { fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> { let fixture = NEXT.lock().unwrap().take().unwrap(); *fixture.view.lock().unwrap() = Some(view.clone()); - view.inner.reject_startup_for_test(fixture.startup_failure.load(Ordering::Acquire) as u8); + view.inner + .reject_startup_for_test(fixture.startup_failure.load(Ordering::Acquire) as u8); match fixture.factory_failure.load(Ordering::Acquire) { - 1 => return Err(Error::failed("controlled factory failure before hook transfer")), + 1 => { + return Err(Error::failed( + "controlled factory failure before hook transfer", + )); + } 2 => std::panic::panic_any(HookPayload(fixture.clone())), _ => {} } @@ -372,6 +393,7 @@ mod controlled { }) .unwrap(), ); + fixture.workers_started.fetch_add(1, Ordering::Release); } drop(worker); if config.mode == 4 { @@ -453,7 +475,9 @@ async fn actual_linked_posttransfer_startup_panics_keep_received_captures_on_phy let (fixture, events) = controlled::fixture(); fixture.startup_failure.store(stage, Ordering::Release); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); let rejected = host.attach_linked_with_config( controlled::linked_module().unwrap(), @@ -472,21 +496,44 @@ async fn actual_linked_posttransfer_startup_panics_keep_received_captures_on_phy assert!(view.inner.request_done_for_test()); assert_eq!(fixture.requests.load(Ordering::Acquire), 1, "stage {stage}"); assert_eq!(fixture.finishes.load(Ordering::Acquire), 1, "stage {stage}"); - assert_eq!(fixture.config_drops.load(Ordering::Acquire), 1, "stage {stage}"); - assert!(observations.iter().any(|event| matches!(event, - controlled::Event::FactoryDrop(name) if name == "tinybus-terminal")), "stage {stage}"); - assert!(observations.iter().filter_map(|event| match event { - controlled::Event::ConfigDrop(name) | controlled::Event::FutureDrop(name) => Some(name), - _ => None, - }).all(|name| name == "tinybus-terminal"), "stage {stage}"); - let launched = observations.iter().any(|event| matches!(event, controlled::Event::Poll(_))); - assert_eq!(fixture.joins.load(Ordering::Acquire), usize::from(launched), "stage {stage}"); + assert_eq!( + fixture.config_drops.load(Ordering::Acquire), + 1, + "stage {stage}" + ); + assert!( + observations.iter().any(|event| matches!(event, + controlled::Event::FactoryDrop(name) if name == "tinybus-terminal")), + "stage {stage}" + ); + assert!( + observations + .iter() + .filter_map(|event| match event { + controlled::Event::ConfigDrop(name) | controlled::Event::FutureDrop(name) => + Some(name), + _ => None, + }) + .all(|name| name == "tinybus-terminal"), + "stage {stage}" + ); + assert_eq!( + fixture.joins.load(Ordering::Acquire), + fixture.workers_started.load(Ordering::Acquire), + "stage {stage}" + ); assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); - assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); assert!(terminal::TerminalState::lookup(view.inner.context()).is_some()); if stage == 5 || stage == 6 { - assert_eq!(view.inner.transport_drop_thread_for_test().as_deref(), - Some("tinybus-terminal"), "actual startup transport stage {stage}"); + assert_eq!( + view.inner.transport_drop_thread_for_test().as_deref(), + Some("tinybus-terminal"), + "actual startup transport stage {stage}" + ); } let resources = view.inner.resources.lock().unwrap(); assert!(resources.runtime.is_none()); @@ -503,7 +550,9 @@ async fn actual_linked_factory_failure_disposes_received_config_on_physical_work let (fixture, events) = controlled::fixture(); fixture.factory_failure.store(mode, Ordering::Release); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); let rejected = host.attach_linked_with_config( controlled::linked_module().unwrap(), @@ -519,11 +568,14 @@ async fn actual_linked_factory_failure_disposes_received_config_on_physical_work // Parsed C already belongs to SDK: its actual Drop must occur on the // original physical job even though no output vtable was returned. assert!(rejected.is_err()); - assert_eq!(completed, if mode == 1 { - TerminalCompletion::PhysicallyStopped - } else { - TerminalCompletion::Uncertain - }); + assert_eq!( + completed, + if mode == 1 { + TerminalCompletion::PhysicallyStopped + } else { + TerminalCompletion::Uncertain + } + ); assert_eq!(fixture.config_drops.load(Ordering::Acquire), 1); assert_eq!(fixture.requests.load(Ordering::Acquire), 0); assert_eq!(fixture.finishes.load(Ordering::Acquire), 0); @@ -533,8 +585,11 @@ async fn actual_linked_factory_failure_disposes_received_config_on_physical_work controlled::Event::ConfigDrop(name) if name == "tinybus-terminal"))); assert_eq!(observations.len(), 1); assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); - assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0, - "SDK failed-init cleanup cannot fabricate host physical completion"); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0, + "SDK failed-init cleanup cannot fabricate host physical completion" + ); assert!(!view.is_live()); if mode == 1 { assert!(terminal::TerminalState::lookup(view.inner.context()).is_none()); @@ -551,11 +606,15 @@ async fn actual_linked_hook_panics_and_uncertain_finish_preserve_once_join_and_a let (fixture, events) = controlled::fixture(); fixture.hook_failure.store(mode, Ordering::Release); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); host.attach_linked_with_config( - controlled::linked_module().unwrap(), serde_json::json!({"mode":1}), - ).unwrap(); + controlled::linked_module().unwrap(), + serde_json::json!({"mode":1}), + ) + .unwrap(); let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); let view = fixture.view.lock().unwrap().take().unwrap(); let context = view.inner.context(); @@ -582,9 +641,17 @@ async fn actual_linked_hook_panics_and_uncertain_finish_preserve_once_join_and_a if mode != 3 { assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); } - let future_drop = observations.iter().position(|event| matches!(event, - controlled::Event::FutureDrop(name) if name == "tinybus-terminal")).unwrap(); - let finish = observations.iter().position(|event| matches!(event, controlled::Event::Finish)).unwrap(); + let future_drop = observations + .iter() + .position(|event| { + matches!(event, + controlled::Event::FutureDrop(name) if name == "tinybus-terminal") + }) + .unwrap(); + let finish = observations + .iter() + .position(|event| matches!(event, controlled::Event::Finish)) + .unwrap(); assert!(future_drop < finish); // Neither a later cause upgrade nor shutdown retry spends hooks again. view.request_fault(); @@ -600,11 +667,15 @@ async fn actual_linked_rejected_physical_spawn_retains_received_resources_withou let _serial = controlled::TEST_LOCK.lock().await; let (fixture, events) = controlled::fixture(); let broker = tinybus::broker::Broker::new(); - broker.clone().spawn(tinybus::transport::memory::MemoryBus::new()); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); host.attach_linked_with_config( - controlled::linked_module().unwrap(), serde_json::json!({"mode":1}), - ).unwrap(); + controlled::linked_module().unwrap(), + serde_json::json!({"mode":1}), + ) + .unwrap(); let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); let view = fixture.view.lock().unwrap().take().unwrap(); let context = view.inner.context(); @@ -625,12 +696,19 @@ async fn actual_linked_rejected_physical_spawn_retains_received_resources_withou assert_eq!(fixture.requests.load(Ordering::Acquire), 1); assert_eq!(fixture.finishes.load(Ordering::Acquire), 0); assert_eq!(fixture.joins.load(Ordering::Acquire), 1); - assert!(!observations.iter().any(|event| matches!(event, - controlled::Event::FactoryDrop(_) | controlled::Event::FutureDrop(_) | controlled::Event::Finish))); + assert!(!observations.iter().any(|event| matches!( + event, + controlled::Event::FactoryDrop(_) + | controlled::Event::FutureDrop(_) + | controlled::Event::Finish + ))); assert!(!view.is_live()); drop(host); view.request_fault(); - assert_eq!(view.inner.wait(Duration::from_millis(20)), TerminalCompletion::Uncertain); + assert_eq!( + view.inner.wait(Duration::from_millis(20)), + TerminalCompletion::Uncertain + ); assert_eq!(fixture.requests.load(Ordering::Acquire), 1); assert_eq!(fixture.finishes.load(Ordering::Acquire), 0); let anchored = terminal::TerminalState::lookup(context).unwrap(); @@ -1103,10 +1181,15 @@ impl Drop for BlockingMethodPayload { let flags = unsafe { host.native_terminal_state.unwrap()(host.host_ctx) }; // This sample is the first foreign payload-disposal action. It reads // actual copied native authority and the real registered request state. - self.0.entered.send(( - flags, state.request_done_for_test(), state.sdk_owner_count_for_test(), - tokio::runtime::Handle::current().id(), - )).unwrap(); + self.0 + .entered + .send(( + flags, + state.request_done_for_test(), + state.sdk_owner_count_for_test(), + tokio::runtime::Handle::current().id(), + )) + .unwrap(); drop(state); self.0.release.wait(); self.0.settled.send(()).unwrap(); @@ -1121,7 +1204,11 @@ impl tinybus::Interface for MethodPayloadObject { fn members(&self) -> Vec { vec!["Panic".parse().unwrap()] } - async fn call(&self, _: &tinybus::MemberName, _: serde_json::Value) -> Result { + async fn call( + &self, + _: &tinybus::MemberName, + _: serde_json::Value, + ) -> Result { std::panic::panic_any(BlockingMethodPayload(self.0.clone())); } } @@ -1133,31 +1220,47 @@ async fn actual_method_observer_fences_native_fault_and_request_done_before_bloc let broker = tinybus::broker::Broker::new(); let bus = tinybus::transport::memory::MemoryBus::new(); broker.clone().spawn(bus.clone()); - let client = Connection::connect(bus.connect().await.unwrap()).await.unwrap(); + let client = Connection::connect(bus.connect().await.unwrap()) + .await + .unwrap(); let host = Arc::new(tinybus::module::ModuleHost::new(broker)); host.attach_linked_with_config( - controlled::linked_module().unwrap(), serde_json::json!({"mode":0}), - ).unwrap(); + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); events.recv_timeout(Duration::from_secs(3)).unwrap(); let view = fixture.view.lock().unwrap().take().unwrap(); let (entered_tx, entered_rx) = std::sync::mpsc::channel(); let (settled_tx, settled_rx) = std::sync::mpsc::channel(); let release = Arc::new(std::sync::Barrier::new(2)); let gate = Arc::new(MethodPayloadGate { - state: Arc::downgrade(&view.inner), entered: entered_tx, - release: release.clone(), settled: settled_tx, + state: Arc::downgrade(&view.inner), + entered: entered_tx, + release: release.clone(), + settled: settled_tx, }); let (connection, module_runtime) = { let resources = view.inner.resources.lock().unwrap(); let connection = resources.connection.lock().unwrap().clone().unwrap(); - (connection, resources.runtime.as_ref().unwrap().handle().id()) + ( + connection, + resources.runtime.as_ref().unwrap().handle().id(), + ) }; let destination = connection.unique_name().unwrap().to_string(); - connection.serve_at("/payload".parse().unwrap(), MethodPayloadObject(gate)).await.unwrap(); + connection + .serve_at("/payload".parse().unwrap(), MethodPayloadObject(gate)) + .await + .unwrap(); drop(connection); let call = tokio::spawn(async move { - client.proxy(&destination, "/payload", "ai.tinyhumans.tinybus.Payload") - .unwrap().with_timeout(Duration::from_secs(8)).call::("Panic", ()).await + client + .proxy(&destination, "/payload", "ai.tinyhumans.tinybus.Payload") + .unwrap() + .with_timeout(Duration::from_secs(8)) + .call::("Panic", ()) + .await }); let sample = entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); // Actual method task is held in foreign payload Drop. Its Connection owner @@ -1175,16 +1278,27 @@ async fn actual_method_observer_fences_native_fault_and_request_done_before_bloc while view.inner.sdk_owner_count_for_test() != 0 { tokio::task::yield_now().await; } - }).await.unwrap(); + }) + .await + .unwrap(); // All controlled payload work, actual method caller and controller worker // have settled before assertions; the conservative original anchor remains. assert_ne!(sample.0 & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); - assert!(sample.1, "installed observer must finish bounded request before foreign Drop"); - assert!(sample.2 > 0, "actual method Connection/task must retain SDK State owner"); + assert!( + sample.1, + "installed observer must finish bounded request before foreign Drop" + ); + assert!( + sample.2 > 0, + "actual method Connection/task must retain SDK State owner" + ); assert_eq!(sample.3, module_runtime); assert_eq!(completion, TerminalCompletion::Uncertain); assert!(anchored && retained_sdk > 0); - assert_eq!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); assert_eq!(fixture.requests.load(Ordering::Acquire), 1); assert_eq!(fixture.finishes.load(Ordering::Acquire), 1); assert_eq!(fixture.joins.load(Ordering::Acquire), 1); diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index a3717aae..b523fb31 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -243,7 +243,10 @@ impl SdkOwner { #[cfg(test)] pub(super) fn record_transport_drop_for_test(&self) { *self.state.transport_drop_thread.lock().unwrap() = Some( - std::thread::current().name().unwrap_or("unnamed").to_owned(), + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned(), ); } pub fn caught_panic(&self) { @@ -613,7 +616,10 @@ impl TerminalState { } #[cfg(test)] pub(super) fn request_done_for_test(&self) -> bool { - matches!(self.lock().request_state, RequestState::Done | RequestState::Panicked) + matches!( + self.lock().request_state, + RequestState::Done | RequestState::Panicked + ) } fn schedule(self: &Arc) { let job = { @@ -636,7 +642,9 @@ impl TerminalState { #[cfg(not(test))] let rejected = false; let submitted = if rejected { - Err(std::io::Error::other("controlled terminal worker spawn failure")) + Err(std::io::Error::other( + "controlled terminal worker spawn failure", + )) } else { std::thread::Builder::new() .name("tinybus-terminal".into()) From e446e1112733895bb85bb18746ea6ebaae933554 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:02:00 +0300 Subject: [PATCH 088/130] test(module): add startup failure fixture before transfer Add a test-only hook that holds SDK-owned C/F before trusted setup consumes them, letting tests exercise a deterministic startup failure with a real cancellable task. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 7 +++++++ crates/tinybus-module/src/terminal.rs | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 329c935f..d38c8f3f 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1743,6 +1743,13 @@ where state.startup_checkpoint_for_test(6); let task_state = state.clone(); let task = handle.spawn(async move { + #[cfg(test)] + if task_state.hold_initial_before_transfer_for_test() { + // Deterministic startup-failure fixture: retain SDK-owned C/F + // before trusted setup consumes them, with a real cancellable task. + terminal_notification(&task_state).await; + return Ok(()); + } let provenance = transport.inner.provenance.clone(); let brokered = transport.inner.host.0.broker_routing.is_some(); let connection = if brokered { diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index b523fb31..dcbb3cd4 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -601,6 +601,10 @@ impl TerminalState { self.startup_failure_stage.store(stage, Ordering::Release); } #[cfg(test)] + pub(super) fn hold_initial_before_transfer_for_test(&self) -> bool { + matches!(self.startup_failure_stage.load(Ordering::Acquire), 7 | 8) + } + #[cfg(test)] pub(super) fn startup_checkpoint_for_test(&self, stage: u8) { if self.startup_failure_stage.load(Ordering::Acquire) == stage { panic!("controlled startup failure after transfer stage {stage}"); From 54b4690361988ede60db1206414273cb61275a7d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:10:35 +0300 Subject: [PATCH 089/130] feat(tinybus): add fault injection modes to terminal example The terminal example fixture now accepts a factory mode that can force setup errors, panics, retained views, and uncertain or panicking native callbacks, so tests can exercise failure paths. New exported helpers report factory invocation counts and sample retained view liveness. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/examples/module_terminal.rs | 63 +++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 9f97d9dd..c346cc55 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -10,6 +10,9 @@ use std::{ use tinybus::{Connection, Result}; static DROPPED: AtomicUsize = AtomicUsize::new(0); static EARLY: Mutex> = Mutex::new(None); +static NEXT_FACTORY_MODE: AtomicUsize = AtomicUsize::new(0); +static FACTORIES: AtomicUsize = AtomicUsize::new(0); +static RETAINED_VIEWS: Mutex> = Mutex::new(Vec::new()); #[derive(Clone, Copy)] struct NativeQuery { ctx: *mut c_void, @@ -40,6 +43,7 @@ impl Controls { struct Controller { controls: Mutex>, view: NativeInstanceLiveness, + mode: AtomicUsize, wake: (Mutex, Condvar), worker: Mutex>>, } @@ -94,14 +98,29 @@ impl Drop for SetupFuture { type FixtureFuture = Pin> + Send>>; type FixtureSetup = Box FixtureFuture + Send + Sync>; fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> { + FACTORIES.fetch_add(1, Ordering::AcqRel); + let factory_mode = NEXT_FACTORY_MODE.swap(0, Ordering::AcqRel); + if factory_mode == 3 { + RETAINED_VIEWS.lock().unwrap().push(view.clone()); + } let early = EARLY .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .take(); - let held_finish = early.is_some(); + if let Some(controls) = early { + controls.emit(9, factory_mode as u32); + } + if factory_mode == 1 { + return Err(tinybus::Error::failed("controlled factory error")); + } + if factory_mode == 2 { + panic!("controlled factory panic"); + } + let held_finish = early.is_some() && factory_mode == 0; let controller = Arc::new(Controller { controls: Mutex::new(early), view, + mode: AtomicUsize::new(factory_mode), wake: (Mutex::new(false), Condvar::new()), worker: Mutex::new(None), }); @@ -122,6 +141,7 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> .cast::() .read_unaligned() }; + controller.mode.store(usize::from(config.mode), Ordering::Release); *controller .controls .lock() @@ -175,6 +195,9 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> .unwrap_or_else(std::sync::PoisonError::into_inner) = true; request.wake.1.notify_all(); request.emit(3, 1); + if request.mode.load(Ordering::Acquire) == 10 { + panic!("controlled native request panic"); + } }, move || { if held_finish { @@ -188,6 +211,11 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> let outcome = worker.map(|worker| worker.join()); let joined = outcome.is_none() || matches!(outcome, Some(Ok(true))); finish.emit(5, u32::from(joined)); + match finish.mode.load(Ordering::Acquire) { + 11 => panic!("controlled native finish panic"), + 12 => return TerminalCompletion::Uncertain, + _ => {} + } if joined { TerminalCompletion::PhysicallyStopped } else { @@ -292,3 +320,36 @@ pub extern "C" fn terminal_test_native_flags() -> u32 { .unwrap_or_else(std::sync::PoisonError::into_inner); query.map_or(0, |query| unsafe { (query.query)(query.ctx) }) } + +/// # Safety +/// Controls must remain live through all emitted callbacks. Modes only configure +/// this example fixture; they do not grant native permission or construct views. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn terminal_test_factory_mode( + mode: usize, + ctx: *mut c_void, + event: unsafe extern "C" fn(*mut c_void, u32, u32), +) { + *EARLY.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Controls { ctx, event }); + NEXT_FACTORY_MODE.store(mode, Ordering::Release); +} +/// Counts actual factory invocations, including those which return no hooks. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_factories() -> usize { + FACTORIES.load(Ordering::Acquire) +} +/// Samples escaped actual factory-provided views against their copied host query. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_retained_live() -> usize { + let views = RETAINED_VIEWS.lock().unwrap().clone(); + views.iter().filter(|view| view.is_live()).count() +} +/// Release escaped actual views outside their fixture storage lock. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_release_views() -> usize { + let views = std::mem::take(&mut *RETAINED_VIEWS.lock().unwrap()); + let count = views.len(); + drop(views); + count +} From 84ceb8860717d304f69d14a2580c834ec510c5c9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:11:28 +0300 Subject: [PATCH 090/130] feat(tinybus-module): expose native lifecycle fixture hooks behind a feature Adds a private `__native-fixture-support` feature that compiles the existing test-only startup checkpoints, fault injection and liveness probes into the cdylib so an out-of-tree native lifecycle fixture can drive them. The hooks remain gated off in normal builds, so no public behaviour changes. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/Cargo.toml | 2 + .../tinybus-module/src/execution_context.rs | 10 ++++ crates/tinybus-module/src/lib.rs | 18 +++---- crates/tinybus-module/src/terminal.rs | 54 ++++++++++++++----- 4 files changed, 63 insertions(+), 21 deletions(-) diff --git a/crates/tinybus-module/Cargo.toml b/crates/tinybus-module/Cargo.toml index 6cf7994d..0b57b82d 100644 --- a/crates/tinybus-module/Cargo.toml +++ b/crates/tinybus-module/Cargo.toml @@ -9,6 +9,8 @@ description = "Module-side runtime and export macro for trusted tinybus cdylibs. [features] static-link = [] +# Private opt-in instrumentation for the compiled native lifecycle fixture only. +__native-fixture-support = [] [dependencies] async-trait.workspace = true diff --git a/crates/tinybus-module/src/execution_context.rs b/crates/tinybus-module/src/execution_context.rs index 93bede00..5ad19a9c 100644 --- a/crates/tinybus-module/src/execution_context.rs +++ b/crates/tinybus-module/src/execution_context.rs @@ -32,6 +32,16 @@ impl ExecutionOwner { live, } } + #[cfg(feature = "__native-fixture-support")] + pub(super) fn is_current_for_fixture(&self) -> bool { + OWNER.with(|slot| slot.borrow().as_ref().is_some_and(|current| { + Arc::ptr_eq(&self.live, ¤t.live) + })) + } + #[cfg(feature = "__native-fixture-support")] + pub(super) fn is_stopped_for_fixture(&self) -> bool { + !self.live.load(Ordering::Acquire) + } pub(super) fn enter(&self) -> ExecutionGuard { let previous = OWNER.with(|slot| slot.replace(Some(self.clone()))); let dispatch = tracing::dispatcher::set_default(&self.dispatch); diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index d38c8f3f..6597afe2 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1536,7 +1536,7 @@ where .unwrap_or_else(std::sync::PoisonError::into_inner) .retained_factory = Some(Box::new(setup.clone())); state.register(hooks); - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] state.startup_checkpoint_for_test(1); if !state.is_live() { return TB_CLOSED; @@ -1559,7 +1559,7 @@ where .setup = Some(Box::new(move |connection| { Box::pin(initial_setup(connection, config)) })); - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] state.startup_checkpoint_for_test(2); let weak = Arc::downgrade(&state); let reinitializer: ReinitializeFactory = Box::new(move |handle, connection_slot| { @@ -1666,7 +1666,7 @@ where .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .reinitializer_factory = Some(reinitializer); - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] state.startup_checkpoint_for_test(3); let mut builder = tokio::runtime::Builder::new_multi_thread(); if let Some(owner) = &state.owner { @@ -1685,7 +1685,7 @@ where .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .runtime = Some(runtime); - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] state.startup_checkpoint_for_test(4); let (inbound_tx, inbound_rx) = mpsc::channel(MODULE_QUEUE_CAPACITY); state @@ -1709,7 +1709,7 @@ where .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .transport = Some(transport.clone()); - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] state.startup_checkpoint_for_test(5); let (handle, connection_slot, factory) = { let mut resources = state @@ -1739,11 +1739,11 @@ where *callback_slot .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) = callback; - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] state.startup_checkpoint_for_test(6); let task_state = state.clone(); let task = handle.spawn(async move { - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] if task_state.hold_initial_before_transfer_for_test() { // Deterministic startup-failure fixture: retain SDK-owned C/F // before trusted setup consumes them, with a real cancellable task. @@ -1817,10 +1817,10 @@ where .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .initial = Some(task); - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] state.startup_checkpoint_for_test(7); let context = state.context(); - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] state.startup_checkpoint_for_test(8); state.mark_vtable_published(); if capacity >= size_of::() as u32 { diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index dcbb3cd4..72de23b1 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -57,6 +57,36 @@ impl NativeInstanceLiveness { pub fn is_live(&self) -> bool { self.inner.is_live() } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_owner_is_current(&self) -> bool { + self.inner.owner.as_ref().is_some_and(ExecutionOwner::is_current_for_fixture) + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_owner_is_stopped(&self) -> bool { + self.inner.owner.as_ref().is_some_and(ExecutionOwner::is_stopped_for_fixture) + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_request_done(&self) -> bool { + self.inner.request_done_for_test() + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_sdk_owners(&self) -> usize { + self.inner.sdk_owner_count_for_test() + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_reject_worker_spawn(&self) { + self.inner.reject_worker_spawn_for_test(); + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_reject_startup(&self, stage: u8) { + self.inner.reject_startup_for_test(stage); + } /// Persist genuine fault and close independent native admission first. pub fn request_fault(&self) { self.inner.signal(TerminalReason::Fault); @@ -225,9 +255,9 @@ pub(super) struct TerminalState { pub resources: StdMutex, sdk_owners: AtomicUsize, vtable_published: AtomicBool, - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] reject_worker_spawn: AtomicBool, - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] startup_failure_stage: AtomicU8, #[cfg(test)] transport_drop_thread: StdMutex>, @@ -325,9 +355,9 @@ impl TerminalState { resources: StdMutex::new(StartupResources::default()), sdk_owners: AtomicUsize::new(0), vtable_published: AtomicBool::new(false), - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] reject_worker_spawn: AtomicBool::new(false), - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] startup_failure_stage: AtomicU8::new(0), #[cfg(test)] transport_drop_thread: StdMutex::new(None), @@ -596,29 +626,29 @@ impl TerminalState { pub(super) fn transport_drop_thread_for_test(&self) -> Option { self.transport_drop_thread.lock().unwrap().clone() } - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] pub(super) fn reject_startup_for_test(&self, stage: u8) { self.startup_failure_stage.store(stage, Ordering::Release); } - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] pub(super) fn hold_initial_before_transfer_for_test(&self) -> bool { matches!(self.startup_failure_stage.load(Ordering::Acquire), 7 | 8) } - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] pub(super) fn startup_checkpoint_for_test(&self, stage: u8) { if self.startup_failure_stage.load(Ordering::Acquire) == stage { panic!("controlled startup failure after transfer stage {stage}"); } } - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] pub(super) fn reject_worker_spawn_for_test(&self) { self.reject_worker_spawn.store(true, Ordering::Release); } - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] pub(super) fn sdk_owner_count_for_test(&self) -> usize { self.sdk_owners.load(Ordering::Acquire) } - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] pub(super) fn request_done_for_test(&self) -> bool { matches!( self.lock().request_state, @@ -641,9 +671,9 @@ impl TerminalState { let state = job.state.upgrade().expect("anchored terminal state"); // Private per-instance rejection exercises the real submitted-job ownership // path without changing process-wide thread creation or public authority. - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] let rejected = self.reject_worker_spawn.load(Ordering::Acquire); - #[cfg(not(test))] + #[cfg(not(any(test, feature = "__native-fixture-support")))] let rejected = false; let submitted = if rejected { Err(std::io::Error::other( From b57d5558c8f3b2c02bd415ecad4f90cc98e9aa88 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:12:15 +0300 Subject: [PATCH 091/130] feat(tinybus): add native fixture lifecycle observation to terminal example The module_terminal example now reports lifecycle checkpoints through a new observe helper, gated behind a private __native-fixture-support feature that is off by default. The feature also lets the example simulate worker spawn and startup rejections so native lifecycle fixtures can exercise failure paths. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/Cargo.toml | 2 ++ crates/tinybus/examples/module_terminal.rs | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/crates/tinybus/Cargo.toml b/crates/tinybus/Cargo.toml index d698f3d3..3a79ee10 100644 --- a/crates/tinybus/Cargo.toml +++ b/crates/tinybus/Cargo.toml @@ -48,6 +48,8 @@ toml = { version = "1", optional = true } libc = { version = "0.2", optional = true } [features] +# Explicit private native lifecycle fixture cohort; never part of defaults. +__native-fixture-support = ["tinybus-module/__native-fixture-support"] # `uds` and `macros` are on by default because a bus you cannot connect to over # a socket, and an interface you have to hand-dispatch, are not what anyone is # installing. The CLI is opt-in: embedding the library should not pull in diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index c346cc55..ec108e50 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -48,6 +48,15 @@ struct Controller { worker: Mutex>>, } impl Controller { + fn observe(&self, event: u32) { + #[cfg(feature = "__native-fixture-support")] + self.emit(event, u32::from(self.view.__fixture_owner_is_current()) + | (u32::from(self.view.__fixture_owner_is_stopped()) << 1) + | (u32::from(self.view.__fixture_request_done()) << 2) + | (u32::from(self.view.__fixture_sdk_owners() != 0) << 3)); + #[cfg(not(feature = "__native-fixture-support"))] + let _ = event; + } fn emit(&self, event: u32, value: u32) { let controls = *self .controls @@ -60,6 +69,7 @@ impl Controller { } impl Drop for Controller { fn drop(&mut self) { + self.observe(15); self.emit( 6, u32::from(std::thread::current().name() == Some("tinybus-terminal")), @@ -77,6 +87,7 @@ impl Future for SetupFuture { fn poll(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { if self.first { self.first = false; + self.controller.observe(11); self.controller.emit(2, u32::from(self.mode)); } match self.mode { @@ -89,6 +100,7 @@ impl Future for SetupFuture { } impl Drop for SetupFuture { fn drop(&mut self) { + self.controller.observe(14); self.controller.emit( 4, u32::from(std::thread::current().name() == Some("tinybus-terminal")), @@ -124,11 +136,19 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> wake: (Mutex::new(false), Condvar::new()), worker: Mutex::new(None), }); + controller.observe(10); + #[cfg(feature = "__native-fixture-support")] + match factory_mode { + 20 => controller.view.__fixture_reject_worker_spawn(), + 21..=28 => controller.view.__fixture_reject_startup((factory_mode - 20) as u8), + _ => {} + } if held_finish { controller.view.request_fault(); } let (request, finish) = (controller.clone(), controller.clone()); let setup = move |_: Connection, config: Config| { + controller.observe(16); if config.test_controls.len() != size_of::() { return Box::pin(async { Err(tinybus::Error::failed("bad test controls")) }) as Pin> + Send>>; @@ -188,6 +208,7 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> }; let hooks = RuntimeHooks::new( move |_| { + request.observe(12); *request .wake .0 @@ -200,6 +221,7 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> } }, move || { + finish.observe(13); if held_finish { finish.emit(8, 0); } From 32333c49ef1c13c990745b1e5ecb603748185a10 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:13:51 +0300 Subject: [PATCH 092/130] test(tinybus-module): assert fixture owner liveness events Extend the native terminal fixture test to check the owner-current, stopped, request-done and sdk-owner bits reported for each lifecycle event, skipping the two owner-state assertions in mode 4 where the owner is expected to be gone. The remaining changes are rustfmt reflows with no behaviour change. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/execution_context.rs | 8 ++++-- .../tinybus-module/src/lib_terminal_tests.rs | 28 +++++++++++++++++++ crates/tinybus-module/src/terminal.rs | 10 +++++-- crates/tinybus/examples/module_terminal.rs | 24 ++++++++++------ 4 files changed, 57 insertions(+), 13 deletions(-) diff --git a/crates/tinybus-module/src/execution_context.rs b/crates/tinybus-module/src/execution_context.rs index 5ad19a9c..5cd40d11 100644 --- a/crates/tinybus-module/src/execution_context.rs +++ b/crates/tinybus-module/src/execution_context.rs @@ -34,9 +34,11 @@ impl ExecutionOwner { } #[cfg(feature = "__native-fixture-support")] pub(super) fn is_current_for_fixture(&self) -> bool { - OWNER.with(|slot| slot.borrow().as_ref().is_some_and(|current| { - Arc::ptr_eq(&self.live, ¤t.live) - })) + OWNER.with(|slot| { + slot.borrow() + .as_ref() + .is_some_and(|current| Arc::ptr_eq(&self.live, ¤t.live)) + }) } #[cfg(feature = "__native-fixture-support")] pub(super) fn is_stopped_for_fixture(&self) -> bool { diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index d9e7d8b6..dec34bec 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -966,6 +966,34 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal "retired successful context must not act on replacement mode {mode}" ); } + // The instrumented fixture compares Arc identity internally. Only + // booleans cross its controls; neither TLS nor these events authorize work. + assert!( + seen.iter() + .any(|(event, bits)| *event == 10 && bits & 1 != 0) + ); + assert!( + seen.iter() + .any(|(event, bits)| *event == 12 && bits & 1 != 0) + ); + assert!( + seen.iter() + .any(|(event, bits)| *event == 13 && bits & 7 == 7) + ); + assert!( + seen.iter() + .any(|(event, bits)| *event == 15 && bits & 7 == 7) + ); + if mode != 4 { + assert!( + seen.iter() + .any(|(event, bits)| *event == 11 && bits & 9 == 9) + ); + assert!( + seen.iter() + .any(|(event, bits)| *event == 14 && bits & 1 != 0) + ); + } assert_eq!(seen.iter().filter(|(event, _)| *event == 3).count(), 1); assert_eq!( seen.iter() diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 72de23b1..8dbce24a 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -60,12 +60,18 @@ impl NativeInstanceLiveness { #[doc(hidden)] #[cfg(feature = "__native-fixture-support")] pub fn __fixture_owner_is_current(&self) -> bool { - self.inner.owner.as_ref().is_some_and(ExecutionOwner::is_current_for_fixture) + self.inner + .owner + .as_ref() + .is_some_and(ExecutionOwner::is_current_for_fixture) } #[doc(hidden)] #[cfg(feature = "__native-fixture-support")] pub fn __fixture_owner_is_stopped(&self) -> bool { - self.inner.owner.as_ref().is_some_and(ExecutionOwner::is_stopped_for_fixture) + self.inner + .owner + .as_ref() + .is_some_and(ExecutionOwner::is_stopped_for_fixture) } #[doc(hidden)] #[cfg(feature = "__native-fixture-support")] diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index ec108e50..6f6f9cae 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -50,10 +50,13 @@ struct Controller { impl Controller { fn observe(&self, event: u32) { #[cfg(feature = "__native-fixture-support")] - self.emit(event, u32::from(self.view.__fixture_owner_is_current()) - | (u32::from(self.view.__fixture_owner_is_stopped()) << 1) - | (u32::from(self.view.__fixture_request_done()) << 2) - | (u32::from(self.view.__fixture_sdk_owners() != 0) << 3)); + self.emit( + event, + u32::from(self.view.__fixture_owner_is_current()) + | (u32::from(self.view.__fixture_owner_is_stopped()) << 1) + | (u32::from(self.view.__fixture_request_done()) << 2) + | (u32::from(self.view.__fixture_sdk_owners() != 0) << 3), + ); #[cfg(not(feature = "__native-fixture-support"))] let _ = event; } @@ -140,7 +143,9 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> #[cfg(feature = "__native-fixture-support")] match factory_mode { 20 => controller.view.__fixture_reject_worker_spawn(), - 21..=28 => controller.view.__fixture_reject_startup((factory_mode - 20) as u8), + 21..=28 => controller + .view + .__fixture_reject_startup((factory_mode - 20) as u8), _ => {} } if held_finish { @@ -161,7 +166,9 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> .cast::() .read_unaligned() }; - controller.mode.store(usize::from(config.mode), Ordering::Release); + controller + .mode + .store(usize::from(config.mode), Ordering::Release); *controller .controls .lock() @@ -352,8 +359,9 @@ pub unsafe extern "C" fn terminal_test_factory_mode( ctx: *mut c_void, event: unsafe extern "C" fn(*mut c_void, u32, u32), ) { - *EARLY.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(Controls { ctx, event }); + *EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Controls { ctx, event }); NEXT_FACTORY_MODE.store(mode, Ordering::Release); } /// Counts actual factory invocations, including those which return no hooks. From b954b9529e9ac6c8562be2d84d3d50033611cf88 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:15:16 +0300 Subject: [PATCH 093/130] feat(module_terminal): seed test controls before factory evaluation The example module now reads its test controls from the host config during init and stores them in the early slot, so the fixture has its trusted callback available before factory evaluation runs. Product authority is unchanged and still comes exclusively from the host query. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/examples/module_terminal.rs | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 6f6f9cae..c1195b07 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -283,6 +283,29 @@ pub unsafe extern "C" fn tinybus_module_init_v1( out: *mut TbModuleVtable, ) -> i32 { if let Some(host) = unsafe { TbHostVtable::read_compatible(host) } { + // The fixture needs its trusted callback before factory evaluation. + // Product authority still comes exclusively from the untouched host query. + if !host.config.ptr.is_null() { + let bytes = unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }; + if let Ok(config) = serde_json::from_slice::(bytes) { + if config.test_controls.len() == size_of::() { + let controls = unsafe { + config + .test_controls + .as_ptr() + .cast::() + .read_unaligned() + }; + let mut early = EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if early.is_none() { + *early = Some(controls); + NEXT_FACTORY_MODE.store(99, Ordering::Release); + } + } + } + } if let Some(query) = host.native_terminal_state { *NATIVE_QUERY .lock() From 690594e1ae41497f378df2162b94fadd60b8e176 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:16:49 +0300 Subject: [PATCH 094/130] fix(tinybus): read test controls from raw json in terminal example The example now parses the host config as an untyped JSON value and extracts test_controls separately, so unknown fields no longer cause deserialization to fail. A Drop impl on Config emits a control signal when the module shuts down, reporting whether it ran on the expected terminal thread. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/examples/module_terminal.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index c1195b07..8a14411c 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -27,6 +27,14 @@ struct Config { mode: u8, test_controls: Vec, } +impl Drop for Config { + fn drop(&mut self) { + if self.test_controls.len() == size_of::() { + let controls = unsafe { self.test_controls.as_ptr().cast::().read_unaligned() }; + controls.emit(19, u32::from(std::thread::current().name() == Some("tinybus-terminal"))); + } + } +} #[repr(C)] #[derive(Clone, Copy)] struct Controls { @@ -287,10 +295,10 @@ pub unsafe extern "C" fn tinybus_module_init_v1( // Product authority still comes exclusively from the untouched host query. if !host.config.ptr.is_null() { let bytes = unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }; - if let Ok(config) = serde_json::from_slice::(bytes) { - if config.test_controls.len() == size_of::() { + if let Ok(config) = serde_json::from_slice::(bytes) { + if let Some(encoded) = config.get("test_controls").and_then(|value| serde_json::from_value::>(value.clone()).ok()).filter(|bytes| bytes.len() == size_of::()) { let controls = unsafe { - config + encoded .test_controls .as_ptr() .cast::() From ababe1515ace125704a855e7908ba0101c4bfd29 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:17:06 +0300 Subject: [PATCH 095/130] chore(examples): remove stray field access in terminal module The terminal example chained a field access on the encoded byte vector before taking its pointer, which does not compile since the vector has no such field. Dropping the line restores the intended pointer cast. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/examples/module_terminal.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 8a14411c..3ba7f3c0 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -299,7 +299,6 @@ pub unsafe extern "C" fn tinybus_module_init_v1( if let Some(encoded) = config.get("test_controls").and_then(|value| serde_json::from_value::>(value.clone()).ok()).filter(|bytes| bytes.len() == size_of::()) { let controls = unsafe { encoded - .test_controls .as_ptr() .cast::() .read_unaligned() From 6262de2f602ed319775ebebe5efcbf637d260086 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:18:10 +0300 Subject: [PATCH 096/130] test(module): cover native factory and hook failure paths Add an ignored integration test that drives the compiled native fixture through factory and request-hook failure modes, asserting that the original finish and fault events are preserved. The example fixture now panics with a payload that emits a drop event, so the test can detect whether the primary payload was retained rather than released. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 151 ++++++++++++++++++ crates/tinybus/examples/module_terminal.rs | 37 +++-- 2 files changed, 177 insertions(+), 11 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index dec34bec..79a63c45 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -1931,3 +1931,154 @@ async fn linked_reinit_synchronous_timer_uses_retained_runtime_on_same_worker() 3 ); } + +// The genuine ModuleHost loader must have pinned the artifact before this helper. +// Temporary OS symbol handles are released; controls never grant product authority. +unsafe fn native_fixture_symbol(path: &std::path::Path, symbol: &std::ffi::CStr) -> T { + #[cfg(unix)] + let raw = { + use std::os::unix::ffi::OsStrExt; + #[cfg_attr(target_os = "linux", link(name = "dl"))] + unsafe extern "C" { + fn dlopen(path: *const std::ffi::c_char, flags: i32) -> *mut c_void; + fn dlsym(handle: *mut c_void, name: *const std::ffi::c_char) -> *mut c_void; + fn dlclose(handle: *mut c_void) -> i32; + } + let path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + let handle = unsafe { dlopen(path.as_ptr(), 2) }; + assert!(!handle.is_null()); + let raw = unsafe { dlsym(handle, symbol.as_ptr()) }; + unsafe { + dlclose(handle); + } + raw + }; + #[cfg(windows)] + let raw = { + use std::os::windows::ffi::OsStrExt; + #[link(name = "kernel32")] + unsafe extern "system" { + fn LoadLibraryW(path: *const u16) -> *mut c_void; + fn GetProcAddress(handle: *mut c_void, name: *const u8) -> *mut c_void; + fn FreeLibrary(handle: *mut c_void) -> i32; + } + let path: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let handle = unsafe { LoadLibraryW(path.as_ptr()) }; + assert!(!handle.is_null()); + let raw = unsafe { GetProcAddress(handle, symbol.as_ptr().cast()) }; + unsafe { + FreeLibrary(handle); + } + raw + }; + assert!(!raw.is_null()); + assert_eq!(size_of::(), size_of::<*mut c_void>()); + unsafe { std::mem::transmute_copy(&raw) } +} +fn native_controls_config(mode: u8, controls: &NativeControls) -> serde_json::Value { + let bytes = unsafe { + std::slice::from_raw_parts( + (controls as *const NativeControls).cast::(), + size_of::(), + ) + }; + serde_json::json!({"mode": mode, "test_controls": bytes}) +} +fn native_fixture_host() -> Arc { + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + Arc::new(tinybus::module::ModuleHost::new(broker)) +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires the privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_factory_and_hook_failures_preserve_original_finish_and_fault() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let (sender, receiver) = std::sync::mpsc::channel(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + let host = native_fixture_host(); + host.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let (_, _, drops, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + let before = drops(); + host.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + unsafe { + drop(Box::from_raw(sender)); + } + let configure: unsafe extern "C" fn( + usize, + *mut c_void, + unsafe extern "C" fn(*mut c_void, u32, u32), + ) = unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + for mode in [1usize, 2, 10, 11, 12] { + let (sender, receiver) = std::sync::mpsc::channel(); + // Uncertain primary/control resources intentionally retain this bounded + // fixture context; a manual join is never substituted for SDK completion. + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + unsafe { + configure(mode, sender.cast(), native_event); + } + let host = native_fixture_host(); + let result = + host.load_file_with_config(&path, native_controls_config(mode as u8, &controls)); + let mut seen = Vec::new(); + let target = if mode < 3 { 19 } else { 2 }; + loop { + let event = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); + seen.push(event); + if event.0 == target { + break; + } + } + if mode >= 10 { + host.shutdown(Duration::from_secs(5)).await; + while !seen.iter().any(|event| event.0 == 5) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + drop(host); + let native = flags(); + seen.extend(receiver.try_iter()); + // All blocking controller work is joined by the original SDK finish + // before these assertions. Factory failures never launch a controller. + if mode < 3 { + assert!(result.is_err()); + assert!(seen.iter().any(|event| *event == (19, 1))); + assert!(!seen.iter().any(|event| matches!(event.0, 2 | 3 | 5 | 7))); + } else { + assert!(result.is_ok()); + assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); + assert_eq!(seen.iter().filter(|event| *event == (5, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| *event == (7, 1)).count(), 1); + assert!(seen.iter().any(|event| event.0 == 13 && event.1 & 7 == 7)); + } + assert!( + !seen.iter().any(|event| event.0 == 17), + "primary payload must remain retained" + ); + assert_eq!( + native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + if mode != 12 { + assert_ne!(native & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + } + } +} diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 3ba7f3c0..d352b67e 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -30,8 +30,16 @@ struct Config { impl Drop for Config { fn drop(&mut self) { if self.test_controls.len() == size_of::() { - let controls = unsafe { self.test_controls.as_ptr().cast::().read_unaligned() }; - controls.emit(19, u32::from(std::thread::current().name() == Some("tinybus-terminal"))); + let controls = unsafe { + self.test_controls + .as_ptr() + .cast::() + .read_unaligned() + }; + controls.emit( + 19, + u32::from(std::thread::current().name() == Some("tinybus-terminal")), + ); } } } @@ -48,6 +56,14 @@ impl Controls { unsafe { (self.event)(self.ctx, event, value) } } } +struct PrimaryPayload(Option); +impl Drop for PrimaryPayload { + fn drop(&mut self) { + if let Some(controls) = self.0 { + controls.emit(17, 1); + } + } +} struct Controller { controls: Mutex>, view: NativeInstanceLiveness, @@ -137,7 +153,7 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> return Err(tinybus::Error::failed("controlled factory error")); } if factory_mode == 2 { - panic!("controlled factory panic"); + std::panic::panic_any(PrimaryPayload(early)); } let held_finish = early.is_some() && factory_mode == 0; let controller = Arc::new(Controller { @@ -232,7 +248,7 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> request.wake.1.notify_all(); request.emit(3, 1); if request.mode.load(Ordering::Acquire) == 10 { - panic!("controlled native request panic"); + std::panic::panic_any(PrimaryPayload(*request.controls.lock().unwrap())); } }, move || { @@ -296,13 +312,12 @@ pub unsafe extern "C" fn tinybus_module_init_v1( if !host.config.ptr.is_null() { let bytes = unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }; if let Ok(config) = serde_json::from_slice::(bytes) { - if let Some(encoded) = config.get("test_controls").and_then(|value| serde_json::from_value::>(value.clone()).ok()).filter(|bytes| bytes.len() == size_of::()) { - let controls = unsafe { - encoded - .as_ptr() - .cast::() - .read_unaligned() - }; + if let Some(encoded) = config + .get("test_controls") + .and_then(|value| serde_json::from_value::>(value.clone()).ok()) + .filter(|bytes| bytes.len() == size_of::()) + { + let controls = unsafe { encoded.as_ptr().cast::().read_unaligned() }; let mut early = EARLY .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); From ca1001fd76d85084e4a0d8baade207cdaa72c506 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:19:19 +0300 Subject: [PATCH 097/130] test(tinybus-module): annotate terminal channel types and panic payload The native terminal fixture tests now specify the channel element type explicitly, and the example's controlled finish panic carries a typed payload instead of a string. This keeps the fixture's panic path consistent with the payload the tests expect to observe. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 4 ++-- crates/tinybus/examples/module_terminal.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 79a63c45..c159bc27 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -1995,7 +1995,7 @@ fn native_fixture_host() -> Arc { #[ignore = "requires the privately pinned __native-fixture-support eight-artifact cohort"] async fn compiled_native_factory_and_hook_failures_preserve_original_finish_and_fault() { let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); - let (sender, receiver) = std::sync::mpsc::channel(); + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); let sender = Box::into_raw(Box::new(sender)); let controls = NativeControls { ctx: sender.cast(), @@ -2024,7 +2024,7 @@ async fn compiled_native_factory_and_hook_failures_preserve_original_finish_and_ unsafe extern "C" fn(*mut c_void, u32, u32), ) = unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; for mode in [1usize, 2, 10, 11, 12] { - let (sender, receiver) = std::sync::mpsc::channel(); + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); // Uncertain primary/control resources intentionally retain this bounded // fixture context; a manual join is never substituted for SDK completion. let sender = Box::into_raw(Box::new(sender)); diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index d352b67e..a72305e3 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -265,7 +265,7 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> let joined = outcome.is_none() || matches!(outcome, Some(Ok(true))); finish.emit(5, u32::from(joined)); match finish.mode.load(Ordering::Acquire) { - 11 => panic!("controlled native finish panic"), + 11 => std::panic::panic_any(PrimaryPayload(*finish.controls.lock().unwrap())), 12 => return TerminalCompletion::Uncertain, _ => {} } From 3c0e426eb1c14cce2f9702b1d11170f446f5d5d4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:20:29 +0300 Subject: [PATCH 098/130] test(tinybus-module): fix event comparison in terminal tests Adjust the closure comparisons to dereference the event reference twice so the tuple equality checks compile against the iterator item type. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index c159bc27..9a796726 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -2065,8 +2065,8 @@ async fn compiled_native_factory_and_hook_failures_preserve_original_finish_and_ } else { assert!(result.is_ok()); assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); - assert_eq!(seen.iter().filter(|event| *event == (5, 1)).count(), 1); - assert_eq!(seen.iter().filter(|event| *event == (7, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (7, 1)).count(), 1); assert!(seen.iter().any(|event| event.0 == 13 && event.1 & 7 == 7)); } assert!( From e5fabe5bbc07926a413583b9b6dd82e9e4eceb33 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:22:27 +0300 Subject: [PATCH 099/130] test(module): cover factory denial after entered init barrier Add a native host reload test that drives the terminal fixture through an entered init barrier and asserts the factory is denied when a stop or fault arrives mid-initialization. The terminal example now emits the barrier event and clears its early controls when the factory was never reached, so the fixture can observe the denial path. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/examples/module_terminal.rs | 16 ++ .../tinybus/src/module/host_reload_tests.rs | 153 ++++++++++++++++++ 2 files changed, 169 insertions(+) diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index a72305e3..9b4f4786 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -337,11 +337,27 @@ pub unsafe extern "C" fn tinybus_module_init_v1( }); } } + if NEXT_FACTORY_MODE.load(Ordering::Acquire) == 30 { + let controls = *EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(controls) = controls { + controls.emit(18, 0); + } + } + let factories_before = FACTORIES.load(Ordering::Acquire); let code = unsafe { tinybus_module::start_terminal_reconfigurable_module_with_factory::( host, out, 1, true, factory, ) }; + if FACTORIES.load(Ordering::Acquire) == factories_before { + EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + NEXT_FACTORY_MODE.store(0, Ordering::Release); + } if code == TB_OK { *LATEST .lock() diff --git a/crates/tinybus/src/module/host_reload_tests.rs b/crates/tinybus/src/module/host_reload_tests.rs index 14e7b67d..a065e4c2 100644 --- a/crates/tinybus/src/module/host_reload_tests.rs +++ b/crates/tinybus/src/module/host_reload_tests.rs @@ -248,3 +248,156 @@ fn native_log_fixture_recovers_poison_without_unwinding_across_abi() { .any(|line| line.contains("actual ABI callback")) })); } + +#[cfg(unix)] +unsafe fn terminal_symbol(path: &Path, name: &std::ffi::CStr) -> T { + use std::os::unix::ffi::OsStrExt; + #[cfg_attr(target_os = "linux", link(name = "dl"))] + unsafe extern "C" { + fn dlopen(path: *const std::ffi::c_char, flags: i32) -> *mut std::ffi::c_void; + fn dlsym( + handle: *mut std::ffi::c_void, + name: *const std::ffi::c_char, + ) -> *mut std::ffi::c_void; + fn dlclose(handle: *mut std::ffi::c_void) -> i32; + } + let path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + let handle = unsafe { dlopen(path.as_ptr(), 2) }; + assert!(!handle.is_null()); + let raw = unsafe { dlsym(handle, name.as_ptr()) }; + unsafe { + dlclose(handle); + } + assert!(!raw.is_null()); + assert_eq!(std::mem::size_of::(), std::mem::size_of_val(&raw)); + unsafe { std::mem::transmute_copy(&raw) } +} +#[repr(C)] +#[derive(Clone, Copy)] +struct TerminalInitControls { + ctx: *mut std::ffi::c_void, + event: unsafe extern "C" fn(*mut std::ffi::c_void, u32, u32), +} +struct TerminalInitBarrier { + entered: std::sync::mpsc::Sender<()>, + release: (std::sync::Mutex, std::sync::Condvar), +} +unsafe extern "C" fn terminal_init_event(ctx: *mut std::ffi::c_void, event: u32, _: u32) { + if event != 18 { + return; + } + let control = unsafe { &*ctx.cast::() }; + let _ = control.entered.send(()); + let mut release = control.release.0.lock().unwrap(); + while !*release { + release = control.release.1.wait(release).unwrap(); + } +} +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires the privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_host_stop_and_fault_deny_factory_after_entered_init_barrier() { + use crate::module::abi::*; + let path = PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + // Real loader admission pins the image before any test symbol resolution. + let artifact = loader::load(&path, false).unwrap(); + let configure: unsafe extern "C" fn( + usize, + *mut std::ffi::c_void, + unsafe extern "C" fn(*mut std::ffi::c_void, u32, u32), + ) = unsafe { terminal_symbol(&path, c"terminal_test_factory_mode") }; + let factories: extern "C" fn() -> usize = + unsafe { terminal_symbol(&path, c"terminal_test_factories") }; + for fault in [false, true] { + let broker = Broker::new(); + let broker_task = broker.clone().spawn(MemoryBus::new()); + let host = ModuleHost::new(broker); + let admitted = host + .validate(&path, &artifact.descriptor, &artifact.manifest) + .unwrap(); + let (sender, receiver) = std::sync::mpsc::channel(); + let control = Box::into_raw(Box::new(TerminalInitBarrier { + entered: sender, + release: (std::sync::Mutex::new(false), std::sync::Condvar::new()), + })); + let controls = TerminalInitControls { + ctx: control.cast(), + event: terminal_init_event, + }; + let encoded = unsafe { + std::slice::from_raw_parts( + (&controls as *const TerminalInitControls).cast::(), + std::mem::size_of::(), + ) + }; + let config = + serde_json::to_vec(&serde_json::json!({"mode": 0, "test_controls": encoded})).unwrap(); + let (transport, calls) = ModuleTransport::new(admitted.name.clone(), config); + unsafe { + configure(30, control.cast(), terminal_init_event); + } + let before = factories(); + transport.defer_initialize(artifact.init, calls); + // Attach the genuine loaded artifact and transport to the real host, + // matching the existing observed_native deferred-init fixture above. + host.attach_transport( + &path, + admitted, + artifact.manifest.clone(), + transport.clone(), + Activation { + lazy_init: true, + lazy_load: false, + descriptor_info: None, + pinned_sha256: None, + }, + ) + .unwrap(); + let initialize = { + let transport = transport.clone(); + tokio::spawn(async move { transport.ensure_initialized().await }) + }; + let entered = receiver.recv_timeout(Duration::from_secs(5)).is_ok(); + if fault { + unsafe { + (calls.fault)(calls.host_ctx, std::ptr::null(), 0); + } + } else { + host.shutdown(Duration::from_millis(20)).await; + } + let flags = unsafe { calls.native_terminal_state.unwrap()(calls.host_ctx) }; + // Release and await the actual opaque init worker before assertions. + unsafe { + *(*control).release.0.lock().unwrap() = true; + (*control).release.1.notify_all(); + } + let initialized = tokio::time::timeout(Duration::from_secs(5), initialize) + .await + .unwrap(); + host.shutdown(Duration::from_secs(5)).await; + drop(host); + let final_flags = unsafe { calls.native_terminal_state.unwrap()(calls.host_ctx) }; + unsafe { + drop(Box::from_raw(control)); + } + broker_task.abort(); + let _ = broker_task.await; + assert!(entered); + assert!(initialized.is_ok_and(|result| result.is_err())); + assert_eq!( + factories(), + before, + "terminal authority must deny before the actual factory" + ); + assert_ne!( + flags + & if fault { + TB_NATIVE_TERMINAL_FAULT + } else { + TB_NATIVE_TERMINAL_STOP_INTENT + }, + 0 + ); + assert_eq!(final_flags & TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); + } +} From 14a21329f9a1dd0e2060a9db0825122190e626f5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:23:58 +0300 Subject: [PATCH 100/130] chore: files changed crates/tinybus-module/src/lib_terminal_tests.rs,crates/tinybus/examples/module_ Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 155 ++++++++++++++++++ crates/tinybus/examples/module_terminal.rs | 15 ++ 2 files changed, 170 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 9a796726..41f81964 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -2082,3 +2082,158 @@ async fn compiled_native_factory_and_hook_failures_preserve_original_finish_and_ } } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_retained_views_bound_256_and_release_only_at_actual_last_drop() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + let bootstrap = native_fixture_host(); + bootstrap + .load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let (context, probe, drops, _, _) = unsafe { terminal_fixture_symbols(&path) }; + let before = drops(); + bootstrap.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + unsafe { + drop(Box::from_raw(sender)); + } + drop(bootstrap); + let configure: unsafe extern "C" fn( + usize, + *mut c_void, + unsafe extern "C" fn(*mut c_void, u32, u32), + ) = unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let factories: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_factories") }; + let retained_live: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_retained_live") }; + let release_one: extern "C" fn() -> bool = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_one_view") }; + let release_all: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_views") }; + let mut first_token = None; + let mut admitted = 0; + let mut all_settled = true; + for _ in 0..256 { + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + unsafe { + configure(3, sender.cast(), native_event); + } + let host = native_fixture_host(); + let result = host.load_file_with_config(&path, native_controls_config(0, &controls)); + if result.is_err() { + unsafe { + drop(Box::from_raw(sender)); + } + break; + } + first_token.get_or_insert(context()); + admitted += 1; + let before = drops(); + host.shutdown(Duration::from_secs(5)).await; + let mut settled = false; + while let Ok(event) = receiver.recv_timeout(Duration::from_secs(5)) { + if event.0 == 6 { + settled = true; + break; + } + } + settled &= tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .is_ok(); + all_settled &= settled; + drop(host); + if settled { + unsafe { + drop(Box::from_raw(sender)); + } + } else { + break; + } + } + let before_factory = factories(); + let denied_host = native_fixture_host(); + let denied = + denied_host.load_file_with_config(&path, serde_json::json!({"mode":0,"test_controls":[]})); + drop(denied_host); + let factories_after_denial = factories(); + let live_after_host_drops = retained_live(); + let released_one = release_one(); + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + unsafe { + configure(3, sender.cast(), native_event); + } + let replacement = native_fixture_host(); + let replacement_result = + replacement.load_file_with_config(&path, native_controls_config(0, &controls)); + let new_token = context(); + let stale = first_token.map(probe); + let before = drops(); + replacement.shutdown(Duration::from_secs(5)).await; + let mut replacement_settled = false; + if replacement_result.is_ok() { + while let Ok(event) = receiver.recv_timeout(Duration::from_secs(5)) { + if event.0 == 6 { + replacement_settled = true; + break; + } + } + replacement_settled &= tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .is_ok(); + } + drop(replacement); + if replacement_settled { + unsafe { + drop(Box::from_raw(sender)); + } + } + let released_rest = release_all(); + // All actual controller joins and SDK capture destruction precede assertions. + // No uncertain anchor was reset; this filter starts in a fresh process. + assert!(all_settled && replacement_settled); + assert_eq!(admitted, 256); + assert!(denied.is_err()); + assert_eq!(factories_after_denial, before_factory); + assert_eq!(live_after_host_drops, 0); + assert!(released_one && replacement_result.is_ok()); + assert_eq!( + new_token & 255, + first_token.unwrap() & 255, + "the exact freed slot must be reusable" + ); + assert_ne!(new_token, first_token.unwrap()); + assert_eq!(stale, Some(TB_CLOSED)); + assert_eq!(released_rest, 256); +} diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 9b4f4786..27d70fed 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -444,3 +444,18 @@ pub extern "C" fn terminal_test_release_views() -> usize { drop(views); count } +/// Release one genuinely retained view, allowing its actual State last-Drop. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_release_one_view() -> bool { + let view = { + let mut views = RETAINED_VIEWS.lock().unwrap(); + if views.is_empty() { + None + } else { + Some(views.remove(0)) + } + }; + let released = view.is_some(); + drop(view); + released +} From e9f371049a39fd216574e84fd46e49d70cfe1842 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:25:30 +0300 Subject: [PATCH 101/130] test(tinybus-module): use closure in stale token probe Replace the point-free `map(probe)` with an explicit closure so the stale token probe compiles without relying on function item coercion. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 41f81964..3d504906 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -2194,7 +2194,7 @@ async fn compiled_native_retained_views_bound_256_and_release_only_at_actual_las let replacement_result = replacement.load_file_with_config(&path, native_controls_config(0, &controls)); let new_token = context(); - let stale = first_token.map(probe); + let stale = first_token.map(|token| probe(token)); let before = drops(); replacement.shutdown(Duration::from_secs(5)).await; let mut replacement_settled = false; From 9504d661c8270ebddbdb59ab16102efcd2d7b78d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:26:53 +0300 Subject: [PATCH 102/130] feat(tinybus-module): add fixture hooks for runtime builder failures Extend the native fixture support so tests can force a runtime builder error and inspect whether resources received by a rejected view are retained. The terminal example gains helpers to join an externally rejected controller and to query the retained resources of a rejected view. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 10 ++++- crates/tinybus-module/src/terminal.rs | 17 +++++++++ crates/tinybus/examples/module_terminal.rs | 44 +++++++++++++++++++++- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 6597afe2..9816841e 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1672,7 +1672,15 @@ where if let Some(owner) = &state.owner { owner.configure(&mut builder); } - let runtime = match builder.worker_threads(worker_threads).enable_all().build() { + #[cfg(feature = "__native-fixture-support")] + let build = if state.view().__fixture_runtime_builder_error() { + Err(std::io::Error::other("controlled runtime Builder error")) + } else { + builder.worker_threads(worker_threads).enable_all().build() + }; + #[cfg(not(feature = "__native-fixture-support"))] + let build = builder.worker_threads(worker_threads).enable_all().build(); + let runtime = match build { Ok(runtime) => runtime, Err(_) => { state.signal(TerminalReason::SetupFailed); diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 8dbce24a..dbc1ba31 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -90,6 +90,23 @@ impl NativeInstanceLiveness { } #[doc(hidden)] #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_runtime_builder_error(&self) -> bool { + self.inner.startup_failure_stage.load(Ordering::Acquire) == 9 + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_received_resources_are_retained(&self) -> bool { + let resources = self + .inner + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + resources.runtime.is_some() + && resources.retained_factory.is_some() + && resources.initial.is_some() + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] pub fn __fixture_reject_startup(&self, stage: u8) { self.inner.reject_startup_for_test(stage); } diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 27d70fed..78d5145b 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -11,6 +11,7 @@ use tinybus::{Connection, Result}; static DROPPED: AtomicUsize = AtomicUsize::new(0); static EARLY: Mutex> = Mutex::new(None); static NEXT_FACTORY_MODE: AtomicUsize = AtomicUsize::new(0); +static REJECTED: Mutex>> = Mutex::new(None); static FACTORIES: AtomicUsize = AtomicUsize::new(0); static RETAINED_VIEWS: Mutex> = Mutex::new(Vec::new()); #[derive(Clone, Copy)] @@ -166,8 +167,11 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> controller.observe(10); #[cfg(feature = "__native-fixture-support")] match factory_mode { - 20 => controller.view.__fixture_reject_worker_spawn(), - 21..=28 => controller + 20 => { + controller.view.__fixture_reject_worker_spawn(); + *REJECTED.lock().unwrap() = Some(Arc::downgrade(&controller)); + } + 21..=29 => controller .view .__fixture_reject_startup((factory_mode - 20) as u8), _ => {} @@ -459,3 +463,39 @@ pub extern "C" fn terminal_test_release_one_view() -> bool { drop(view); released } + +/// Externally controlled fixture cleanup, NOT original SDK finish or completion. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_external_controller_join() -> bool { + let controller = REJECTED + .lock() + .unwrap() + .as_ref() + .and_then(std::sync::Weak::upgrade); + let Some(controller) = controller else { + return false; + }; + *controller.wake.0.lock().unwrap() = true; + controller.wake.1.notify_all(); + let worker = controller.worker.lock().unwrap().take(); + let joined = worker.is_some_and(|worker| matches!(worker.join(), Ok(true))); + controller.emit(20, u32::from(joined)); + joined +} +/// Inspects actual SDK-received resources through the genuine rejected view. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_rejected_resources() -> bool { + #[cfg(feature = "__native-fixture-support")] + return REJECTED + .lock() + .unwrap() + .as_ref() + .and_then(std::sync::Weak::upgrade) + .is_some_and(|controller| { + controller.view.__fixture_received_resources_are_retained() + && controller.view.__fixture_request_done() + && !controller.view.is_live() + }); + #[cfg(not(feature = "__native-fixture-support"))] + false +} From 12f0ec4fa0a129bacb20d1f6ff53f5740e25a8be Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:27:22 +0300 Subject: [PATCH 103/130] refactor(tinybus-module): drop unused native fixture hooks Remove the fixture-only runtime builder error injection and the received-resources probe along with the example helpers that exercised them, since the corresponding test scenarios are no longer needed. The remaining startup rejection fixture now covers the same failure paths. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 10 +---- crates/tinybus-module/src/terminal.rs | 17 --------- crates/tinybus/examples/module_terminal.rs | 44 +--------------------- 3 files changed, 3 insertions(+), 68 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 9816841e..6597afe2 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1672,15 +1672,7 @@ where if let Some(owner) = &state.owner { owner.configure(&mut builder); } - #[cfg(feature = "__native-fixture-support")] - let build = if state.view().__fixture_runtime_builder_error() { - Err(std::io::Error::other("controlled runtime Builder error")) - } else { - builder.worker_threads(worker_threads).enable_all().build() - }; - #[cfg(not(feature = "__native-fixture-support"))] - let build = builder.worker_threads(worker_threads).enable_all().build(); - let runtime = match build { + let runtime = match builder.worker_threads(worker_threads).enable_all().build() { Ok(runtime) => runtime, Err(_) => { state.signal(TerminalReason::SetupFailed); diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index dbc1ba31..8dbce24a 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -90,23 +90,6 @@ impl NativeInstanceLiveness { } #[doc(hidden)] #[cfg(feature = "__native-fixture-support")] - pub fn __fixture_runtime_builder_error(&self) -> bool { - self.inner.startup_failure_stage.load(Ordering::Acquire) == 9 - } - #[doc(hidden)] - #[cfg(feature = "__native-fixture-support")] - pub fn __fixture_received_resources_are_retained(&self) -> bool { - let resources = self - .inner - .resources - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - resources.runtime.is_some() - && resources.retained_factory.is_some() - && resources.initial.is_some() - } - #[doc(hidden)] - #[cfg(feature = "__native-fixture-support")] pub fn __fixture_reject_startup(&self, stage: u8) { self.inner.reject_startup_for_test(stage); } diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 78d5145b..27d70fed 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -11,7 +11,6 @@ use tinybus::{Connection, Result}; static DROPPED: AtomicUsize = AtomicUsize::new(0); static EARLY: Mutex> = Mutex::new(None); static NEXT_FACTORY_MODE: AtomicUsize = AtomicUsize::new(0); -static REJECTED: Mutex>> = Mutex::new(None); static FACTORIES: AtomicUsize = AtomicUsize::new(0); static RETAINED_VIEWS: Mutex> = Mutex::new(Vec::new()); #[derive(Clone, Copy)] @@ -167,11 +166,8 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> controller.observe(10); #[cfg(feature = "__native-fixture-support")] match factory_mode { - 20 => { - controller.view.__fixture_reject_worker_spawn(); - *REJECTED.lock().unwrap() = Some(Arc::downgrade(&controller)); - } - 21..=29 => controller + 20 => controller.view.__fixture_reject_worker_spawn(), + 21..=28 => controller .view .__fixture_reject_startup((factory_mode - 20) as u8), _ => {} @@ -463,39 +459,3 @@ pub extern "C" fn terminal_test_release_one_view() -> bool { drop(view); released } - -/// Externally controlled fixture cleanup, NOT original SDK finish or completion. -#[unsafe(no_mangle)] -pub extern "C" fn terminal_test_external_controller_join() -> bool { - let controller = REJECTED - .lock() - .unwrap() - .as_ref() - .and_then(std::sync::Weak::upgrade); - let Some(controller) = controller else { - return false; - }; - *controller.wake.0.lock().unwrap() = true; - controller.wake.1.notify_all(); - let worker = controller.worker.lock().unwrap().take(); - let joined = worker.is_some_and(|worker| matches!(worker.join(), Ok(true))); - controller.emit(20, u32::from(joined)); - joined -} -/// Inspects actual SDK-received resources through the genuine rejected view. -#[unsafe(no_mangle)] -pub extern "C" fn terminal_test_rejected_resources() -> bool { - #[cfg(feature = "__native-fixture-support")] - return REJECTED - .lock() - .unwrap() - .as_ref() - .and_then(std::sync::Weak::upgrade) - .is_some_and(|controller| { - controller.view.__fixture_received_resources_are_retained() - && controller.view.__fixture_request_done() - && !controller.view.is_live() - }); - #[cfg(not(feature = "__native-fixture-support"))] - false -} From 1685c57b99645483a705da39f2b558fe17366b1c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:28:07 +0300 Subject: [PATCH 104/130] test(tinybus-module): use contains for event assertion Replace the manual iterator any check with a direct contains call on the collected events, which expresses the same assertion more clearly. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 3d504906..05bba6b1 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -2060,7 +2060,7 @@ async fn compiled_native_factory_and_hook_failures_preserve_original_finish_and_ // before these assertions. Factory failures never launch a controller. if mode < 3 { assert!(result.is_err()); - assert!(seen.iter().any(|event| *event == (19, 1))); + assert!(seen.contains(&(19, 1))); assert!(!seen.iter().any(|event| matches!(event.0, 2 | 3 | 5 | 7))); } else { assert!(result.is_ok()); From 180d36600d88c5a4d964ceae5b101ca182d38d6b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:30:23 +0300 Subject: [PATCH 105/130] test(tinybus-module): wait for terminal entry event before probing stale context The native terminal test now drains events until the expected entry event arrives instead of assuming the first event is the entry, since the number of preceding events differs by mode. This keeps the stale-context probe and shutdown ordering correct for both modes. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 05bba6b1..05346b91 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -941,10 +941,17 @@ async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stal let (context, old_probe, drops, _, _) = unsafe { terminal_fixture_symbols(&path) }; let dropped_before = dropped_total; let actual_context = context(); - let first = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); + let mut seen = Vec::new(); + let actual_entered = if mode == 4 { 16 } else { 2 }; + loop { + let event = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); + seen.push(event); + if event.0 == actual_entered { + break; + } + } let stale_result = old_context.map(|token| old_probe(token)); host.shutdown(Duration::from_secs(5)).await; - let mut seen = vec![first]; while !seen.iter().any(|(event, _)| *event == 6) { seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); } From c8d841a0aa15c9d924656ce2e42f8dca8e5b92a9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:41:43 +0300 Subject: [PATCH 106/130] feat(tinybus-module): expose terminal fixture hooks for resource and transport checks Extend the native fixture support so tests can observe terminal resource retention, confirm transport disposal on the physical worker thread, and force a controlled runtime build failure. The example fixture gains matching entry points and an external controller handle so SDK spawn rejection can be cleaned up without claiming an SDK finish. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 24 +++++++- crates/tinybus-module/src/terminal.rs | 40 ++++++++++++- crates/tinybus/examples/module_terminal.rs | 69 +++++++++++++++++++++- 3 files changed, 126 insertions(+), 7 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 6597afe2..b000b1f7 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -596,7 +596,17 @@ where owner.configure(&mut builder); } - let runtime = match builder.worker_threads(worker_threads).enable_all().build() { + #[cfg(any(test, feature = "__native-fixture-support"))] + let build_result = if state.reject_runtime_build_for_fixture() { + Err(std::io::Error::other( + "controlled terminal runtime build failure", + )) + } else { + builder.worker_threads(worker_threads).enable_all().build() + }; + #[cfg(not(any(test, feature = "__native-fixture-support")))] + let build_result = builder.worker_threads(worker_threads).enable_all().build(); + let runtime = match build_result { Ok(runtime) => runtime, Err(_) => return TB_CLOSED, }; @@ -1672,7 +1682,17 @@ where if let Some(owner) = &state.owner { owner.configure(&mut builder); } - let runtime = match builder.worker_threads(worker_threads).enable_all().build() { + #[cfg(any(test, feature = "__native-fixture-support"))] + let build_result = if state.reject_runtime_build_for_fixture() { + Err(std::io::Error::other( + "controlled terminal runtime build failure", + )) + } else { + builder.worker_threads(worker_threads).enable_all().build() + }; + #[cfg(not(any(test, feature = "__native-fixture-support")))] + let build_result = builder.worker_threads(worker_threads).enable_all().build(); + let runtime = match build_result { Ok(runtime) => runtime, Err(_) => { state.signal(TerminalReason::SetupFailed); diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 8dbce24a..a980c048 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -93,6 +93,16 @@ impl NativeInstanceLiveness { pub fn __fixture_reject_startup(&self, stage: u8) { self.inner.reject_startup_for_test(stage); } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_resources(&self) -> u32 { + self.inner.resources_for_fixture() + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_transport_dropped_on_physical_worker(&self) -> bool { + self.inner.transport_drop_thread.lock().unwrap().as_deref() == Some("tinybus-terminal") + } /// Persist genuine fault and close independent native admission first. pub fn request_fault(&self) { self.inner.signal(TerminalReason::Fault); @@ -265,7 +275,7 @@ pub(super) struct TerminalState { reject_worker_spawn: AtomicBool, #[cfg(any(test, feature = "__native-fixture-support"))] startup_failure_stage: AtomicU8, - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] transport_drop_thread: StdMutex>, lease: SlotLease, token: usize, @@ -276,7 +286,7 @@ pub(super) struct SdkOwner { state: Arc, } impl SdkOwner { - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] pub(super) fn record_transport_drop_for_test(&self) { *self.state.transport_drop_thread.lock().unwrap() = Some( std::thread::current() @@ -365,7 +375,7 @@ impl TerminalState { reject_worker_spawn: AtomicBool::new(false), #[cfg(any(test, feature = "__native-fixture-support"))] startup_failure_stage: AtomicU8::new(0), - #[cfg(test)] + #[cfg(any(test, feature = "__native-fixture-support"))] transport_drop_thread: StdMutex::new(None), lease: SlotLease { admission: Arc::downgrade(admission), @@ -632,6 +642,30 @@ impl TerminalState { pub(super) fn transport_drop_thread_for_test(&self) -> Option { self.transport_drop_thread.lock().unwrap().clone() } + #[cfg(feature = "__native-fixture-support")] + fn resources_for_fixture(&self) -> u32 { + let resources = self.resources.lock().unwrap(); + let flags = [ + resources.retained_factory.is_some(), + resources.setup.is_some(), + resources.reinitializer_factory.is_some(), + resources.runtime.is_some(), + resources.transport.is_some(), + resources.reinitialize.lock().unwrap().is_some(), + resources.initial.is_some(), + resources.config.is_some(), + resources.initial_future.lock().unwrap().is_some(), + resources.reinit_future.lock().unwrap().is_some(), + ]; + flags + .iter() + .enumerate() + .fold(0, |mask, (index, held)| mask | (u32::from(*held) << index)) + } + #[cfg(any(test, feature = "__native-fixture-support"))] + pub(super) fn reject_runtime_build_for_fixture(&self) -> bool { + self.startup_failure_stage.load(Ordering::Acquire) == 9 + } #[cfg(any(test, feature = "__native-fixture-support"))] pub(super) fn reject_startup_for_test(&self, stage: u8) { self.startup_failure_stage.store(stage, Ordering::Release); diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 27d70fed..0d89f325 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -12,6 +12,7 @@ static DROPPED: AtomicUsize = AtomicUsize::new(0); static EARLY: Mutex> = Mutex::new(None); static NEXT_FACTORY_MODE: AtomicUsize = AtomicUsize::new(0); static FACTORIES: AtomicUsize = AtomicUsize::new(0); +static EXTERNAL_CONTROLLER: Mutex>> = Mutex::new(None); static RETAINED_VIEWS: Mutex> = Mutex::new(Vec::new()); #[derive(Clone, Copy)] struct NativeQuery { @@ -104,6 +105,20 @@ impl Drop for Controller { DROPPED.fetch_add(1, Ordering::Release); } } +struct FactoryCapture(Arc); +impl FactoryCapture { + fn controller(&self) -> &Arc { + &self.0 + } +} +impl Drop for FactoryCapture { + fn drop(&mut self) { + self.0.emit( + 20, + u32::from(std::thread::current().name() == Some("tinybus-terminal")), + ); + } +} struct SetupFuture { controller: Arc, mode: u8, @@ -139,7 +154,7 @@ type FixtureSetup = Box FixtureFuture + Send + Syn fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> { FACTORIES.fetch_add(1, Ordering::AcqRel); let factory_mode = NEXT_FACTORY_MODE.swap(0, Ordering::AcqRel); - if factory_mode == 3 { + if factory_mode == 3 || (20..=29).contains(&factory_mode) { RETAINED_VIEWS.lock().unwrap().push(view.clone()); } let early = EARLY @@ -167,16 +182,21 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> #[cfg(feature = "__native-fixture-support")] match factory_mode { 20 => controller.view.__fixture_reject_worker_spawn(), - 21..=28 => controller + 21..=29 => controller .view .__fixture_reject_startup((factory_mode - 20) as u8), _ => {} } + if factory_mode == 20 { + *EXTERNAL_CONTROLLER.lock().unwrap() = Some(Arc::downgrade(&controller)); + } if held_finish { controller.view.request_fault(); } let (request, finish) = (controller.clone(), controller.clone()); + let capture = FactoryCapture(controller); let setup = move |_: Connection, config: Config| { + let controller = capture.controller(); controller.observe(16); if config.test_controls.len() != size_of::() { return Box::pin(async { Err(tinybus::Error::failed("bad test controls")) }) @@ -240,6 +260,8 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> let hooks = RuntimeHooks::new( move |_| { request.observe(12); + #[cfg(feature = "__native-fixture-support")] + request.emit(22, request.view.__fixture_resources()); *request .wake .0 @@ -459,3 +481,46 @@ pub extern "C" fn terminal_test_release_one_view() -> bool { drop(view); released } + +/// Observes resources of the last genuinely escaped factory view. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_resources() -> u32 { + #[cfg(feature = "__native-fixture-support")] + { + let view = RETAINED_VIEWS.lock().unwrap().last().cloned(); + view.map_or(0, |view| view.__fixture_resources()) + } + #[cfg(not(feature = "__native-fixture-support"))] + { + 0 + } +} +/// Observes actual received transport disposal, independently of finish. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_transport_drop() -> bool { + #[cfg(feature = "__native-fixture-support")] + { + let view = RETAINED_VIEWS.lock().unwrap().last().cloned(); + view.is_some_and(|view| view.__fixture_transport_dropped_on_physical_worker()) + } + #[cfg(not(feature = "__native-fixture-support"))] + { + false + } +} +/// Explicit fixture cleanup after SDK spawn rejection; never claims SDK finish. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_external_join() -> bool { + let controller = EXTERNAL_CONTROLLER + .lock() + .unwrap() + .take() + .and_then(|weak| weak.upgrade()); + let Some(controller) = controller else { + return false; + }; + let worker = controller.worker.lock().unwrap().take(); + let joined = worker.is_some_and(|worker| worker.join().unwrap_or(false)); + controller.emit(21, u32::from(joined)); + joined +} From f0bb67654b0b9a87111507cc0202dbf2136d1999 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:42:55 +0300 Subject: [PATCH 107/130] chore: files changed crates/tinybus-module/src/lib_terminal_tests.rs Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 05346b91..d697c0f1 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -2244,3 +2244,171 @@ async fn compiled_native_retained_views_bound_256_and_release_only_at_actual_las assert_eq!(stale, Some(TB_CLOSED)); assert_eq!(released_rest, 256); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_spawn_rejection_retains_received_runtime_task_and_original_finish() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + // Keep the library loaded while resolving its fixture controls. + let warm = native_fixture_host(); + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let configure: unsafe extern "C" fn( + usize, + *mut c_void, + unsafe extern "C" fn(*mut c_void, u32, u32), + ) = unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let resources: extern "C" fn() -> u32 = + unsafe { native_fixture_symbol(&path, c"terminal_test_resources") }; + let join: extern "C" fn() -> bool = + unsafe { native_fixture_symbol(&path, c"terminal_test_external_join") }; + let (_, _, _, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + // The following control context is deliberately retained with the uncertain + // original SDK owner. The warm context is independent. + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + unsafe { + configure(20, sender.cast(), native_event); + } + let host = native_fixture_host(); + host.load_file_with_config(&path, native_controls_config(1, &controls)) + .unwrap(); + let mut seen = Vec::new(); + loop { + let event = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); + seen.push(event); + if event.0 == 2 { + break; + } + } + host.shutdown(Duration::from_millis(30)).await; + // Physical spawn failed. Only this fixture cleanup joins its independently + // received controller; original finish and SDK settlement never run. + let externally_joined = join(); + let held = resources(); + host.shutdown(Duration::from_millis(30)).await; + drop(host); + let after = resources(); + seen.extend(receiver.try_iter()); + assert!(externally_joined); + assert_eq!(seen.iter().filter(|event| event.0 == 21).count(), 1); + assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); + assert!(!seen.iter().any(|event| matches!(event.0, 4 | 5 | 6 | 20))); + // Actual retained F, Runtime, transport, reinit callback, sole initial + // JoinHandle and already-polled owned future. No timeout/refund inference. + let expected = 1 | 8 | 16 | 32 | 64 | 256; + assert_eq!(held & expected, expected); + assert_eq!(after & expected, expected); + assert_ne!(flags() & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!( + flags() & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_startup_transfers_and_runtime_build_error_dispose_on_physical_worker() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let warm = native_fixture_host(); + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let configure: unsafe extern "C" fn( + usize, + *mut c_void, + unsafe extern "C" fn(*mut c_void, u32, u32), + ) = unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let resources: extern "C" fn() -> u32 = + unsafe { native_fixture_symbol(&path, c"terminal_test_resources") }; + let transport_drop: extern "C" fn() -> bool = + unsafe { native_fixture_symbol(&path, c"terminal_test_transport_drop") }; + let release: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_views") }; + let (_, _, drops, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + for (stage, expected) in [ + (1, 129u32), + (2, 3), + (3, 7), + (4, 15), + (5, 31), + (6, 59), + (7, 123), + (8, 123), + (9, 7), + ] { + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + unsafe { + configure(20 + stage, sender.cast(), native_event); + } + let before = drops(); + let host = native_fixture_host(); + let result = host.load_file_with_config(&path, native_controls_config(1, &controls)); + let mut seen = Vec::new(); + // Both real F and SDK-owned C destruction plus controller last Drop + // settle before assertions. No task has entered trusted F at stages7/8. + while ![6, 19, 20] + .iter() + .all(|kind| seen.iter().any(|event: &(u32, u32)| event.0 == *kind)) + { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before || resources() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let dropped_transport = transport_drop(); + host.shutdown(Duration::from_secs(5)).await; + drop(host); + let native = flags(); + release(); + seen.extend(receiver.try_iter()); + unsafe { + drop(Box::from_raw(sender)); + } + assert!(result.is_err(), "stage {stage}"); + assert!( + seen.contains(&(22, expected)), + "received stage {stage}: {seen:?}" + ); + assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert!(seen.contains(&(6, 1))); + assert!(seen.contains(&(19, 1))); + assert!(seen.contains(&(20, 1))); + assert!(!seen.iter().any(|event| event.0 == 2)); + assert_eq!(dropped_transport, (5..=8).contains(&stage), "stage {stage}"); + assert_ne!(native & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!( + native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + } +} From 29d29f28e0b4a665ccd71030ea899f18b9706c48 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:43:35 +0300 Subject: [PATCH 108/130] refactor(module): remove fixture-only runtime build failure hook The runtime construction no longer branches on a test-only flag to simulate a build failure, so the builder is now invoked directly. This drops the `__native-fixture-support` conditional and the associated state check. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index b000b1f7..5ee8cefb 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -596,17 +596,7 @@ where owner.configure(&mut builder); } - #[cfg(any(test, feature = "__native-fixture-support"))] - let build_result = if state.reject_runtime_build_for_fixture() { - Err(std::io::Error::other( - "controlled terminal runtime build failure", - )) - } else { - builder.worker_threads(worker_threads).enable_all().build() - }; - #[cfg(not(any(test, feature = "__native-fixture-support")))] - let build_result = builder.worker_threads(worker_threads).enable_all().build(); - let runtime = match build_result { + let runtime = match builder.worker_threads(worker_threads).enable_all().build() { Ok(runtime) => runtime, Err(_) => return TB_CLOSED, }; From 903c678f8a7727f10844e6519293ebc8f654a512 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:47:03 +0300 Subject: [PATCH 109/130] feat(tinybus): add native reinit and unowned fixture hooks Extend the terminal module example with new fixture modes that exercise reinitialization failures, panics during setup, and drop-time emissions, and expose exported helpers for invoking the SDK reinitializer and checking owner state from an external caller. The transport drop hook is now also compiled under the native fixture support feature so these paths can be observed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 2 +- crates/tinybus/examples/module_terminal.rs | 47 +++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 5ee8cefb..81d7b9d1 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1311,7 +1311,7 @@ struct TerminalTransport { inner: ModuleTransport, _owner: terminal::SdkOwner, } -#[cfg(test)] +#[cfg(any(test, feature = "__native-fixture-support"))] impl Drop for TerminalTransport { fn drop(&mut self) { self._owner.record_transport_drop_for_test(); diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 0d89f325..624a3945 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -132,7 +132,14 @@ impl Future for SetupFuture { self.controller.observe(11); self.controller.emit(2, u32::from(self.mode)); } + if self.mode == 41 || self.mode == 43 || self.mode == 45 { + self.controller.emit(24, u32::from(self.mode)); + } match self.mode { + 43 => Poll::Ready(Err(tinybus::Error::failed( + "ready native reinit source error", + ))), + 45 => std::panic::panic_any(PrimaryPayload(*self.controller.controls.lock().unwrap())), 0 => Poll::Ready(Ok(())), 2 => panic!("controlled native setup poll"), 5 => Poll::Ready(Err(tinybus::Error::failed("controlled native setup error"))), @@ -142,6 +149,9 @@ impl Future for SetupFuture { } impl Drop for SetupFuture { fn drop(&mut self) { + if self.mode == 42 { + self.controller.emit(25, 0); + } self.controller.observe(14); self.controller.emit( 4, @@ -154,7 +164,7 @@ type FixtureSetup = Box FixtureFuture + Send + Syn fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> { FACTORIES.fetch_add(1, Ordering::AcqRel); let factory_mode = NEXT_FACTORY_MODE.swap(0, Ordering::AcqRel); - if factory_mode == 3 || (20..=29).contains(&factory_mode) { + if factory_mode == 3 || (20..=29).contains(&factory_mode) || factory_mode == 46 { RETAINED_VIEWS.lock().unwrap().push(view.clone()); } let early = EARLY @@ -179,6 +189,7 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> worker: Mutex::new(None), }); controller.observe(10); + controller.observe(27); #[cfg(feature = "__native-fixture-support")] match factory_mode { 20 => controller.view.__fixture_reject_worker_spawn(), @@ -198,6 +209,8 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> let setup = move |_: Connection, config: Config| { let controller = capture.controller(); controller.observe(16); + controller.emit(26, u32::from(tokio::runtime::Handle::try_current().is_ok())); + controller.observe(28); if config.test_controls.len() != size_of::() { return Box::pin(async { Err(tinybus::Error::failed("bad test controls")) }) as Pin> + Send>>; @@ -248,6 +261,12 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> ); } drop(worker); + if config.mode == 40 || config.mode == 44 { + controller.emit(23, u32::from(config.mode)); + } + if config.mode == 44 { + std::panic::panic_any(PrimaryPayload(*controller.controls.lock().unwrap())); + } if config.mode == 4 { panic!("controlled native setup construction"); } @@ -524,3 +543,29 @@ pub extern "C" fn terminal_test_external_join() -> bool { controller.emit(21, u32::from(joined)); joined } + +/// Invoke the exact exported SDK reinitializer with a previously observed token. +/// # Safety +/// Input must remain readable throughout this synchronous callback. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn terminal_test_reinit(token: usize, ptr: *const u8, len: usize) -> i32 { + let table = *LATEST.lock().unwrap(); + table + .and_then(|table| table.0.reinitialize) + .map_or(TB_CLOSED, |reinit| unsafe { + reinit(std::ptr::with_exposed_provenance_mut(token), ptr, len) + }) +} +/// Current SDK owner observation on an external caller, with no scope installed. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_unowned() -> bool { + #[cfg(feature = "__native-fixture-support")] + { + let views = RETAINED_VIEWS.lock().unwrap().clone(); + views.iter().all(|view| !view.__fixture_owner_is_current()) + } + #[cfg(not(feature = "__native-fixture-support"))] + { + true + } +} From d249e33f8be53f5cc3a673dcf92fbc7a11cdb2a9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:49:15 +0300 Subject: [PATCH 110/130] test(tinybus-module): cover native terminal shutdown barriers Add an ignored multi-threaded test that drives the native terminal fixture through blocked construction, poll, and drop paths, both on initial startup and during reinit, to verify that entered work settles before the original finish and that genuine source errors or panics dominate a simultaneous ready stop. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index d697c0f1..4f08d0e6 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -2412,3 +2412,167 @@ async fn compiled_native_startup_transfers_and_runtime_build_error_dispose_on_ph ); } } + +struct NativeBarrierControls { + events: std::sync::mpsc::Sender<(u32, u32)>, + blocked_event: u32, + release: (StdMutex, std::sync::Condvar), +} +impl NativeBarrierControls { + fn release(&self) { + *self.release.0.lock().unwrap() = true; + self.release.1.notify_all(); + } +} +unsafe extern "C" fn native_barrier_event(ctx: *mut c_void, event: u32, value: u32) { + let controls = unsafe { &*ctx.cast::() }; + let _ = controls.events.send((event, value)); + if event == controls.blocked_event { + let mut released = controls.release.0.lock().unwrap(); + while !*released { + released = controls.release.1.wait(released).unwrap(); + } + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_entered_initial_and_reinit_work_settle_before_original_finish() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + // Modes40/41 block actual synchronous construction/poll; mode42 blocks + // actual retained future Drop. Reinit43 returns a genuine ready Err only + // after the same poll has observed an already-committed real host Stop. + for (reinitializing, mode) in [ + (false, 40u8), + (false, 41), + (false, 42), + (false, 44), + (false, 45), + (true, 40), + (true, 41), + (true, 42), + (true, 43), + (true, 44), + (true, 45), + ] { + let (events, receiver) = std::sync::mpsc::channel(); + let barrier = Box::into_raw(Box::new(NativeBarrierControls { + events, + blocked_event: match mode { + 40 | 44 => 23, + 42 => 25, + _ => 24, + }, + release: (StdMutex::new(false), std::sync::Condvar::new()), + })); + let controls = NativeControls { + ctx: barrier.cast(), + event: native_barrier_event, + }; + let host = native_fixture_host(); + host.load_file_with_config( + &path, + native_controls_config(if reinitializing { 0 } else { mode }, &controls), + ) + .unwrap(); + let (context, _, drops, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + let reinit: unsafe extern "C" fn(usize, *const u8, usize) -> i32 = + unsafe { native_fixture_symbol(&path, c"terminal_test_reinit") }; + let before = drops(); + let mut seen = Vec::new(); + if reinitializing { + while !seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + let call = reinitializing.then(|| { + let token = context(); + let bytes = serde_json::to_vec(&native_controls_config(mode, &controls)).unwrap(); + std::thread::spawn(move || unsafe { reinit(token, bytes.as_ptr(), bytes.len()) }) + }); + // Drop42 is first entered on actual physical cleanup after shutdown. + if mode != 42 { + while !seen + .iter() + .any(|event| event.0 == unsafe { &*barrier }.blocked_event) + { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } else if !reinitializing { + while !seen.iter().any(|event| event.0 == 2) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } else { + while seen.iter().filter(|event| event.0 == 2).count() != 2 { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + host.shutdown(Duration::from_millis(30)).await; + if mode == 42 { + while !seen.iter().any(|event| event.0 == 25) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + let blocked_native = flags(); + seen.extend(receiver.try_iter()); + let finish_before_release = seen.iter().any(|event| event.0 == 5); + let drop_before_release = drops(); + // Release before any assertion; join the real ABI caller, actual + // initial task/future and original independent controller below. + unsafe { &*barrier }.release(); + let reinit_result = call.map(|call| call.join().unwrap()); + host.shutdown(Duration::from_secs(5)).await; + while !seen.iter().any(|event| event.0 == 6) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let settled_native = flags(); + seen.extend(receiver.try_iter()); + drop(host); + unsafe { + drop(Box::from_raw(barrier)); + } + assert!( + !finish_before_release, + "entered mode {mode}, reinit {reinitializing}" + ); + assert_eq!(drop_before_release, before); + assert_ne!( + blocked_native & tinybus::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT, + 0 + ); + assert_eq!( + blocked_native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (7, 1)).count(), 1); + assert!(seen.iter().any(|event| event.0 == 13 && event.1 & 7 == 7)); + assert!(seen.iter().any(|event| event.0 == 16 && event.1 & 1 == 1)); + assert!( + seen.iter() + .filter(|event| event.0 == 26) + .all(|event| event.1 == 1) + ); + if let Some(result) = reinit_result { + assert_eq!(result, if mode == 44 { TB_PANICKED } else { TB_CLOSED }); + } + if mode == 43 || mode == 44 || mode == 45 { + assert_ne!( + settled_native & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, + 0, + "genuine source error/panic must dominate simultaneous ready Stop" + ); + assert_eq!( + settled_native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert!(!seen.iter().any(|event| event.0 == 17)); + } + } +} From 7438870687b008b88f91127563a8a472f59e4ab8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:52:34 +0300 Subject: [PATCH 111/130] test(module): cover sdk owner retention across payload and connection drops Add an ignored native fixture test that exercises method-payload and whole-connection teardown paths, asserting that actual SDK-owned Connection tasks and drops survive until the controlled work settles. The terminal example gains matching fixture modes, a Panic method, and an owner-counting export so the test can observe real ownership rather than messages or futures. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 156 ++++++++++++++++++ crates/tinybus/examples/module_terminal.rs | 73 +++++++- 2 files changed, 227 insertions(+), 2 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 4f08d0e6..b78be935 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -2576,3 +2576,159 @@ async fn compiled_native_entered_initial_and_reinit_work_settle_before_original_ } } } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_method_payload_and_whole_connection_drop_retain_actual_sdk_owners() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let warm = native_fixture_host(); + let (events, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let events = Box::into_raw(Box::new(events)); + let controls = NativeControls { + ctx: events.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let configure: unsafe extern "C" fn( + usize, + *mut c_void, + unsafe extern "C" fn(*mut c_void, u32, u32), + ) = unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let owners: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_sdk_owners") }; + let release_views: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_views") }; + let (_, _, drops, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + for mode in [50u8, 51, 52] { + let (events, receiver) = std::sync::mpsc::channel(); + let barrier = Box::into_raw(Box::new(NativeBarrierControls { + events, + blocked_event: if mode == 50 { 30 } else { 32 }, + release: (StdMutex::new(false), std::sync::Condvar::new()), + })); + let controls = NativeControls { + ctx: barrier.cast(), + event: native_barrier_event, + }; + unsafe { + configure(46, barrier.cast(), native_barrier_event); + } + let broker = tinybus::broker::Broker::new(); + let bus = tinybus::transport::memory::MemoryBus::new(); + broker.clone().spawn(bus.clone()); + let client = Connection::connect(bus.connect().await.unwrap()) + .await + .unwrap(); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + let before = drops(); + host.load_file_with_config(&path, native_controls_config(mode, &controls)) + .unwrap(); + let mut seen = Vec::new(); + while !seen.iter().any(|event: &(u32, u32)| event.0 == 31) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + let call = (mode == 50).then(|| { + tokio::spawn(async move { + client + .proxy( + "ai.tinyhumans.TerminalFixture", + "/terminal", + "ai.tinyhumans.TerminalFixture", + ) + .unwrap() + .with_timeout(Duration::from_secs(10)) + .call::("Panic", ()) + .await + }) + }); + if mode == 50 { + while !seen.iter().any(|event| event.0 == 30) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + let stopping_host = host.clone(); + let stopping = tokio::spawn(async move { + stopping_host.shutdown(Duration::from_secs(8)).await; + }); + if mode != 50 { + while !seen.iter().any(|event| event.0 == 32) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + stopping.await.unwrap(); // actual SDK Runtime shutdown_timeout elapsed + let held_owners = owners(); + let timed_out_native = flags(); + seen.extend(receiver.try_iter()); + let actual_finish = seen.iter().filter(|event| **event == (5, 1)).count(); + let actual_join = seen.iter().filter(|event| **event == (7, 1)).count(); + let retained_controller = drops() == before; + // Always settle controlled work before asserting: release real foreign + // Drop, await actual call and actual counted last Connection observer. + unsafe { &*barrier }.release(); + let call_result = match call { + Some(call) => Some(call.await.unwrap()), + None => None, + }; + while !seen.iter().any(|event| event.0 == 6) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + tokio::time::timeout(Duration::from_secs(5), async { + while owners() != 0 || drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let settled_native = flags(); + release_views(); + seen.extend(receiver.try_iter()); + drop(host); + unsafe { + drop(Box::from_raw(barrier)); + } + assert!( + held_owners > 0, + "whole SDK-owned Connection task/Drop mode {mode}" + ); + assert!(retained_controller); + assert_eq!( + actual_finish, 1, + "actual original controller finish is separate" + ); + assert_eq!(actual_join, 1); + assert_eq!( + timed_out_native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert_eq!( + settled_native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + if mode == 50 { + assert!( + seen.contains(&(29, 7)), + "Fault/request-done/actualSDKowner before first payload Drop" + ); + assert!(call_result.unwrap().is_err()); + assert_ne!( + timed_out_native & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, + 0 + ); + } else { + assert!( + seen.iter().any(|event| event.0 == 35 && event.1 & 6 == 6), + "owner already stopped, bounded request done before whole Connection Drop" + ); + if mode == 52 { + assert_ne!( + settled_native & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, + 0, + "explicit resource-drop observer fences Fault even after owner suppression" + ); + } + } + } +} diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 624a3945..15987644 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -206,7 +206,7 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> } let (request, finish) = (controller.clone(), controller.clone()); let capture = FactoryCapture(controller); - let setup = move |_: Connection, config: Config| { + let setup = move |connection: Connection, config: Config| { let controller = capture.controller(); controller.observe(16); controller.emit(26, u32::from(tokio::runtime::Handle::try_current().is_ok())); @@ -270,6 +270,21 @@ fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> if config.mode == 4 { panic!("controlled native setup construction"); } + if (50..=52).contains(&config.mode) { + let object = TerminalObject { + controller: controller.clone(), + mode: config.mode, + }; + let work = controller.clone(); + return Box::pin(async move { + connection.serve_at("/terminal".parse()?, object).await?; + connection + .request_name("ai.tinyhumans.TerminalFixture") + .await?; + work.emit(31, 1); + Ok(()) + }) as FixtureFuture; + } Box::pin(SetupFuture { controller: controller.clone(), mode: config.mode, @@ -328,7 +343,7 @@ pub extern "C" fn tinybus_module_manifest_v1() -> TbSlice { name: "module-terminal", version: env!("CARGO_PKG_VERSION"), provides: &["ai.tinyhumans.TerminalFixture"], - methods: &[], + methods: &["Panic"], signals: &[], requires: &[], optional: &[], @@ -569,3 +584,57 @@ pub extern "C" fn terminal_test_unowned() -> bool { true } } + +struct MethodPayload(Arc); +impl Drop for MethodPayload { + fn drop(&mut self) { + let query = *NATIVE_QUERY.lock().unwrap(); + let native = query.map_or(0, |query| unsafe { (query.query)(query.ctx) }); + #[cfg(feature = "__native-fixture-support")] + let evidence = u32::from(native & TB_NATIVE_TERMINAL_FAULT != 0) + | (u32::from(self.0.view.__fixture_request_done()) << 1) + | (u32::from(self.0.view.__fixture_sdk_owners() != 0) << 2); + #[cfg(not(feature = "__native-fixture-support"))] + let evidence = native; + // First payload-disposal action reads the actual native query and + // actual request state before entering the trusted blocking control. + self.0.emit(29, evidence); + self.0.emit(30, 0); + self.0.emit(33, 1); + } +} +struct TerminalObject { + controller: Arc, + mode: u8, +} +#[tinybus::interface(name = "ai.tinyhumans.TerminalFixture")] +impl TerminalObject { + async fn panic(&self) -> Result<()> { + std::panic::panic_any(MethodPayload(self.controller.clone())); + } +} +impl Drop for TerminalObject { + fn drop(&mut self) { + if self.mode == 51 || self.mode == 52 { + self.controller.observe(35); + self.controller.emit(32, u32::from(self.mode)); + self.controller.emit(33, u32::from(self.mode)); + if self.mode == 52 { + panic!("controlled native Connection object destructor"); + } + } + } +} +/// Counts actual surviving SDK Connection/task owners, not messages/futures. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_sdk_owners() -> usize { + #[cfg(feature = "__native-fixture-support")] + { + let view = RETAINED_VIEWS.lock().unwrap().last().cloned(); + view.map_or(0, |view| view.__fixture_sdk_owners()) + } + #[cfg(not(feature = "__native-fixture-support"))] + { + 0 + } +} From 270ed045b654351c21935929f8ad8ebb85bd1e98 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:56:21 +0300 Subject: [PATCH 112/130] chore: files changed crates/tinybus-module/src/lib_terminal_tests.rs,crates/tinybus/examples/module_ Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 242 ++++++++++++++++++ crates/tinybus/examples/module_terminal.rs | 26 +- 2 files changed, 267 insertions(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index b78be935..cb51c20e 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -2732,3 +2732,245 @@ async fn compiled_native_method_payload_and_whole_connection_drop_retain_actual_ } } } + +type NativeConfigure = + unsafe extern "C" fn(usize, *mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)); +type NativeReinit = unsafe extern "C" fn(usize, *const u8, usize) -> i32; +struct NativeNestedControls { + events: std::sync::mpsc::Sender<(u32, u32)>, + path: std::path::PathBuf, + host: Arc, + other: NativeControls, + configure: NativeConfigure, + reinit: NativeReinit, + context: extern "C" fn() -> usize, + token: std::sync::atomic::AtomicUsize, + phase: std::sync::atomic::AtomicUsize, +} +unsafe extern "C" fn native_nested_event(ctx: *mut c_void, event: u32, value: u32) { + let controls = unsafe { &*ctx.cast::() }; + let _ = controls.events.send((event, value)); + if event == 10 { + unsafe { + (controls.configure)(46, controls.other.ctx, controls.other.event); + } + let loaded = controls + .host + .load_file_with_config(&controls.path, native_controls_config(0, &controls.other)); + controls + .token + .store((controls.context)(), Ordering::Release); + let _ = controls.events.send((36, u32::from(loaded.is_ok()))); + } + if event == 26 && controls.phase.load(Ordering::Acquire) == 1 { + let bytes = serde_json::to_vec(&native_controls_config(0, &controls.other)).unwrap(); + let result = unsafe { + (controls.reinit)( + controls.token.load(Ordering::Acquire), + bytes.as_ptr(), + bytes.len(), + ) + }; + let _ = controls.events.send((37, u32::from(result == TB_OK))); + } +} +struct NativeLogCount(Arc); +impl tracing::Subscriber for NativeLogCount { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + if event.metadata().target() == "tinybus_module" { + self.0.fetch_add(1, Ordering::Release); + } + } + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_nested_factory_reinit_restore_exact_owner_and_suppress_captured_dispatch() +{ + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let warm = native_fixture_host(); + let (events, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let events = Box::into_raw(Box::new(events)); + let controls = NativeControls { + ctx: events.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let configure: NativeConfigure = + unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let reinit: NativeReinit = unsafe { native_fixture_symbol(&path, c"terminal_test_reinit") }; + let dispatch: extern "C" fn(usize) -> bool = + unsafe { native_fixture_symbol(&path, c"terminal_test_dispatch") }; + let unowned: extern "C" fn() -> bool = + unsafe { native_fixture_symbol(&path, c"terminal_test_unowned") }; + let release_views: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_views") }; + let retained_live: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_retained_live") }; + let (context, old_probe, drops, _, _) = unsafe { terminal_fixture_symbols(&path) }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + let (b_events, b_receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let b_events = Box::into_raw(Box::new(b_events)); + let b_controls = NativeControls { + ctx: b_events.cast(), + event: native_event, + }; + let host_b = native_fixture_host(); + let (a_events, a_receiver) = std::sync::mpsc::channel(); + let nested = Box::into_raw(Box::new(NativeNestedControls { + events: a_events, + path: path.clone(), + host: host_b.clone(), + other: b_controls, + configure, + reinit, + context, + token: std::sync::atomic::AtomicUsize::new(0), + phase: std::sync::atomic::AtomicUsize::new(0), + })); + let a_controls = NativeControls { + ctx: nested.cast(), + event: native_nested_event, + }; + unsafe { + configure(46, nested.cast(), native_nested_event); + } + let host_a = native_fixture_host(); + host_a + .load_file_with_config(&path, native_controls_config(0, &a_controls)) + .unwrap(); + let token_a = context(); + let token_b = unsafe { &*nested }.token.load(Ordering::Acquire); + let mut a_seen = Vec::new(); + let mut b_seen = Vec::new(); + while !a_seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + a_seen.push(a_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + while !b_seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + b_seen.push(b_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + unsafe { &*nested }.phase.store(1, Ordering::Release); + let bytes = serde_json::to_vec(&native_controls_config(0, &a_controls)).unwrap(); + let first_reinit = unsafe { reinit(token_a, bytes.as_ptr(), bytes.len()) }; + let restored_external = unowned(); + let logs = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let probe = |index| { + let before = logs.load(Ordering::Acquire); + let enabled = + tracing::subscriber::with_default(NativeLogCount(logs.clone()), || dispatch(index)); + (enabled, logs.load(Ordering::Acquire) - before) + }; + let a_live_probe = probe(0); + let b_live_probe = probe(1); + let before = drops(); + host_a.shutdown(Duration::from_secs(5)).await; + while !a_seen.iter().any(|event| event.0 == 6) { + a_seen.push(a_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let a_stopped_probe = probe(0); + let b_after_a_probe = probe(1); + let live_after_a = retained_live(); + let b_bytes = serde_json::to_vec(&native_controls_config(0, &b_controls)).unwrap(); + let b_after_a_reinit = unsafe { reinit(token_b, b_bytes.as_ptr(), b_bytes.len()) }; + host_b.shutdown(Duration::from_secs(5)).await; + while !b_seen.iter().any(|event| event.0 == 6) { + b_seen.push(b_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + let b_stopped_probe = probe(1); + // A newly loaded third native instance has its own exact owner; retired + // opaque tokens cannot select it, and old factory dispatchers remain quiet. + let (c_events, c_receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let c_events = Box::into_raw(Box::new(c_events)); + let c_controls = NativeControls { + ctx: c_events.cast(), + event: native_event, + }; + unsafe { + configure(46, c_events.cast(), native_event); + } + let host_c = native_fixture_host(); + host_c + .load_file_with_config(&path, native_controls_config(0, &c_controls)) + .unwrap(); + let mut c_seen = Vec::new(); + while !c_seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + c_seen.push(c_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + let c_live_probe = probe(2); + let old_a = old_probe(token_a); + let old_b = old_probe(token_b); + let old_dispatch_after_reload = [probe(0), probe(1)]; + let before = drops(); + host_c.shutdown(Duration::from_secs(5)).await; + while !c_seen.iter().any(|event| event.0 == 6) { + c_seen.push(c_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + a_seen.extend(a_receiver.try_iter()); + b_seen.extend(b_receiver.try_iter()); + c_seen.extend(c_receiver.try_iter()); + let finally_unowned = unowned(); + release_views(); + drop(host_a); + drop(host_b); + drop(host_c); + // All real original finishes/joins and F/controller captures settled. + unsafe { + drop(Box::from_raw(nested)); + drop(Box::from_raw(b_events)); + drop(Box::from_raw(c_events)); + } + assert_eq!(first_reinit, TB_OK); + assert_eq!(b_after_a_reinit, TB_OK); + assert!(restored_external && finally_unowned); + assert!(a_seen.contains(&(36, 1)) && a_seen.contains(&(37, 1))); + assert!( + a_seen.iter().any(|event| event.0 == 27 && event.1 & 3 == 1), + "factory B scope restores actual factory A" + ); + assert!( + a_seen + .iter() + .filter(|event| event.0 == 28) + .all(|event| event.1 & 3 == 1), + "nested B reinit restores actual worker A" + ); + assert_eq!(a_live_probe, (true, 1)); + assert_eq!(b_live_probe, (true, 1)); + assert_eq!(a_stopped_probe, (false, 0)); + assert_eq!(b_after_a_probe, (true, 1)); + assert_eq!(b_stopped_probe, (false, 0)); + assert_eq!(c_live_probe, (true, 1)); + assert_eq!(old_dispatch_after_reload, [(false, 0), (false, 0)]); + assert_eq!(live_after_a, 1); + assert_eq!((old_a, old_b), (TB_CLOSED, TB_CLOSED)); + for seen in [a_seen, b_seen, c_seen] { + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (7, 1)).count(), 1); + assert!(seen.iter().any(|event| event.0 == 13 && event.1 & 7 == 7)); + } +} diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 15987644..80a9bf66 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -12,6 +12,7 @@ static DROPPED: AtomicUsize = AtomicUsize::new(0); static EARLY: Mutex> = Mutex::new(None); static NEXT_FACTORY_MODE: AtomicUsize = AtomicUsize::new(0); static FACTORIES: AtomicUsize = AtomicUsize::new(0); +static FACTORY_DISPATCHES: Mutex> = Mutex::new(Vec::new()); static EXTERNAL_CONTROLLER: Mutex>> = Mutex::new(None); static RETAINED_VIEWS: Mutex> = Mutex::new(Vec::new()); #[derive(Clone, Copy)] @@ -149,9 +150,12 @@ impl Future for SetupFuture { } impl Drop for SetupFuture { fn drop(&mut self) { - if self.mode == 42 { + if self.mode == 42 || self.mode == 47 { self.controller.emit(25, 0); } + if self.mode == 47 { + std::panic::panic_any(PrimaryPayload(*self.controller.controls.lock().unwrap())); + } self.controller.observe(14); self.controller.emit( 4, @@ -164,6 +168,12 @@ type FixtureSetup = Box FixtureFuture + Send + Syn fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> { FACTORIES.fetch_add(1, Ordering::AcqRel); let factory_mode = NEXT_FACTORY_MODE.swap(0, Ordering::AcqRel); + if factory_mode == 46 { + FACTORY_DISPATCHES + .lock() + .unwrap() + .push(tracing::dispatcher::get_default(Clone::clone)); + } if factory_mode == 3 || (20..=29).contains(&factory_mode) || factory_mode == 46 { RETAINED_VIEWS.lock().unwrap().push(view.clone()); } @@ -638,3 +648,17 @@ pub extern "C" fn terminal_test_sdk_owners() -> usize { 0 } } + +/// Probe a dispatcher captured by the genuine factory's actual caller scope. +/// It supplies no execution owner or native admission authority. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_dispatch(index: usize) -> bool { + let dispatch = FACTORY_DISPATCHES.lock().unwrap().get(index).cloned(); + dispatch.is_some_and(|dispatch| { + tracing::dispatcher::with_default(&dispatch, || { + let enabled = !tracing::info_span!("terminal_factory_dispatch").is_disabled(); + tracing::info!("retained native factory dispatcher"); + enabled + }) + }) +} From c6e4de91d0196e07ebe4fd42bfd489edc3d6528c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:58:10 +0300 Subject: [PATCH 113/130] test(tinybus-module): cover first foreign cleanup panic keeping later resources Add an ignored multi-threaded test asserting that when the first foreign cleanup panics, later resources stay retained and the original finish is still attempted, with the terminal left faulted rather than physically stopped. The terminal example's dispatch probe now checks the span's own metadata against the dispatcher instead of relying on the global subscriber, so the probe reflects the retained factory dispatcher. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 57 +++++++++++++++++++ crates/tinybus/examples/module_terminal.rs | 5 +- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index cb51c20e..65d6acaf 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -2974,3 +2974,60 @@ async fn compiled_native_nested_factory_reinit_restore_exact_owner_and_suppress_ assert!(seen.iter().any(|event| event.0 == 13 && event.1 & 7 == 7)); } } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_first_foreign_cleanup_panic_keeps_later_resources_and_attempts_original_finish() + { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let (events, receiver) = std::sync::mpsc::channel(); + // This callback owner deliberately stays retained with the SDK's uncertain + // untouched later F/Runtime/callback resources after the first Drop failure. + let barrier = Box::into_raw(Box::new(NativeBarrierControls { + events, + blocked_event: 25, + release: (StdMutex::new(false), std::sync::Condvar::new()), + })); + let controls = NativeControls { + ctx: barrier.cast(), + event: native_barrier_event, + }; + let host = native_fixture_host(); + host.load_file_with_config(&path, native_controls_config(47, &controls)) + .unwrap(); + let (_, _, _, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + let mut seen = Vec::new(); + while !seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + host.shutdown(Duration::from_millis(30)).await; + while !seen.iter().any(|event| event.0 == 25) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + let before_release = flags(); + seen.extend(receiver.try_iter()); + let finished_early = seen.iter().any(|event| event.0 == 5); + unsafe { &*barrier }.release(); + while !seen.iter().any(|event| event.0 == 5) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + host.shutdown(Duration::from_secs(5)).await; + host.shutdown(Duration::from_millis(30)).await; + drop(host); + let after = flags(); + seen.extend(receiver.try_iter()); + assert!(!finished_early); + assert_eq!( + before_release & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (7, 1)).count(), 1); + assert!(seen.iter().any(|event| event.0 == 13 && event.1 & 7 == 7)); + assert!(!seen.iter().any(|event| matches!(event.0, 6 | 17 | 20))); + assert_ne!(after & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!( + after & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); +} diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 80a9bf66..eb2825a4 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -656,7 +656,10 @@ pub extern "C" fn terminal_test_dispatch(index: usize) -> bool { let dispatch = FACTORY_DISPATCHES.lock().unwrap().get(index).cloned(); dispatch.is_some_and(|dispatch| { tracing::dispatcher::with_default(&dispatch, || { - let enabled = !tracing::info_span!("terminal_factory_dispatch").is_disabled(); + let span = tracing::info_span!("terminal_factory_dispatch"); + let enabled = span + .metadata() + .is_some_and(|metadata| dispatch.enabled(metadata)); tracing::info!("retained native factory dispatcher"); enabled }) From 91b6f4340693235343e69bbf59a3a03f601f50ca Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 19:59:35 +0300 Subject: [PATCH 114/130] test(module): assert retained resources after first foreign cleanup panic Warm the fixture before the panic scenario and configure the callback owner so the test can observe which resources survive. The new assertion checks that the later F/Runtime/callback resources stay retained after the first Drop failure, matching the test's stated intent. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 65d6acaf..f1296393 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -2980,6 +2980,24 @@ async fn compiled_native_nested_factory_reinit_restore_exact_owner_and_suppress_ async fn compiled_native_first_foreign_cleanup_panic_keeps_later_resources_and_attempts_original_finish() { let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let warm = native_fixture_host(); + let (warm_events, warm_receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let warm_events = Box::into_raw(Box::new(warm_events)); + let warm_controls = NativeControls { + ctx: warm_events.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &warm_controls)) + .unwrap(); + let configure: NativeConfigure = + unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let resources: extern "C" fn() -> u32 = + unsafe { native_fixture_symbol(&path, c"terminal_test_resources") }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!( + warm_receiver.recv_timeout(Duration::from_secs(5)), + Ok((6, _)) + ) {} let (events, receiver) = std::sync::mpsc::channel(); // This callback owner deliberately stays retained with the SDK's uncertain // untouched later F/Runtime/callback resources after the first Drop failure. @@ -2992,6 +3010,9 @@ async fn compiled_native_first_foreign_cleanup_panic_keeps_later_resources_and_a ctx: barrier.cast(), event: native_barrier_event, }; + unsafe { + configure(46, barrier.cast(), native_barrier_event); + } let host = native_fixture_host(); host.load_file_with_config(&path, native_controls_config(47, &controls)) .unwrap(); @@ -3015,8 +3036,10 @@ async fn compiled_native_first_foreign_cleanup_panic_keeps_later_resources_and_a host.shutdown(Duration::from_millis(30)).await; drop(host); let after = flags(); + let held_resources = resources(); seen.extend(receiver.try_iter()); assert!(!finished_early); + assert_eq!(held_resources & (1 | 8 | 16 | 32), 1 | 8 | 16 | 32); assert_eq!( before_release & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0 From 7ddcefef9e71fe39b866aa4ab76b69ce05965a9f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 20:05:59 +0300 Subject: [PATCH 115/130] test(module): cover prior reinit future drop settlement Add an ignored terminal test that exercises dropping a previous reinit future across modes 48 and 49, asserting the configured Tokio handle is used and that settlement and retained resources behave as expected. The native fixture example now supports the new modes so the test can drive the drop path and observe the resulting events. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 120 ++++++++++++++++++ crates/tinybus/examples/module_terminal.rs | 10 +- 2 files changed, 127 insertions(+), 3 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index f1296393..6bc67e22 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -3054,3 +3054,123 @@ async fn compiled_native_first_foreign_cleanup_panic_keeps_later_resources_and_a 0 ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_prior_reinit_future_drop_uses_configured_handle_and_retains_settlement() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let warm = native_fixture_host(); + let (events, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let events = Box::into_raw(Box::new(events)); + let controls = NativeControls { + ctx: events.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let configure: NativeConfigure = + unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let reinit: NativeReinit = unsafe { native_fixture_symbol(&path, c"terminal_test_reinit") }; + let resources: extern "C" fn() -> u32 = + unsafe { native_fixture_symbol(&path, c"terminal_test_resources") }; + let release_views: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_views") }; + let (context, _, drops, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + for mode in [48u8, 49] { + let (events, receiver) = std::sync::mpsc::channel(); + let barrier = Box::into_raw(Box::new(NativeBarrierControls { + events, + blocked_event: 25, + release: (StdMutex::new(false), std::sync::Condvar::new()), + })); + let controls = NativeControls { + ctx: barrier.cast(), + event: native_barrier_event, + }; + unsafe { + configure(46, barrier.cast(), native_barrier_event); + } + let host = native_fixture_host(); + let before = drops(); + host.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let mut seen = Vec::new(); + while !seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + let token = context(); + let bytes = serde_json::to_vec(&native_controls_config(mode, &controls)).unwrap(); + let first_result = unsafe { reinit(token, bytes.as_ptr(), bytes.len()) }; + let next_bytes = serde_json::to_vec(&native_controls_config(0, &controls)).unwrap(); + let next = std::thread::spawn(move || unsafe { + reinit(token, next_bytes.as_ptr(), next_bytes.len()) + }); + while !seen.iter().any(|event| event.0 == 25) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + host.shutdown(Duration::from_millis(30)).await; + let held_runtime = resources() & 8 != 0; + let held_native = flags(); + seen.extend(receiver.try_iter()); + let finish_before_release = seen.iter().any(|event| event.0 == 5); + // Real previous-future Drop owns the sole pending retirement operation. + // Release it and join real second ABI caller before asserting context. + unsafe { &*barrier }.release(); + let second_result = next.join().unwrap(); + host.shutdown(Duration::from_secs(5)).await; + while !seen + .iter() + .any(|event| event.0 == if mode == 48 { 6 } else { 5 }) + { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + if mode == 48 { + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } + let after = flags(); + let later_resources = resources(); + seen.extend(receiver.try_iter()); + drop(host); + release_views(); + if mode == 48 { + unsafe { + drop(Box::from_raw(barrier)); + } + } + // Mode49 context stays conservatively retained with later resources. + assert_eq!(first_result, TB_OK); + assert_eq!(second_result, TB_CLOSED); + assert!(held_runtime && !finish_before_release); + assert_ne!( + held_native & tinybus::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT, + 0 + ); + assert_eq!( + held_native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (7, 1)).count(), 1); + assert!( + seen.contains(&(38, 1)), + "actual previous future Drop requires its configured Tokio Handle: {seen:?}" + ); + if mode == 49 { + assert_eq!(later_resources & (1 | 8 | 16 | 32), 1 | 8 | 16 | 32); + assert!(!seen.iter().any(|event| matches!(event.0, 6 | 17 | 20))); + assert_ne!(after & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!( + after & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + } + } +} diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index eb2825a4..539558f3 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -141,7 +141,7 @@ impl Future for SetupFuture { "ready native reinit source error", ))), 45 => std::panic::panic_any(PrimaryPayload(*self.controller.controls.lock().unwrap())), - 0 => Poll::Ready(Ok(())), + 0 | 48 | 49 => Poll::Ready(Ok(())), 2 => panic!("controlled native setup poll"), 5 => Poll::Ready(Err(tinybus::Error::failed("controlled native setup error"))), _ => Poll::Pending, @@ -150,10 +150,14 @@ impl Future for SetupFuture { } impl Drop for SetupFuture { fn drop(&mut self) { - if self.mode == 42 || self.mode == 47 { + if self.mode == 48 || self.mode == 49 { + self.controller + .emit(38, u32::from(tokio::runtime::Handle::try_current().is_ok())); + } + if self.mode == 42 || self.mode == 47 || self.mode == 48 || self.mode == 49 { self.controller.emit(25, 0); } - if self.mode == 47 { + if self.mode == 47 || self.mode == 49 { std::panic::panic_any(PrimaryPayload(*self.controller.controls.lock().unwrap())); } self.controller.observe(14); From 8ad52bf5a4f4eca20779335b5f10ab5693f5be6b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 20:07:42 +0300 Subject: [PATCH 116/130] test(tinybus-module): assert owner restoration in prior reinit drop The terminal fixture now records whether the previous future's Drop runs on the tinybus-terminal worker thread and expects the construction Handle guard to end before its ack, so the test verifies that the owner is restored rather than merely that a configured Handle was available. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 11 ++++++++--- crates/tinybus/examples/module_terminal.rs | 5 ++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 6bc67e22..de48bb87 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -3057,7 +3057,7 @@ async fn compiled_native_first_foreign_cleanup_panic_keeps_later_resources_and_a #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] -async fn compiled_native_prior_reinit_future_drop_uses_configured_handle_and_retains_settlement() { +async fn compiled_native_prior_reinit_future_drop_restores_owner_and_retains_settlement() { let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); let warm = native_fixture_host(); let (events, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); @@ -3160,8 +3160,13 @@ async fn compiled_native_prior_reinit_future_drop_uses_configured_handle_and_ret assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); assert_eq!(seen.iter().filter(|event| **event == (7, 1)).count(), 1); assert!( - seen.contains(&(38, 1)), - "actual previous future Drop requires its configured Tokio Handle: {seen:?}" + seen.contains(&(38, 0)), + "construction Handle guard ends before its ack and later foreign Drop: {seen:?}" + ); + assert!(seen.iter().any(|event| event.0 == 39 && event.1 & 1 == 1)); + assert!( + seen.contains(&(25, 1)), + "actual previous Drop on the same physical worker" ); if mode == 49 { assert_eq!(later_resources & (1 | 8 | 16 | 32), 1 | 8 | 16 | 32); diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index 539558f3..a9fb9063 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -155,7 +155,10 @@ impl Drop for SetupFuture { .emit(38, u32::from(tokio::runtime::Handle::try_current().is_ok())); } if self.mode == 42 || self.mode == 47 || self.mode == 48 || self.mode == 49 { - self.controller.emit(25, 0); + self.controller.emit( + 25, + u32::from(std::thread::current().name() == Some("tinybus-terminal")), + ); } if self.mode == 47 || self.mode == 49 { std::panic::panic_any(PrimaryPayload(*self.controller.controls.lock().unwrap())); From 5e6beda1a5e4e51150c5371b2ce5e460de7a1357 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 20:08:26 +0300 Subject: [PATCH 117/130] chore(examples): observe mode 39 in terminal setup drop The terminal example now records an observation of mode 39 when the setup future is dropped in modes 48 or 49, keeping the example's mode bookkeeping consistent with the emitted state. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus/examples/module_terminal.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs index a9fb9063..50044c2c 100644 --- a/crates/tinybus/examples/module_terminal.rs +++ b/crates/tinybus/examples/module_terminal.rs @@ -151,6 +151,7 @@ impl Future for SetupFuture { impl Drop for SetupFuture { fn drop(&mut self) { if self.mode == 48 || self.mode == 49 { + self.controller.observe(39); self.controller .emit(38, u32::from(tokio::runtime::Handle::try_current().is_ok())); } From 52b3ca45658903f3f19237f99ba0b80f7ecbdb50 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 20:30:21 +0300 Subject: [PATCH 118/130] test(tinybus-module): cover terminal callback context and argument validation Add tests asserting that terminal callbacks treat zero and unknown context tokens as closed, that legacy callbacks still report a null context as a bad argument, and that invalid frame and config sizes remain bad arguments. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinybus-module/src/lib_terminal_tests.rs | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index de48bb87..32475921 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -35,6 +35,80 @@ fn output() -> TbModuleVtable { reinitialize: None, } } +#[test] +fn terminal_callbacks_reject_zero_and_unknown_context_tokens_as_closed() { + let bytes = b"{}"; + for address in [0, 1, 255, usize::MAX] { + let ctx = std::ptr::with_exposed_provenance_mut(address); + let results = unsafe { + [ + ("deliver", terminal_deliver(ctx, bytes.as_ptr(), bytes.len())), + ("shutdown", terminal_shutdown(ctx, 1)), + ( + "reinitialize", + terminal_reinitialize(ctx, bytes.as_ptr(), bytes.len()), + ), + ] + }; + assert_eq!( + results, + [ + ("deliver", TB_CLOSED), + ("shutdown", TB_CLOSED), + ("reinitialize", TB_CLOSED), + ], + "context token {address} with otherwise valid arguments" + ); + } +} + +#[test] +fn legacy_callbacks_keep_null_context_bad_argument() { + let bytes = b"{}"; + let ctx = std::ptr::null_mut(); + let mut out = output(); + let results = unsafe { + [ + ( + "init", + start_module(std::ptr::null(), &mut out, 1, false, |_| async { + Ok(()) + }), + ), + ("deliver", deliver(ctx, bytes.as_ptr(), bytes.len())), + ("shutdown", shutdown(ctx, 1)), + ( + "reinitialize", + reinitialize(ctx, bytes.as_ptr(), bytes.len()), + ), + ] + }; + assert_eq!( + results, + [ + ("init", TB_BAD_ARGUMENT), + ("deliver", TB_BAD_ARGUMENT), + ("shutdown", TB_BAD_ARGUMENT), + ("reinitialize", TB_BAD_ARGUMENT), + ] + ); +} + +#[test] +fn terminal_callbacks_keep_invalid_frame_and_config_bad_argument() { + let ctx = std::ptr::null_mut(); + let bytes = b"{}"; + let results = unsafe { + [ + terminal_deliver(ctx, std::ptr::null(), 0), + terminal_deliver(ctx, bytes.as_ptr(), MAX_FRAME_LEN + 1), + terminal_reinitialize(ctx, std::ptr::null(), 1), + terminal_reinitialize(ctx, bytes.as_ptr(), 1024 * 1024 + 1), + ] + }; + assert_eq!(results, [TB_BAD_ARGUMENT; 4]); +} + #[test] fn native_query_is_required_before_terminal_factory_evaluation() { use tinybus::module::abi::*; From 6241d38839c7e075b09239aecffca92108011c83 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 20:30:58 +0300 Subject: [PATCH 119/130] fix(tinybus-module): drop redundant null checks on terminal ctx The terminal callbacks no longer reject a null ctx up front, since TerminalState::lookup already handles that case and returns TB_CLOSED. This keeps the argument validation consistent across the three entry points and avoids returning TB_BAD_ARGUMENT for a state that is simply gone. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 81d7b9d1..9d6534f8 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1336,7 +1336,7 @@ impl Transport for TerminalTransport { // The existing opaque module_ctx field is a bounded slot/generation token, // never a dereferenced pointer. Retirement denies old callbacks permanently. unsafe extern "C" fn terminal_deliver(ctx: *mut c_void, ptr: *const u8, len: usize) -> i32 { - if ctx.is_null() || ptr.is_null() || len > MAX_FRAME_LEN { + if ptr.is_null() || len > MAX_FRAME_LEN { return TB_BAD_ARGUMENT; } let Some(state) = terminal::TerminalState::lookup(ctx) else { @@ -1361,9 +1361,6 @@ unsafe extern "C" fn terminal_deliver(ctx: *mut c_void, ptr: *const u8, len: usi } } unsafe extern "C" fn terminal_shutdown(ctx: *mut c_void, deadline_ms: u64) -> i32 { - if ctx.is_null() { - return TB_BAD_ARGUMENT; - } let Some(state) = terminal::TerminalState::lookup(ctx) else { return TB_CLOSED; }; @@ -1380,7 +1377,7 @@ unsafe extern "C" fn terminal_shutdown(ctx: *mut c_void, deadline_ms: u64) -> i3 TB_OK } unsafe extern "C" fn terminal_reinitialize(ctx: *mut c_void, ptr: *const u8, len: usize) -> i32 { - if ctx.is_null() || len > 1024 * 1024 || (ptr.is_null() && len != 0) { + if len > 1024 * 1024 || (ptr.is_null() && len != 0) { return TB_BAD_ARGUMENT; } let Some(state) = terminal::TerminalState::lookup(ctx) else { From e17a70b73cb0fd6ba010336fc1f48d88b38e2d9d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 20:31:44 +0300 Subject: [PATCH 120/130] chore: files changed crates/tinybus-module/src/lib_terminal_tests.rs Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 32475921..2ff486b2 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -42,7 +42,10 @@ fn terminal_callbacks_reject_zero_and_unknown_context_tokens_as_closed() { let ctx = std::ptr::with_exposed_provenance_mut(address); let results = unsafe { [ - ("deliver", terminal_deliver(ctx, bytes.as_ptr(), bytes.len())), + ( + "deliver", + terminal_deliver(ctx, bytes.as_ptr(), bytes.len()), + ), ("shutdown", terminal_shutdown(ctx, 1)), ( "reinitialize", @@ -71,9 +74,7 @@ fn legacy_callbacks_keep_null_context_bad_argument() { [ ( "init", - start_module(std::ptr::null(), &mut out, 1, false, |_| async { - Ok(()) - }), + start_module(std::ptr::null(), &mut out, 1, false, |_| async { Ok(()) }), ), ("deliver", deliver(ctx, bytes.as_ptr(), bytes.len())), ("shutdown", shutdown(ctx, 1)), From ff6c00d47aec050d4ef40d8c96df169490cd5b8a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 20:35:40 +0300 Subject: [PATCH 121/130] feat(module): document terminal setup factory contract Clarify the obligations of the terminal setup factory and its runtime hooks, covering partial-failure cleanup, worker launch timing, and reinitialization reuse. Also document that null or unknown context tokens return TB_CLOSED for valid terminal callbacks while legacy shutdown keeps its null-context behavior, and note that the native fixture support feature is private and off by default. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/Cargo.toml | 3 ++- crates/tinybus-module/src/lib.rs | 20 ++++++++++++++++++-- crates/tinybus-module/src/terminal.rs | 24 ++++++++++++++++++++++++ docs/modules/module/README.md | 16 ++++++++++++++-- docs/modules/module/abi.md | 4 +++- 5 files changed, 61 insertions(+), 6 deletions(-) diff --git a/crates/tinybus-module/Cargo.toml b/crates/tinybus-module/Cargo.toml index 0b57b82d..ea359d0d 100644 --- a/crates/tinybus-module/Cargo.toml +++ b/crates/tinybus-module/Cargo.toml @@ -9,7 +9,8 @@ description = "Module-side runtime and export macro for trusted tinybus cdylibs. [features] static-link = [] -# Private opt-in instrumentation for the compiled native lifecycle fixture only. +# Private, off-by-default instrumentation for compiled native lifecycle fixtures. +# Not a production API or a source of native authority. __native-fixture-support = [] [dependencies] diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 9d6534f8..6f1fafb7 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -1002,7 +1002,15 @@ where /// state is reused for that instance's reinitializations, allowing admission to /// remain bound without process-global state. Separate instances receive fresh /// state. A factory panic returns `TB_PANICKED` without unwinding into the host. -/// Existing `setup = function` callbacks retain their behavior. +/// Configured modules may instead use `terminal_setup_factory = factory`. Its +/// factory receives [`NativeInstanceLiveness`] and returns `Result<(F, +/// RuntimeHooks)>`, with the same setup callback bounds. A supported, exactly +/// live native query and an available instance slot are required before factory +/// evaluation; otherwise initialization returns `TB_CLOSED`. The factory owns +/// partial-failure cleanup until it returns and must launch independent workers +/// only from setup after hook registration. See [`RuntimeHooks`] for controller +/// obligations. Hooks are registered once and never replaced by reinitialization. +/// Existing `setup = function` and `setup_factory` callbacks retain their behavior. #[macro_export] macro_rules! module_export { (@linked) => { @@ -1189,7 +1197,9 @@ macro_rules! module_export { /// Generate Rust-addressable entry points for linking several modules into a /// single executable. The entry points have no shared C linker symbol names. -/// Both configured `setup` and per-instance `setup_factory` forms are supported. +/// Configured `setup`, per-instance `setup_factory`, and opted-in +/// `terminal_setup_factory` forms are supported; see [`module_export`]. +/// Terminal factories require the supplied host's native query even when linked. #[macro_export] macro_rules! module_export_static { (terminal_setup_factory = $factory:path, config = $config:ty, $($rest:tt)*) => { @@ -1264,6 +1274,12 @@ mod lifecycle_test; /// Initialize an opted-in instance with native admission and physical hooks. /// +/// The factory returns one shared setup callback and one controller hook pair. +/// It owns partial-failure disposal until return and must defer worker launch +/// until setup, after hook registration. Reinitialization reuses this controller. +/// Admission exhaustion or an unsupported/non-live query returns `TB_CLOSED` +/// before factory evaluation. [`RuntimeHooks`] defines physical finish obligations. +/// /// # Safety /// Uphold [`start_module`]'s ABI and output contracts. The compatible native /// query and copied host context must remain valid for every retained view, diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index a980c048..62b15b23 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -12,18 +12,30 @@ use tinybus::module::abi::{ /// precedence over ordinary stop or host disappearance. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum TerminalReason { + /// Ordinary stop intent; it does not assert worker completion. Stop, + /// The native query is unavailable or no longer grants live admission. HostClosed, + /// A genuine native or module fault, including a caught panic. Fault, + /// The factory or setup returned an error; native fault is also reported. SetupFailed, } /// Actual controller cleanup outcome, independent of native admission. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum TerminalCompletion { + /// The controller has fenced admission and joined its actual workers. + /// This assertion alone proves neither SDK teardown nor backing last-Drop. PhysicallyStopped, + /// Actual controller cleanup could not be proved. The SDK retains the + /// uncertain owner; a deadline cannot convert this into successful cleanup. Uncertain, } /// Controller callbacks supplied by the terminal factory. +/// +/// The factory owns partial-failure cleanup until it returns these hooks. It +/// must launch independent workers only during setup, after hook registration. +/// The same controller remains responsible through every reinitialization. pub struct RuntimeHooks { request: Arc, finish: Box TerminalCompletion + Send>, @@ -34,6 +46,9 @@ impl RuntimeHooks { /// independently fences and joins the controller after SDK setup cancellation /// has settled; it must not depend on cancelled futures or host shutdown. /// Returning `PhysicallyStopped` asserts actual fence/join, not a timeout. + /// The SDK additionally requires its own resource teardown without sticky + /// uncertainty before accepting physical completion. Trusted callbacks must + /// uphold these obligations; the SDK cannot force arbitrary Rust to finish. pub fn new(request: R, finish: F) -> Self where R: Fn(TerminalReason) + Send + Sync + 'static, @@ -47,6 +62,9 @@ impl RuntimeHooks { } /// A retained view of one actual admitted native instance. Its lifetime retains /// the instance's admission permit even after successful physical cleanup. +/// Each SDK image admits at most 256 retained instances; views keep their slot +/// occupied until actual last-Drop. This does not bound controller resource bytes +/// or the host's existing context and mapped-library retention. #[derive(Clone)] pub struct NativeInstanceLiveness { pub(super) inner: Arc, @@ -54,6 +72,10 @@ pub struct NativeInstanceLiveness { impl NativeInstanceLiveness { /// Only a supported, exactly-live native query allows new admission. /// Observed terminal state never reopens. This is not a publication lease. + /// A controller must check its immutable instance/receiver binding and local + /// terminal flags, then this view, then local flags again under its physical + /// phase gate. A cached `true` cannot authorize a later phase; a later stop + /// cannot undo an operation whose admission already linearized. pub fn is_live(&self) -> bool { self.inner.is_live() } @@ -104,6 +126,8 @@ impl NativeInstanceLiveness { self.inner.transport_drop_thread.lock().unwrap().as_deref() == Some("tinybus-terminal") } /// Persist genuine fault and close independent native admission first. + /// Signals the bounded request hook without joining. Call this after + /// unwinding out of the controller's physical phase gate. pub fn request_fault(&self) { self.inner.signal(TerminalReason::Fault); } diff --git a/docs/modules/module/README.md b/docs/modules/module/README.md index 5ba266b4..cad1f0a6 100644 --- a/docs/modules/module/README.md +++ b/docs/modules/module/README.md @@ -51,12 +51,19 @@ factory, config = Config, ...)`; the static export macro accepts the same arguments. The factory receives one `NativeInstanceLiveness` and returns its setup callback plus `RuntimeHooks`. An unsupported or non-live native query rejects initialization before the factory runs. Linked modules require the -same explicit host query and retain the embedding process's panic hook. +same explicit host query and retain the embedding process's panic hook. The +factory owns cleanup of partial construction until it returns its hooks. It +must defer independent worker launch until setup, after hook registration; +reinitialization reuses the same controller and never replaces the hooks. A retained liveness view refers to that exact instance. Only a fresh, exactly Live native query admits work; any observed terminal intent permanently closes the view. `request_fault()` first reports genuine native fault. The view grants no publication lease and does not turn local cancellation into native success. +Under its physical phase gate, the trusted controller must check immutable +instance/receiver binding and local terminal flags, sample the view, then +recheck local flags. A cached Live result cannot authorize a later phase; +subsequent intent cannot undo an operation already admitted. The request hook runs at most once and must only close controller admission and perform bounded cancel/wake. It must not join, do I/O, wait for a gate, or reenter @@ -93,9 +100,14 @@ allocation. Callback lookup acquires the exact State before releasing registry exclusion. Successful teardown retires its registry anchor; only actual State last-Drop releases the slot, so old views and surviving SDK resources count against the limit. Generations never wrap: exhaustion permanently denies that -slot. Stale or unknown tokens return Closed and cannot select a replacement. +slot. Null, stale or unknown context tokens return `TB_CLOSED` for otherwise +valid terminal callbacks and cannot select a replacement. Legacy shutdown's +null-context behavior remains unchanged. Tokens are private bookkeeping, not native authentication. +The SDK's `__native-fixture-support` feature is private and disabled by default. +It exposes only compiled-fixture instrumentation, not production authority. + This metadata bound does not bound existing host-context or mapped-library retention. Libraries remain mapped until process exit, including a rejected initialization whose independent SDK finish job is still running. A rejected diff --git a/docs/modules/module/abi.md b/docs/modules/module/abi.md index ca8d5ad1..9a77980d 100644 --- a/docs/modules/module/abi.md +++ b/docs/modules/module/abi.md @@ -66,7 +66,9 @@ clones the exact matching State under registry exclusion, then releases that lock before instance operations. Successful resource teardown may retire the anchor, but slot reuse waits for actual State last-Drop, including retained liveness views and SDK resources. Generation exhaustion permanently denies the -slot instead of wrapping. Unknown or retired tokens return Closed; an old +slot instead of wrapping. Null, unknown or retired tokens return `TB_CLOSED` +when the callback's other arguments are valid; legacy shutdown retains its +existing null-context behavior. An old instance token cannot act on a replacement. These tokens are not serialized or logged and do not authenticate a native host. From a1960815b2d38454a94c7d92d16206aa9cdcc71a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 12 Oct 2026 20:36:22 +0300 Subject: [PATCH 122/130] docs(module): clarify uncertain terminal cleanup semantics The `Uncertain` terminal completion variant now states that either controller or SDK physical cleanup may be unproved, since even joined workers can leave uncertain SDK resources behind. The ABI documentation was reworded to match, without changing any behaviour. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/terminal.rs | 5 +++-- docs/modules/module/abi.md | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs index 62b15b23..849df1ec 100644 --- a/crates/tinybus-module/src/terminal.rs +++ b/crates/tinybus-module/src/terminal.rs @@ -27,8 +27,9 @@ pub enum TerminalCompletion { /// The controller has fenced admission and joined its actual workers. /// This assertion alone proves neither SDK teardown nor backing last-Drop. PhysicallyStopped, - /// Actual controller cleanup could not be proved. The SDK retains the - /// uncertain owner; a deadline cannot convert this into successful cleanup. + /// Controller or SDK physical cleanup could not be proved. Even joined + /// workers can leave uncertain SDK resources. The SDK retains the owner; + /// a deadline cannot convert this into successful cleanup. Uncertain, } /// Controller callbacks supplied by the terminal factory. diff --git a/docs/modules/module/abi.md b/docs/modules/module/abi.md index 9a77980d..656429d4 100644 --- a/docs/modules/module/abi.md +++ b/docs/modules/module/abi.md @@ -68,8 +68,8 @@ anchor, but slot reuse waits for actual State last-Drop, including retained liveness views and SDK resources. Generation exhaustion permanently denies the slot instead of wrapping. Null, unknown or retired tokens return `TB_CLOSED` when the callback's other arguments are valid; legacy shutdown retains its -existing null-context behavior. An old -instance token cannot act on a replacement. These tokens are not serialized or +existing null-context behavior. An old instance token cannot act on a +replacement. These tokens are not serialized or logged and do not authenticate a native host. Terminal reinitialization uses result-first polling when completion and terminal From eb05eace9d758aa4653b14db76065c4d2631e9ea Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 13 Oct 2026 23:07:14 +0300 Subject: [PATCH 123/130] test: run terminal fixture scenarios in isolated processes The e2e and coverage scripts now build and pin the module_terminal fixture alongside the other loader examples, and each terminal SDK scenario runs in its own process so retained fixture state cannot leak between tests. The scripts also resolve the Cargo target directory via cargo metadata instead of assuming target/debug. Auto-committed-on: dragonfly Co-authored-by: Medulla --- scripts/check-coverage.sh | 42 +++++++++++++++++++++++++++++---------- scripts/test-e2e.ps1 | 36 +++++++++++++++++++++++---------- scripts/test-e2e.sh | 41 ++++++++++++++++++++++++++++---------- 3 files changed, 88 insertions(+), 31 deletions(-) diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index adea2aaf..649368e1 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -1,16 +1,18 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." -cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --no-default-features --features modules,macros +cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --example module_terminal --no-default-features --features modules,macros,__native-fixture-support set -euo pipefail -module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_clock.so' -print -quit) -module_two=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_clock_two.so' -print -quit) -wrong_target=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_wrong_target.so' -print -quit) -context_module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_context.so' -print -quit) -context_eager_module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_context_eager.so' -print -quit) -host_callback_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) -controlled_init_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) -for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module"; do +target_dir=$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])') +module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f -name 'libmodule_clock.so' -print -quit) +module_two=$(find "$target_dir/debug/examples" -maxdepth 1 -type f -name 'libmodule_clock_two.so' -print -quit) +wrong_target=$(find "$target_dir/debug/examples" -maxdepth 1 -type f -name 'libmodule_wrong_target.so' -print -quit) +context_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f -name 'libmodule_context.so' -print -quit) +context_eager_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f -name 'libmodule_context_eager.so' -print -quit) +host_callback_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) +controlled_init_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) +terminal_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_terminal.so' -o -name 'libmodule_terminal.dylib' \) -print -quit) +for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module" "$terminal_module"; do if [ -z "$artifact" ] || [ ! -f "$artifact" ]; then echo "a module loader fixture is missing" >&2 exit 1 @@ -27,6 +29,20 @@ else digest="$(shasum -a 256 "$private/$library" | awk '{print $1}')" fi printf '"%s" = "%s"\n' "$library" "$digest" > "$private/modules.toml" +# Terminal fixtures can retain uncertain owners; pin a private copy and run +# each SDK scenario in a fresh process so its retained state cannot leak. +terminal_private="$PWD/target/private-module-tests/module-terminal" +mkdir -p "$terminal_private" +chmod 700 "$terminal_private" +cp "$terminal_module" "$terminal_private/" +terminal_library="$(basename "$terminal_module")" +if command -v sha256sum >/dev/null 2>&1; then + terminal_digest="$(sha256sum "$terminal_private/$terminal_library" | awk '{print $1}')" +else + terminal_digest="$(shasum -a 256 "$terminal_private/$terminal_library" | awk '{print $1}')" +fi +printf '"%s" = "%s"\n' "$terminal_library" "$terminal_digest" > "$terminal_private/modules.toml" +export TINYBUS_TEST_TERMINAL_MODULE="$terminal_private/$terminal_library" # Port traits declare contracts only; their transport implementations # exercise those contracts, but traits have no executable body to cover. # CLI/host/connection integration surfaces and the network release path are @@ -34,4 +50,10 @@ printf '"%s" = "%s"\n' "$library" "$digest" > "$private/modules.toml" # process/network branches are not meaningful per-file coverage. The # test-support module is integration-test glue whose platform-specific # dynamic-loader and timeout paths run in the dedicated OS jobs instead. -TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$PWD/$controlled_init_module" cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 -- --include-ignored +export TINYBUS_TEST_MODULE="$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$controlled_init_module" +cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --no-report -- --include-ignored --skip terminal_runtime_tests::compiled_native_ + +while IFS= read -r test; do + cargo llvm-cov --no-report -p tinybus-module --all-features "$test" -- --ignored --exact +done < <(cargo test --locked -p tinybus-module --all-features -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') +cargo llvm-cov report --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 diff --git a/scripts/test-e2e.ps1 b/scripts/test-e2e.ps1 index bbd3f966..116e0097 100644 --- a/scripts/test-e2e.ps1 +++ b/scripts/test-e2e.ps1 @@ -1,4 +1,7 @@ $ErrorActionPreference = 'Stop' +$targetDirectory = (cargo metadata --format-version 1 --no-deps | ConvertFrom-Json).target_directory +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +$examples = Join-Path $targetDirectory debug/examples $private = Join-Path $PWD target/private-module-tests New-Item -ItemType Directory -Force $private | Out-Null $identity = [System.Security.Principal.WindowsIdentity]::GetCurrent() @@ -15,15 +18,16 @@ $rule = [System.Security.AccessControl.FileSystemAccessRule]::new( ) $acl.AddAccessRule($rule) Set-Acl -LiteralPath $private -AclObject $acl -Copy-Item target/debug/examples/module_clock.dll $private -Copy-Item target/debug/examples/module_context.dll $private -Copy-Item target/debug/examples/module_context_eager.dll $private -Copy-Item target/debug/examples/module_host_callback.dll $private -Copy-Item target/debug/examples/module_controlled_init.dll $private -Copy-Item target/debug/examples/module_wrong_target.dll $private +Copy-Item (Join-Path $examples module_clock.dll) $private +Copy-Item (Join-Path $examples module_context.dll) $private +Copy-Item (Join-Path $examples module_context_eager.dll) $private +Copy-Item (Join-Path $examples module_host_callback.dll) $private +Copy-Item (Join-Path $examples module_controlled_init.dll) $private +Copy-Item (Join-Path $examples module_terminal.dll) $private +Copy-Item (Join-Path $examples module_wrong_target.dll) $private # Admit every intentional fixture by its actual built bytes so later # checks, including the wrong-target gate, can exercise their contract. -$fixtureAllowlist = @('module_clock.dll', 'module_context.dll', 'module_context_eager.dll', 'module_host_callback.dll', 'module_controlled_init.dll', 'module_wrong_target.dll') | ForEach-Object { +$fixtureAllowlist = @('module_clock.dll', 'module_context.dll', 'module_context_eager.dll', 'module_host_callback.dll', 'module_controlled_init.dll', 'module_terminal.dll', 'module_wrong_target.dll') | ForEach-Object { $digest = (Get-FileHash -LiteralPath (Join-Path $private $_) -Algorithm SHA256).Hash.ToLowerInvariant() '"' + $_ + '" = "' + $digest + '"' } @@ -31,14 +35,14 @@ $fixtureAllowlist | Set-Content -LiteralPath (Join-Path $private modules.toml) $clockTwoDirectory = Join-Path $private module-clock-two New-Item -ItemType Directory -Force $clockTwoDirectory | Out-Null Set-Acl -LiteralPath $clockTwoDirectory -AclObject $acl -Copy-Item target/debug/examples/module_clock_two.dll $clockTwoDirectory +Copy-Item (Join-Path $examples module_clock_two.dll) $clockTwoDirectory $clockTwoHash = (Get-FileHash -LiteralPath (Join-Path $clockTwoDirectory module_clock_two.dll) -Algorithm SHA256).Hash.ToLowerInvariant() '"module_clock_two.dll" = "' + $clockTwoHash + '"' | Set-Content -LiteralPath (Join-Path $clockTwoDirectory modules.toml) -NoNewline $mixed = Join-Path $private mixed New-Item -ItemType Directory -Force $mixed | Out-Null Set-Acl -LiteralPath $mixed -AclObject $acl -Copy-Item target/debug/examples/module_clock.dll $mixed -Copy-Item target/debug/examples/module_wrong_target.dll $mixed +Copy-Item (Join-Path $examples module_clock.dll) $mixed +Copy-Item (Join-Path $examples module_wrong_target.dll) $mixed $module = (Get-ChildItem "$private/module_clock.dll" | Select-Object -First 1).FullName $moduleTwo = (Get-ChildItem "$clockTwoDirectory/module_clock_two.dll" | Select-Object -First 1).FullName $wrongTarget = (Get-ChildItem "$private/module_wrong_target.dll" | Select-Object -First 1).FullName @@ -47,13 +51,23 @@ $env:TMP = $private $env:TINYBUS_TEST_CONTEXT_MODULE = (Get-Item "$private/module_context.dll").FullName $env:TINYBUS_TEST_CONTEXT_EAGER_MODULE = (Get-Item "$private/module_context_eager.dll").FullName $env:TINYBUS_TEST_CONTROLLED_INIT_MODULE = (Get-Item "$private/module_controlled_init.dll").FullName +$env:TINYBUS_TEST_TERMINAL_MODULE = (Get-Item "$private/module_terminal.dll").FullName $env:TINYBUS_TEST_HOST_CALLBACK_MODULE = (Get-Item "$private/module_host_callback.dll").FullName $env:TINYBUS_TEST_MODULE = $module $env:TINYBUS_TEST_MODULE_TWO = $moduleTwo $env:TINYBUS_TEST_WRONG_TARGET = $wrongTarget $env:TINYBUS_TEST_MODULE_DIRECTORY = $mixed -cargo test --locked --no-default-features --features modules,macros,test-support -- --ignored +cargo test --locked --no-default-features --workspace --features modules,macros,test-support,__native-fixture-support -- --ignored --skip terminal_runtime_tests::compiled_native_ +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +# Retained fixture owners require a fresh process for each SDK scenario. +$testList = cargo test --locked -p tinybus-module --features __native-fixture-support -- --list --ignored if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +foreach ($line in $testList) { + if ($line -match '^(terminal_runtime_tests::compiled_native_.*): test$') { + cargo test --locked -p tinybus-module --features __native-fixture-support $Matches[1] -- --ignored --exact + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } +} # Directory admission against the real ACLs: a bundle Everyone can # write falls back to the verified release cache. cargo test --locked --no-default-features --features modules,macros admission_tests -- --include-ignored diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh index 72579373..2dd9fb05 100755 --- a/scripts/test-e2e.sh +++ b/scripts/test-e2e.sh @@ -1,20 +1,22 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." -cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --no-default-features --features modules,macros +cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --example module_terminal --no-default-features --features modules,macros,__native-fixture-support cargo test --locked --no-default-features --features modules,macros,test-support test_support::tests case "${RUNNER_OS:-$(uname -s)}" in Windows|MINGW*|MSYS*|CYGWIN*) pwsh -NoProfile -File scripts/test-e2e.ps1 ;; *) set -euo pipefail -module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_clock.so' -o -name 'libmodule_clock.dylib' \) -print -quit) -module_two=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_clock_two.so' -o -name 'libmodule_clock_two.dylib' \) -print -quit) -wrong_target=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_wrong_target.so' -o -name 'libmodule_wrong_target.dylib' \) -print -quit) -context_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_context.so' -o -name 'libmodule_context.dylib' \) -print -quit) -context_eager_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_context_eager.so' -o -name 'libmodule_context_eager.dylib' \) -print -quit) -host_callback_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) -controlled_init_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) -for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module"; do +target_dir=$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])') +module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_clock.so' -o -name 'libmodule_clock.dylib' \) -print -quit) +module_two=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_clock_two.so' -o -name 'libmodule_clock_two.dylib' \) -print -quit) +wrong_target=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_wrong_target.so' -o -name 'libmodule_wrong_target.dylib' \) -print -quit) +context_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_context.so' -o -name 'libmodule_context.dylib' \) -print -quit) +context_eager_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_context_eager.so' -o -name 'libmodule_context_eager.dylib' \) -print -quit) +host_callback_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) +controlled_init_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) +terminal_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_terminal.so' -o -name 'libmodule_terminal.dylib' \) -print -quit) +for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module" "$terminal_module"; do if [ -z "$artifact" ] || [ ! -f "$artifact" ]; then echo "a module example artifact is missing; the loader tests would not exercise it" >&2 exit 1 @@ -31,7 +33,26 @@ else digest="$(shasum -a 256 "$private/$library" | awk '{print $1}')" fi printf '"%s" = "%s"\n' "$library" "$digest" > "$private/modules.toml" -TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$PWD/$controlled_init_module" cargo test --locked --no-default-features --features modules,macros,test-support -- --ignored +# Terminal fixtures can retain uncertain owners; pin a private copy and run +# each SDK scenario in a fresh process so its retained state cannot leak. +terminal_private="$PWD/target/private-module-tests/module-terminal" +mkdir -p "$terminal_private" +chmod 700 "$terminal_private" +cp "$terminal_module" "$terminal_private/" +terminal_library="$(basename "$terminal_module")" +if command -v sha256sum >/dev/null 2>&1; then + terminal_digest="$(sha256sum "$terminal_private/$terminal_library" | awk '{print $1}')" +else + terminal_digest="$(shasum -a 256 "$terminal_private/$terminal_library" | awk '{print $1}')" +fi +printf '"%s" = "%s"\n' "$terminal_library" "$terminal_digest" > "$terminal_private/modules.toml" +export TINYBUS_TEST_TERMINAL_MODULE="$terminal_private/$terminal_library" +export TINYBUS_TEST_MODULE="$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$controlled_init_module" +cargo test --locked --no-default-features --workspace --features modules,macros,test-support,__native-fixture-support -- --ignored --skip terminal_runtime_tests::compiled_native_ + +while IFS= read -r test; do + cargo test --locked -p tinybus-module --features __native-fixture-support "$test" -- --ignored --exact +done < <(cargo test --locked -p tinybus-module --features __native-fixture-support -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') cargo test --locked --no-default-features --features modules,macros admission_tests -- --include-ignored TINYBUS_TEST_UMASK=1 cargo test --locked --no-default-features --features modules,macros permissive_umask -- --test-threads=1 From 9dfaf03c0f842b0ea674d7608ab6b3ca098a23bd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 13 Oct 2026 23:08:04 +0300 Subject: [PATCH 124/130] fix(scripts): collect compiled terminal tests before running them The e2e and coverage scripts now list the compiled terminal scenarios up front and fail early when none are found, instead of feeding the test runner from a process substitution that could silently yield nothing. The coverage script also passes --no-clean so per-test runs accumulate into a single report. Auto-committed-on: dragonfly Co-authored-by: Medulla --- scripts/check-coverage.sh | 6 ++++-- scripts/test-e2e.sh | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index 649368e1..3c2f4c66 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -53,7 +53,9 @@ export TINYBUS_TEST_TERMINAL_MODULE="$terminal_private/$terminal_library" export TINYBUS_TEST_MODULE="$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$controlled_init_module" cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --no-report -- --include-ignored --skip terminal_runtime_tests::compiled_native_ +terminal_tests=$(cargo test --locked -p tinybus-module --all-features -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') +[ -n "$terminal_tests" ] || { echo "no compiled terminal scenarios found" >&2; exit 1; } while IFS= read -r test; do - cargo llvm-cov --no-report -p tinybus-module --all-features "$test" -- --ignored --exact -done < <(cargo test --locked -p tinybus-module --all-features -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') + cargo llvm-cov --no-clean --no-report -p tinybus-module --all-features "$test" -- --ignored --exact +done <<< "$terminal_tests" cargo llvm-cov report --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh index 2dd9fb05..8db97d22 100755 --- a/scripts/test-e2e.sh +++ b/scripts/test-e2e.sh @@ -50,9 +50,11 @@ export TINYBUS_TEST_TERMINAL_MODULE="$terminal_private/$terminal_library" export TINYBUS_TEST_MODULE="$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$controlled_init_module" cargo test --locked --no-default-features --workspace --features modules,macros,test-support,__native-fixture-support -- --ignored --skip terminal_runtime_tests::compiled_native_ +terminal_tests=$(cargo test --locked -p tinybus-module --features __native-fixture-support -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') +[ -n "$terminal_tests" ] || { echo "no compiled terminal scenarios found" >&2; exit 1; } while IFS= read -r test; do cargo test --locked -p tinybus-module --features __native-fixture-support "$test" -- --ignored --exact -done < <(cargo test --locked -p tinybus-module --features __native-fixture-support -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') +done <<< "$terminal_tests" cargo test --locked --no-default-features --features modules,macros admission_tests -- --include-ignored TINYBUS_TEST_UMASK=1 cargo test --locked --no-default-features --features modules,macros permissive_umask -- --test-threads=1 From 3ce209ce09233a9402b7417d17d96b08ba1ee463 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 13 Oct 2026 23:09:26 +0300 Subject: [PATCH 125/130] fix(scripts): enable tinybus/macros for compiled terminal scenarios The per-scenario test and coverage invocations for the compiled native terminal fixtures now pass the tinybus/macros feature alongside the existing flags, so those scenarios build with the macro support they require instead of failing to compile. Auto-committed-on: dragonfly Co-authored-by: Medulla --- scripts/check-coverage.sh | 4 ++-- scripts/test-e2e.ps1 | 4 ++-- scripts/test-e2e.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index 3c2f4c66..8a26224e 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -53,9 +53,9 @@ export TINYBUS_TEST_TERMINAL_MODULE="$terminal_private/$terminal_library" export TINYBUS_TEST_MODULE="$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$controlled_init_module" cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --no-report -- --include-ignored --skip terminal_runtime_tests::compiled_native_ -terminal_tests=$(cargo test --locked -p tinybus-module --all-features -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') +terminal_tests=$(cargo test --locked -p tinybus-module --all-features --features tinybus/macros -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') [ -n "$terminal_tests" ] || { echo "no compiled terminal scenarios found" >&2; exit 1; } while IFS= read -r test; do - cargo llvm-cov --no-clean --no-report -p tinybus-module --all-features "$test" -- --ignored --exact + cargo llvm-cov --no-clean --no-report -p tinybus-module --all-features --features tinybus/macros "$test" -- --ignored --exact done <<< "$terminal_tests" cargo llvm-cov report --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 diff --git a/scripts/test-e2e.ps1 b/scripts/test-e2e.ps1 index 116e0097..8dfb9467 100644 --- a/scripts/test-e2e.ps1 +++ b/scripts/test-e2e.ps1 @@ -60,11 +60,11 @@ $env:TINYBUS_TEST_MODULE_DIRECTORY = $mixed cargo test --locked --no-default-features --workspace --features modules,macros,test-support,__native-fixture-support -- --ignored --skip terminal_runtime_tests::compiled_native_ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Retained fixture owners require a fresh process for each SDK scenario. -$testList = cargo test --locked -p tinybus-module --features __native-fixture-support -- --list --ignored +$testList = cargo test --locked -p tinybus-module --features __native-fixture-support,tinybus/macros -- --list --ignored if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } foreach ($line in $testList) { if ($line -match '^(terminal_runtime_tests::compiled_native_.*): test$') { - cargo test --locked -p tinybus-module --features __native-fixture-support $Matches[1] -- --ignored --exact + cargo test --locked -p tinybus-module --features __native-fixture-support,tinybus/macros $Matches[1] -- --ignored --exact if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } } } diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh index 8db97d22..8973c7c5 100755 --- a/scripts/test-e2e.sh +++ b/scripts/test-e2e.sh @@ -50,10 +50,10 @@ export TINYBUS_TEST_TERMINAL_MODULE="$terminal_private/$terminal_library" export TINYBUS_TEST_MODULE="$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$controlled_init_module" cargo test --locked --no-default-features --workspace --features modules,macros,test-support,__native-fixture-support -- --ignored --skip terminal_runtime_tests::compiled_native_ -terminal_tests=$(cargo test --locked -p tinybus-module --features __native-fixture-support -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') +terminal_tests=$(cargo test --locked -p tinybus-module --features __native-fixture-support,tinybus/macros -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') [ -n "$terminal_tests" ] || { echo "no compiled terminal scenarios found" >&2; exit 1; } while IFS= read -r test; do - cargo test --locked -p tinybus-module --features __native-fixture-support "$test" -- --ignored --exact + cargo test --locked -p tinybus-module --features __native-fixture-support,tinybus/macros "$test" -- --ignored --exact done <<< "$terminal_tests" cargo test --locked --no-default-features --features modules,macros admission_tests -- --include-ignored From 1df4c0acfc02ddcf0d0130a21cc7c90404464a15 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 13 Oct 2026 23:09:52 +0300 Subject: [PATCH 126/130] fix(scripts): use cargo llvm-cov test for per-test coverage runs The per-test coverage loop invoked cargo llvm-cov without the test subcommand, which is not a valid invocation. Adding the subcommand makes the ignored terminal scenario tests actually run under coverage. Auto-committed-on: dragonfly Co-authored-by: Medulla --- scripts/check-coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index 8a26224e..03b57b93 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -56,6 +56,6 @@ cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybu terminal_tests=$(cargo test --locked -p tinybus-module --all-features --features tinybus/macros -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') [ -n "$terminal_tests" ] || { echo "no compiled terminal scenarios found" >&2; exit 1; } while IFS= read -r test; do - cargo llvm-cov --no-clean --no-report -p tinybus-module --all-features --features tinybus/macros "$test" -- --ignored --exact + cargo llvm-cov test --no-clean --no-report -p tinybus-module --all-features --features tinybus/macros "$test" -- --ignored --exact done <<< "$terminal_tests" cargo llvm-cov report --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 From 83ea9d7a91de6c6412e25afd3dced2157aa4aee7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 13 Oct 2026 23:11:03 +0300 Subject: [PATCH 127/130] fix(scripts): drop --no-clean from per-test coverage runs The per-test llvm-cov invocations in the terminal scenario loop passed --no-clean, which wiped the accumulated coverage data between tests and left the final report incomplete. Removing the flag lets each run append to the existing profile so the aggregate report reflects every scenario. Auto-committed-on: dragonfly Co-authored-by: Medulla --- scripts/check-coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index 03b57b93..04f76779 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -56,6 +56,6 @@ cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybu terminal_tests=$(cargo test --locked -p tinybus-module --all-features --features tinybus/macros -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') [ -n "$terminal_tests" ] || { echo "no compiled terminal scenarios found" >&2; exit 1; } while IFS= read -r test; do - cargo llvm-cov test --no-clean --no-report -p tinybus-module --all-features --features tinybus/macros "$test" -- --ignored --exact + cargo llvm-cov test --no-report -p tinybus-module --all-features --features tinybus/macros "$test" -- --ignored --exact done <<< "$terminal_tests" cargo llvm-cov report --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 From ee42b0c4e9eada2863b6540d21761dd4a1dbc195 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 13 Oct 2026 23:11:44 +0300 Subject: [PATCH 128/130] chore(scripts): move private coverage fixtures under target/private-module-coverage The private module fixtures used by the coverage check were staged under target/private-module-tests, which collides with the directory used by the test suite. They now live under target/private-module-coverage so the two runs no longer share state. Auto-committed-on: dragonfly Co-authored-by: Medulla --- scripts/check-coverage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index 04f76779..9f1cd67a 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -18,7 +18,7 @@ for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$cont exit 1 fi done -private="$PWD/target/private-module-tests/module-clock-two" +private="$PWD/target/private-module-coverage/module-clock-two" mkdir -p "$private" chmod 700 "$private" cp "$module_two" "$private/" @@ -31,7 +31,7 @@ fi printf '"%s" = "%s"\n' "$library" "$digest" > "$private/modules.toml" # Terminal fixtures can retain uncertain owners; pin a private copy and run # each SDK scenario in a fresh process so its retained state cannot leak. -terminal_private="$PWD/target/private-module-tests/module-terminal" +terminal_private="$PWD/target/private-module-coverage/module-terminal" mkdir -p "$terminal_private" chmod 700 "$terminal_private" cp "$terminal_module" "$terminal_private/" From 6321d150d82ce6abb5b1dfa46edc601fd869c1cd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 13 Oct 2026 23:15:37 +0300 Subject: [PATCH 129/130] test(module): cover fixture calls, terminal ownership, and runtime identity Add tests for JSON and object fixture calls preserving deadlines and confidential refusals, and for unavailable loads and explicit reports using only the registered identity. Also assert the retained terminal view reports the exact execution owner and surviving SDK resources, and that a started runtime exposes a unique connection name. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/terminal_tests.rs | 45 +++++++++++ crates/tinybus/src/module/client_tests.rs | 90 +++++++++++++++++++++ crates/tinybus/src/module/runtime_tests.rs | 1 + 3 files changed, 136 insertions(+) diff --git a/crates/tinybus-module/src/terminal_tests.rs b/crates/tinybus-module/src/terminal_tests.rs index 87c0448a..cd829cbf 100644 --- a/crates/tinybus-module/src/terminal_tests.rs +++ b/crates/tinybus-module/src/terminal_tests.rs @@ -303,3 +303,48 @@ fn first_cleanup_drop_panic_retains_all_unprocessed_resource_owners() { ); assert!(state.resources.lock().unwrap().retained_factory.is_some()); } + +#[cfg(feature = "__native-fixture-support")] +#[test] +fn retained_view_reports_exact_execution_owner_and_surviving_sdk_resources() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + let admission = Admission::new(); + let owner = ExecutionOwner::new(host()); + let state = TerminalState::admit_in(&admission, host(), Some(owner.clone())).unwrap(); + let view = state.view(); + assert!(!view.__fixture_owner_is_current()); + assert!(!view.__fixture_owner_is_stopped()); + { + let _scope = owner.enter(); + assert!(view.__fixture_owner_is_current()); + let other = ExecutionOwner::new(host()); + { + let _other_scope = other.enter(); + assert!(!view.__fixture_owner_is_current()); + } + assert!(view.__fixture_owner_is_current()); + } + assert!(!view.__fixture_owner_is_current()); + assert_eq!(view.__fixture_resources(), 0); + let sdk_owner = state.sdk_owner(); + assert_eq!(view.__fixture_sdk_owners(), 1); + assert!(!view.__fixture_request_done()); + drop(sdk_owner); + assert_eq!(view.__fixture_sdk_owners(), 0); + state.register(RuntimeHooks::new( + |_| {}, + || TerminalCompletion::PhysicallyStopped, + )); + state.publish(); + view.request_fault(); + assert_eq!( + state.wait(Duration::from_secs(3)), + TerminalCompletion::PhysicallyStopped + ); + assert!(view.__fixture_request_done()); + assert!(view.__fixture_owner_is_stopped()); + assert!(!view.is_live()); +} diff --git a/crates/tinybus/src/module/client_tests.rs b/crates/tinybus/src/module/client_tests.rs index c25b8946..dce7fc5b 100644 --- a/crates/tinybus/src/module/client_tests.rs +++ b/crates/tinybus/src/module/client_tests.rs @@ -171,3 +171,93 @@ async fn a_fixture_proxy_is_classified_and_reported_when_the_call_fails() { assert!(result.is_err()); assert_eq!(sink.0.lock().unwrap().len(), 1); } + +struct JsonFixture; +#[async_trait::async_trait] +impl crate::Interface for JsonFixture { + fn name(&self) -> crate::InterfaceName { + crate::InterfaceName::new(RECORD.bus_name).unwrap() + } + fn members(&self) -> Vec { + vec![crate::MemberName::new("Json").unwrap()] + } + async fn call( + &self, + _: &crate::MemberName, + _: serde_json::Value, + ) -> crate::Result { + Ok(serde_json::json!("{\"answer\":42}")) + } +} + +#[tokio::test] +async fn fixture_json_and_object_calls_preserve_deadlines_and_confidential_refusals() { + let (sink, resolver) = resolver(true); + let bus = crate::transport::memory::MemoryBus::new(); + crate::broker::Broker::new().spawn(bus.clone()); + let service = crate::Connection::connect(bus.connect().await.unwrap()) + .await + .unwrap(); + service.request_name(RECORD.bus_name).await.unwrap(); + for path in [RECORD.object_path, "/child"] { + service + .serve_at(crate::ObjectPath::new(path).unwrap(), JsonFixture) + .await + .unwrap(); + } + let caller = crate::Connection::connect(bus.connect().await.unwrap()) + .await + .unwrap(); + let proxy = caller + .proxy(RECORD.bus_name, RECORD.object_path, RECORD.bus_name) + .unwrap(); + let client = ModuleClient::new(resolver) + .with_fixture(proxy) + .with_timeout(Duration::from_secs(1)); + let reply: serde_json::Value = client.call_json(RECORD.id, "Json", ()).await.unwrap(); + assert_eq!(reply, serde_json::json!({"answer":42})); + let object = client.object_client(RECORD.id, "/child").await.unwrap(); + let reply: String = object.call("Json", ()).await.unwrap(); + assert_eq!(reply, "{\"answer\":42}"); + assert!( + client + .object_client(RECORD.id, "invalid path") + .await + .is_err() + ); + let reply: Result = object.call_confidential("Json", ()).await; + assert_eq!(reply, Err(ModuleCallError::TransportFailed)); + let reply: Result = client.call_confidential(RECORD.id, "Json", ()).await; + assert_eq!(reply, Err(ModuleCallError::TransportFailed)); + let reply: Result = + client.call_confidential_json(RECORD.id, "Json", ()).await; + assert_eq!(reply, Err(ModuleCallError::TransportFailed)); + assert!( + sink.0 + .lock() + .unwrap() + .iter() + .all(|(_, reason)| *reason == Reason::TransportFailed) + ); +} + +#[tokio::test] +async fn unavailable_loads_and_explicit_reports_use_only_registered_identity() { + let (sink, resolver) = resolver(true); + let client = ModuleClient::new(resolver); + let reply: Result = client.call(RECORD.id, "Json", ()).await; + assert_eq!(reply, Err(ModuleCallError::Unavailable)); + assert!(client.object_client(RECORD.id, "/child").await.is_err()); + client.report_incompatible_contract(RECORD.id); + client.report_unavailable(RECORD.id); + client.report_malformed_reply("unknown"); + client.report_incompatible_contract("unknown"); + client.report_unavailable("unknown"); + assert!( + sink.0 + .lock() + .unwrap() + .iter() + .all(|(module, _)| module == RECORD.id) + ); +} diff --git a/crates/tinybus/src/module/runtime_tests.rs b/crates/tinybus/src/module/runtime_tests.rs index 978a789c..63274014 100644 --- a/crates/tinybus/src/module/runtime_tests.rs +++ b/crates/tinybus/src/module/runtime_tests.rs @@ -15,6 +15,7 @@ async fn setup_runs_before_the_runtime_is_handed_out() { let runtime = ModuleRuntime::start(&RuntimeNames::default(), Some(setup)).unwrap(); assert!(ran.load(Ordering::SeqCst)); assert!(runtime.host().list().is_empty()); + assert!(runtime.connection().unique_name().is_some()); } #[tokio::test] From b65f241956d155d5ff419b20aa934778afbf1d51 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 13 Oct 2026 23:17:55 +0300 Subject: [PATCH 130/130] test(tinybus-module): wait for worker finish before asserting shutdown The test previously assumed host shutdown returned only after the physical worker finished, but the shutdown deadline can elapse first. It now drains the channel until the expected finish messages arrive, so the assertions run against the worker's actual completion. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinybus-module/src/lib_terminal_tests.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs index 2ff486b2..796e3368 100644 --- a/crates/tinybus-module/src/lib_terminal_tests.rs +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -2733,7 +2733,13 @@ async fn compiled_native_method_payload_and_whole_connection_drop_retain_actual_ seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); } } - stopping.await.unwrap(); // actual SDK Runtime shutdown_timeout elapsed + stopping.await.unwrap(); + // Host shutdown has a deadline and can return before the physical + // worker finishes. Observe the original finish/join directly while + // the SDK-owned destructor is still held by the fixture barrier. + while !seen.contains(&(5, 1)) || !seen.contains(&(7, 1)) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } let held_owners = owners(); let timed_out_native = flags(); seen.extend(receiver.try_iter());