diff --git a/crates/tinybus-module/src/execution_context_tests.rs b/crates/tinybus-module/src/execution_context_tests.rs index 3d7e0dc9..7b0d052a 100644 --- a/crates/tinybus-module/src/execution_context_tests.rs +++ b/crates/tinybus-module/src/execution_context_tests.rs @@ -27,6 +27,7 @@ fn owner() -> (ExecutionOwner, &'static AtomicUsize) { }, ready, broker_routing: None, + native_terminal_state: None, }); (ExecutionOwner::new(host), count) } diff --git a/crates/tinybus-module/src/lib_lifecycle_tests.rs b/crates/tinybus-module/src/lib_lifecycle_tests.rs index 00390d73..d8d76078 100644 --- a/crates/tinybus-module/src/lib_lifecycle_tests.rs +++ b/crates/tinybus-module/src/lib_lifecycle_tests.rs @@ -23,6 +23,7 @@ fn host() -> TbHostVtable { }, ready, broker_routing: None, + native_terminal_state: None, } } struct DropCounter(Arc); diff --git a/crates/tinybus-module/src/lib_tests.rs b/crates/tinybus-module/src/lib_tests.rs index 3d3ec6aa..07bf9f4f 100644 --- a/crates/tinybus-module/src/lib_tests.rs +++ b/crates/tinybus-module/src/lib_tests.rs @@ -74,6 +74,7 @@ fn host(config: &[u8]) -> TbHostVtable { }, ready: host_ready, broker_routing: None, + native_terminal_state: None, } } @@ -745,6 +746,7 @@ fn an_older_host_prefix_is_copied_without_reading_the_additive_tail() { let ptr = std::ptr::from_ref(&prefix).cast::(); let normalized = unsafe { TbHostVtable::read_compatible(ptr) }.unwrap(); assert!(normalized.broker_routing.is_none()); + assert!(normalized.native_terminal_state.is_none()); assert_eq!( unsafe { parse_config::(ptr) }.unwrap(), serde_json::json!({}) @@ -758,9 +760,12 @@ fn an_older_host_prefix_is_copied_without_reading_the_additive_tail() { #[test] fn an_incomplete_host_tail_never_reads_the_callback() { let mut current = host(b"{}"); - current.size = size_of::() as u32 - 1; + current.size = (std::mem::offset_of!(TbHostVtable, broker_routing) + + size_of:: u32>>() + - 1) as u32; let normalized = unsafe { TbHostVtable::read_compatible(¤t) }.unwrap(); assert!(normalized.broker_routing.is_none()); + assert!(normalized.native_terminal_state.is_none()); assert!(unsafe { TbHostVtable::read_compatible(std::ptr::null()) }.is_none()); current.size = tinybus::module::abi::TB_HOST_VTABLE_BASE_SIZE - 1; assert!(unsafe { TbHostVtable::read_compatible(¤t) }.is_none()); diff --git a/crates/tinybus/Cargo.toml b/crates/tinybus/Cargo.toml index b567ed1e..b98b3681 100644 --- a/crates/tinybus/Cargo.toml +++ b/crates/tinybus/Cargo.toml @@ -82,7 +82,7 @@ test-support = ["modules", "macros"] [dev-dependencies] -tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time", "io-util"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time", "io-util", "test-util"] } tempfile = "3" [[bin]] @@ -135,3 +135,8 @@ required-features = ["modules"] name = "module_host_callback" crate-type = ["cdylib"] required-features = ["modules"] + +[[example]] +name = "module_controlled_init" +crate-type = ["cdylib"] +required-features = ["modules"] diff --git a/crates/tinybus/examples/module_controlled_init.rs b/crates/tinybus/examples/module_controlled_init.rs new file mode 100644 index 00000000..014ae391 --- /dev/null +++ b/crates/tinybus/examples/module_controlled_init.rs @@ -0,0 +1,139 @@ +//! Native ABI fixture with bounded, externally released test barriers. +use std::ffi::c_void; +use std::sync::{Arc, Condvar, Mutex}; +use tinybus::module::abi::*; + +#[repr(C)] +#[derive(Clone, Copy)] +struct Controls { + ctx: *mut c_void, + entered: unsafe extern "C" fn(*mut c_void, *const TbHostVtable), + joined: unsafe extern "C" fn(*mut c_void), +} +unsafe impl Send for Controls {} + +struct Instance { + controls: Controls, + host: TbHostVtable, + stop: Arc<(Mutex, Condvar)>, + worker: Option>, +} + +#[unsafe(no_mangle)] +pub static TINYBUS_MODULE_ABI_V1: TbAbiDescriptor = + TbAbiDescriptor::current("controlled-init", env!("CARGO_PKG_VERSION")); + +#[unsafe(no_mangle)] +pub extern "C" fn tinybus_module_manifest_v1() -> TbSlice { + tinybus_module::manifest_slice(tinybus_module::ManifestDeclaration { + name: "controlled-init", + version: env!("CARGO_PKG_VERSION"), + provides: &["ai.tinyhumans.ControlledInit"], + methods: &[], + signals: &[], + requires: &[], + optional: &[], + lazy: true, + worker_threads: 1, + }) +} + +unsafe extern "C" fn deliver(_: *mut c_void, _: *const u8, _: usize) -> i32 { + TB_OK +} +unsafe extern "C" fn reinitialize(ctx: *mut c_void, _: *const u8, _: usize) -> i32 { + let instance = unsafe { &*ctx.cast::() }; + unsafe { + (instance.controls.entered)(instance.controls.ctx, &instance.host); + } + TB_OK +} +unsafe extern "C" fn shutdown(ctx: *mut c_void, _: u64) -> i32 { + let mut instance = unsafe { Box::from_raw(ctx.cast::()) }; + *instance.stop.0.lock().unwrap() = true; + instance.stop.1.notify_all(); + instance.worker.take().unwrap().join().unwrap(); + unsafe { + (instance.controls.joined)(instance.controls.ctx); + } + TB_OK +} + +/// # Safety +/// Host pointers are valid for the ABI call; config contains test Controls. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tinybus_module_init_v1( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + let host = unsafe { &*host }; + let copied_config = + unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }.to_vec(); + // The public loader serializes JSON. This declared test fixture accepts the + // same finite Controls bytes in a JSON array, without product-side seams. + let encoded; + let mut ready = false; + let bytes = if copied_config.len() == size_of::() { + &copied_config + } else { + let Ok(value) = serde_json::from_slice::(&copied_config) else { + return TB_BAD_ARGUMENT; + }; + ready = value + .get("test_ready") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + let Some(value) = value.get("test_controls") else { + return TB_BAD_ARGUMENT; + }; + let Ok(value) = serde_json::from_value::>(value.clone()) else { + return TB_BAD_ARGUMENT; + }; + encoded = value; + &encoded + }; + if bytes.len() != size_of::() { + return TB_BAD_ARGUMENT; + } + let controls = unsafe { bytes.as_ptr().cast::().read_unaligned() }; + // The host-side barrier blocks actual opaque C entry, retaining its config + // borrow through fault publication and initiating waiter abandonment. + unsafe { + (controls.entered)(controls.ctx, host); + } + assert_eq!( + unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }, + copied_config + ); + let stop = Arc::new((Mutex::new(false), Condvar::new())); + let worker_stop = stop.clone(); + let worker = std::thread::spawn(move || { + let mut stopped = worker_stop.0.lock().unwrap(); + while !*stopped { + stopped = worker_stop.1.wait(stopped).unwrap(); + } + }); + let instance = Box::new(Instance { + controls, + host: *host, + stop, + worker: Some(worker), + }); + unsafe { + *out = TbModuleVtable { + size: size_of::() as u32, + _reserved: 0, + module_ctx: Box::into_raw(instance).cast(), + deliver, + shutdown, + reinitialize: Some(reinitialize), + }; + } + if ready { + // Actual opaque initialization emits Ready; tests never fabricate it. + unsafe { + (host.ready)(host.host_ctx); + } + } + TB_OK +} diff --git a/crates/tinybus/src/module/abi.rs b/crates/tinybus/src/module/abi.rs index b9ffeecb..07d33db4 100644 --- a/crates/tinybus/src/module/abi.rs +++ b/crates/tinybus/src/module/abi.rs @@ -3,6 +3,7 @@ //! ABI revision one grows through optional vtable tails. Read the frozen prefix //! first and check its declared size before reading a tail; an older host has no //! broker-routing callback and therefore supplies no authenticated call context. +//! Each optional field has its own complete-field size check. use std::ffi::c_void; @@ -28,6 +29,22 @@ pub const TB_TIMEOUT: i32 = -4; /// A pointer, length, or frame was invalid. pub const TB_BAD_ARGUMENT: i32 = -5; +/// Exactly this value permits native per-instance admission. Zero, unknown +/// bits, or any combination with terminal bits must deny admission. A sampled +/// value is not a reusable permit for a later operation. +pub const TB_NATIVE_TERMINAL_LIVE: u32 = 1; +/// Terminal stop intent; once committed, the instance can never become live. +pub const TB_NATIVE_TERMINAL_STOP_INTENT: u32 = 2; +/// Actual fault intent, with public fault state taking precedence over stop. +pub const TB_NATIVE_TERMINAL_FAULT: u32 = 4; +/// An accepted opaque shutdown callback actually returned TB_OK. Fault and +/// stop intent remain set and continue to deny admission even with this bit. +/// A legacy callback's TB_OK alone proves neither independent worker completion +/// nor receiver backing destruction. For native terminal hooks, TB_OK must +/// additionally require actual worker/resource finish without sticky uncertainty +/// or fault; that stronger module contract is separate from this host evidence. +pub const TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED: u32 = 8; + /// Revision-one module initialization entrypoint. pub type TbModuleInit = unsafe extern "C" fn(*const TbHostVtable, *mut TbModuleVtable) -> i32; @@ -166,6 +183,25 @@ pub struct TbHostVtable { /// The SDK trusts this callback under the unsafe module-init host contract; /// it is not authentication of arbitrary user-provided C vtables. pub broker_routing: Option u32>, + /// Optional live query for this native module instance, read only when + /// `size` covers this entire field. Absence means unsupported and must deny + /// native admission. Only exactly [`TB_NATIVE_TERMINAL_LIVE`] permits it; + /// zero, unknown bits, combined values and terminal flags deny. Terminal + /// intent is monotonic and must never return to Live. + /// + /// Modules copy this callback and `host_ctx`, never retain the borrowed + /// vtable or config storage. The host must keep the copied callback and + /// context valid and thread-safe for every retained view use, including + /// independent workers after runtime shutdown and host destruction. A + /// destroyed host leaves a terminal tombstone, never recycles or reopens + /// this context. Custom unsafe hosts must truthfully uphold this contract; + /// the SDK cannot authenticate an arbitrary in-process C host. + /// + /// Query success linearizes admission only; it neither grants publication + /// authority nor cancels an operation already admitted before later intent. + /// The SDK's existing process-lifetime context retention is unbounded; + /// this optional callback adds no aggregate resource bound. + pub native_terminal_state: Option u32>, } /// Size required from an older host before reading its frozen prefix. @@ -176,12 +212,16 @@ impl TbHostVtable { /// /// Returns `None` for a null pointer or a size below /// [`TB_HOST_VTABLE_BASE_SIZE`]. A prefix-only or incomplete-tail host is - /// accepted with `broker_routing: None`; only a full tail is copied. + /// accepted. Each absent or incomplete optional callback normalizes to + /// `None`, independently of any later tail field. /// /// # Safety /// A non-null `host` must be aligned and readable for its first size field /// and every byte that size declares. Its callbacks must be valid and - /// thread-safe, and their context must outlive the module runtime. An + /// thread-safe, and their context must outlive the module runtime. If + /// `native_terminal_state` is supplied, its copied callback and context + /// must remain valid for every retained view use, even after runtime + /// shutdown or host destruction, as documented on that field. An /// affirmative routing callback (returning exactly 1) must truthfully /// guarantee exclusive broker delivery at the time each frame is queued. #[doc(hidden)] @@ -190,8 +230,17 @@ impl TbHostVtable { return None; } let base = unsafe { host.cast::().read() }; - let broker_routing = if base.size as usize >= size_of::() { - unsafe { std::ptr::addr_of!((*host).broker_routing).read() } + let query_size = size_of:: u32>>(); + let broker_routing = + if base.size as usize >= std::mem::offset_of!(Self, broker_routing) + query_size { + unsafe { std::ptr::addr_of!((*host).broker_routing).read() } + } else { + None + }; + let native_terminal_state = if base.size as usize + >= std::mem::offset_of!(Self, native_terminal_state) + query_size + { + unsafe { std::ptr::addr_of!((*host).native_terminal_state).read() } } else { None }; @@ -206,6 +255,7 @@ impl TbHostVtable { config: base.config, ready: base.ready, broker_routing, + native_terminal_state, }) } } diff --git a/crates/tinybus/src/module/abi_tests.rs b/crates/tinybus/src/module/abi_tests.rs index 6ebb313a..e7020fc4 100644 --- a/crates/tinybus/src/module/abi_tests.rs +++ b/crates/tinybus/src/module/abi_tests.rs @@ -41,3 +41,159 @@ fn an_uninitialized_module_vtable_refuses_calls() { TB_CLOSED ); } + +// These are the original ABI allocations, not truncated new vtables. Keeping +// their declarations independent detects accidental prefix or tail growth. +#[repr(C)] +struct FrozenHostPrefix { + size: u32, + reserved: u32, + ctx: *mut c_void, + send: unsafe extern "C" fn(*mut c_void, *const u8, usize) -> i32, + wake: unsafe extern "C" fn(*mut c_void), + log: unsafe extern "C" fn(*mut c_void, u32, *const u8, usize), + fault: unsafe extern "C" fn(*mut c_void, *const u8, usize), + config: TbSlice, + ready: unsafe extern "C" fn(*mut c_void), +} + +#[repr(C)] +struct FrozenRoutingHost { + prefix: FrozenHostPrefix, + routing: Option u32>, +} + +unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { + TB_OK +} +unsafe extern "C" fn wake(_: *mut c_void) {} +unsafe extern "C" fn log(_: *mut c_void, _: u32, _: *const u8, _: usize) {} +unsafe extern "C" fn fault(_: *mut c_void, _: *const u8, _: usize) {} +unsafe extern "C" fn query(ctx: *mut c_void) -> u32 { + unsafe { ctx.cast::().read() } +} + +fn frozen_prefix(size: usize) -> FrozenHostPrefix { + FrozenHostPrefix { + size: size as u32, + reserved: 0, + ctx: std::ptr::null_mut(), + send, + wake, + log, + fault, + config: TbSlice { + ptr: std::ptr::null(), + len: 0, + }, + ready: wake, + } +} + +#[test] +fn routing_tail_survives_newer_optional_tail() { + let value = 1u32; + let mut old = FrozenRoutingHost { + prefix: frozen_prefix(size_of::()), + routing: Some(query), + }; + old.prefix.ctx = std::ptr::from_ref(&value).cast_mut().cast(); + let normalized = + unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&old).cast()) }.unwrap(); + let routing = normalized + .broker_routing + .expect("complete original routing tail is retained"); + assert_eq!(unsafe { routing(normalized.host_ctx) }, 1); + assert!(normalized.native_terminal_state.is_none()); +} + +#[test] +fn prefix_hosts_normalize_both_tails() { + assert_eq!(ABI_REVISION, 1); + assert_eq!( + size_of::(), + TB_HOST_VTABLE_BASE_SIZE as usize + ); + assert_eq!( + size_of::(), + std::mem::offset_of!(TbHostVtable, broker_routing) + ); + assert!(unsafe { TbHostVtable::read_compatible(std::ptr::null()) }.is_none()); + // The rejected host owns only its declared bytes, with vtable alignment. + let short = TB_HOST_VTABLE_BASE_SIZE - 1; + let layout = + std::alloc::Layout::from_size_align(short as usize, align_of::()).unwrap(); + let storage = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!storage.is_null()); + unsafe { storage.cast::().write(short) }; + let rejected = unsafe { TbHostVtable::read_compatible(storage.cast()) }.is_none(); + unsafe { std::alloc::dealloc(storage, layout) }; + assert!(rejected); + let prefix = frozen_prefix(size_of::()); + let normalized = + unsafe { TbHostVtable::read_compatible(std::ptr::from_ref(&prefix).cast()) }.unwrap(); + assert!(normalized.broker_routing.is_none()); + assert!(normalized.native_terminal_state.is_none()); +} + +#[test] +fn terminal_tail_requires_its_complete_field() { + // Byte allocations are sized to the declared boundary and aligned to the + // vtable. No reference to a full vtable is created from shorter storage. + let routing_end = std::mem::offset_of!(TbHostVtable, broker_routing) + + size_of:: u32>>(); + let terminal_end = std::mem::offset_of!(TbHostVtable, native_terminal_state) + + size_of:: u32>>(); + let state = TB_NATIVE_TERMINAL_LIVE; + for size in [ + routing_end - 1, + routing_end, + terminal_end - 1, + terminal_end, + terminal_end + 16, + ] { + for terminal_present in [false, true] { + let layout = + std::alloc::Layout::from_size_align(size, align_of::()).unwrap(); + let storage = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!storage.is_null()); + let mut prefix = frozen_prefix(size); + prefix.ctx = std::ptr::from_ref(&state).cast_mut().cast(); + unsafe { storage.cast::().write(prefix) }; + if size >= routing_end { + unsafe { + storage + .add(std::mem::offset_of!(TbHostVtable, broker_routing)) + .cast:: u32>>() + .write(Some(query)) + }; + } + if size >= terminal_end { + unsafe { + storage + .add(std::mem::offset_of!(TbHostVtable, native_terminal_state)) + .cast:: u32>>() + .write(terminal_present.then_some(query)) + }; + } + let normalized = unsafe { TbHostVtable::read_compatible(storage.cast()) }.unwrap(); + unsafe { std::alloc::dealloc(storage, layout) }; + assert_eq!( + normalized.broker_routing.is_some(), + size >= routing_end, + "routing size {size}" + ); + assert_eq!( + normalized.native_terminal_state.is_some(), + size >= terminal_end && terminal_present, + "terminal size {size}" + ); + if let Some(query) = normalized.native_terminal_state { + assert_eq!( + unsafe { query(normalized.host_ctx) }, + TB_NATIVE_TERMINAL_LIVE + ); + } + } + } +} diff --git a/crates/tinybus/src/module/host.rs b/crates/tinybus/src/module/host.rs index d71be188..23887549 100644 --- a/crates/tinybus/src/module/host.rs +++ b/crates/tinybus/src/module/host.rs @@ -1022,16 +1022,13 @@ impl ModuleHost { transport.defer_initialize(artifact.init, host_vtable); } else { admitted.state = ModuleState::Initializing; - let mut module_vtable = TbModuleVtable::default(); - let code = unsafe { (artifact.init)(&host_vtable, &mut module_vtable) }; - transport.clear_config(); - if code != TB_OK { - let error = Error::module_refused(path, "module initialization failed"); - self.record_manifest_rejection(&error, manifest, RefusalClass::Failed); - return Err(error); - } - if transport.initialize(module_vtable).is_err() { - let error = Error::module_refused(path, "module returned an invalid vtable"); + if let Err(failure) = transport.eager_initialize(artifact.init, host_vtable) { + let reason = if failure.to_string().contains("vtable") { + "module returned an invalid vtable" + } else { + "module initialization failed" + }; + let error = Error::module_refused(path, reason); self.record_manifest_rejection(&error, manifest, RefusalClass::Failed); return Err(error); } @@ -1146,13 +1143,14 @@ impl ModuleHost { { Ok(change) => change, Err(_) => { + transport.commit_terminal(true); self.inner .broker .router .lock() .expect("router lock") .detach_unique(&unique); - let _ = transport.stop_sync(Duration::from_millis(0)); + transport.request_terminal(true); let error = Error::module_refused(path, "module bus name is already owned"); self.record_manifest_rejection(&error, manifest, RefusalClass::Unresolved); return Err(error); @@ -1503,7 +1501,7 @@ impl ModuleControl for ModuleHostInner { .lock() .expect("router lock") .revoke_callbacks_for_peer(&module.unique_name); - module.transition_from = Some(old); + module.transition_from.get_or_insert(old); module.lifecycle_busy = true; (module.transport.clone(), module.callback_state.clone()) }; @@ -1531,12 +1529,26 @@ impl ModuleControl for ModuleHostInner { match stopped { Ok(_) => {} Err(StopError::NotStarted(error)) => { - module.transition_from = None; + if !module.transport.stop_task_spawned() { + module.transition_from = None; + } return Err(error); } Err(StopError::Started(error)) => return Err(error), } - module.info.state = ModuleState::Stopped; + module.info.state = if module.transport.is_faulted() { + if module.transport.init_failed() { + ModuleState::Failed { + reason: "module initialization failed".into(), + } + } else { + ModuleState::Faulted { + reason: "module reported an unrecoverable fault".into(), + } + } + } else { + ModuleState::Stopped + }; Ok(module.info.clone()) } @@ -1686,9 +1698,11 @@ impl ModuleControl for ModuleHostInner { let module = loaded .iter_mut() .find(|module| &module.unique_name == unique_name)?; - if let Some(old) = module.transition_from.take() { - module.info.state = ModuleState::Stopped; - return Some((module.info.name.clone(), old, ModuleState::Stopped)); + if !module.transport.is_faulted() { + if let Some(old) = module.transition_from.take() { + module.info.state = ModuleState::Stopped; + return Some((module.info.name.clone(), old, ModuleState::Stopped)); + } } if !module.transport.is_faulted() || matches!( diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 2fff2132..3108561c 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -2,6 +2,7 @@ use super::*; use crate::message::Header; +use crate::module::transport::AdmissionTermination; use crate::transport::memory::MemoryTransport; use crate::{CallContext, Connection, Interface, InterfaceName, MemberName, ObjectPath}; use async_trait::async_trait; @@ -146,6 +147,13 @@ pub struct HostCallbackBinding { connection: Connection, state: Arc, transport: Arc, + // Bounded test observers only; neither hook supplies production authority. + #[cfg(test)] + after_attach: Mutex>>, + #[cfg(test)] + fixture_controller: Mutex>>, + #[cfg(test)] + before_load_decision: Mutex, tokio::sync::oneshot::Receiver<()>)>>, } /// Outgoing operations on the binding's private connection. @@ -437,6 +445,12 @@ impl ModuleHost { broker: self.inner.broker.clone(), connection, transport, + #[cfg(test)] + after_attach: Mutex::new(None), + #[cfg(test)] + fixture_controller: Mutex::new(None), + #[cfg(test)] + before_load_decision: Mutex::new(None), state: Arc::new(CallbackState { destination, endpoints, @@ -495,6 +509,7 @@ impl ModuleHost { host: self, transport: None, committed: false, + cause: AdmissionTermination::Failed, }; let path = path.as_ref(); if let Some(parent) = path.parent() { @@ -532,6 +547,19 @@ impl ModuleHost { let fingerprint = callback_config_fingerprint(&bytes); drop(config); let (transport, host) = ModuleTransport::new(admitted.name.clone(), bytes); + #[cfg(test)] + { + // Transfer the fixture config borrow before C can enter, so + // setup unwind and loader cancellation retain the same owner. + *transport + .fixture_controller + .lock() + .expect("fixture controller lock") = binding + .fixture_controller + .lock() + .expect("fixture controller lock") + .take(); + } transport.pause_initialization(); transport.defer_initialize(artifact.init, host); let name = admitted.name.clone(); @@ -549,6 +577,17 @@ impl ModuleHost { }, )?; rollback.transport = Some(transport.clone()); + #[cfg(test)] + { + let observer = binding + .after_attach + .lock() + .expect("test attach observer") + .take(); + if let Some(observer) = observer { + observer(); + } + } let mut loaded = self.inner.loaded.lock().expect("module list lock"); let module = loaded .iter_mut() @@ -582,18 +621,40 @@ impl ModuleHost { transport.release_initialization(true); transport }; + // All setup operations above are genuine failures. From this point a + // dropped waiter is cancellation; actual init error/deadline is explicit. + rollback.cause = AdmissionTermination::Canceled; let initialized = transport.ensure_initialized().await; - let ready = if initialized.is_ok() { - tokio::time::timeout(Duration::from_secs(5), transport.wait_ready()) - .await - .is_ok() - && transport.is_ready() - } else { - false + let failure = match initialized { + Err(Error::ConnectionClosed) => Some(AdmissionTermination::Canceled), + Err(_) => Some(AdmissionTermination::Failed), + Ok(()) => { + match tokio::time::timeout(Duration::from_secs(5), transport.wait_ready()).await { + Err(_) => Some(AdmissionTermination::Failed), + Ok(()) if !transport.is_ready() => Some(AdmissionTermination::Canceled), + Ok(()) => None, + } + } }; - if !ready { + if let Some(cause) = failure { + rollback.cause = cause; + } + #[cfg(test)] + { + let observer = binding + .before_load_decision + .lock() + .expect("test decision observer") + .take(); + if let Some((entered, release)) = observer { + entered.notify_one(); + release.await.expect("test decision observer released"); + } + } + if let Some(cause) = failure { + let fault = transport.commit_admission_termination(cause); binding.revoke(); - let _ = transport.stop_sync(Duration::ZERO); + transport.request_terminal(fault); let _ = transport.close().await; self.inner .loaded @@ -620,6 +681,7 @@ impl ModuleHost { // turn the returned successful snapshot into Faulted or Stopped. if lifetime.revoked || !lifetime.active + || transport.admission_closed() || !transport.is_ready() || !matches!(info.state, ModuleState::Ready) { @@ -638,12 +700,17 @@ struct LoadRollback<'a> { host: &'a ModuleHost, transport: Option>, committed: bool, + cause: AdmissionTermination, } impl Drop for LoadRollback<'_> { fn drop(&mut self) { if self.committed { return; } + let fault = self + .transport + .as_ref() + .map(|transport| transport.commit_admission_termination(self.cause)); self.binding.revoke(); self.binding.transport.close_outbound(); self.binding @@ -675,7 +742,7 @@ impl Drop for LoadRollback<'_> { } // A timed-out initializer may still run. Stop is best effort; // revocation/detach removes authority without claiming cessation. - let _ = transport.stop_sync(Duration::ZERO); + transport.request_terminal(fault.expect("rollback transport cause")); self.host .inner .loaded @@ -688,12 +755,22 @@ impl Drop for LoadRollback<'_> { impl Drop for ModuleHostInner { fn drop(&mut self) { - for module in self.loaded.get_mut().expect("module list lock").iter() { - self.broker - .router - .lock() - .expect("router lock") - .revoke_callbacks_for_peer(&module.unique_name); + let modules = self.loaded.get_mut().expect("module list lock"); + let transports: Vec<_> = modules + .iter() + .map(|module| module.transport.clone()) + .collect(); + for transport in &transports { + transport.commit_terminal(false); + } + { + let mut router = self.broker.router.lock().expect("router lock"); + for module in modules.iter() { + router.revoke_callbacks_for_peer(&module.unique_name); + } + } + for transport in transports { + transport.request_terminal(false); } } } diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 4ddd7b2f..8c964d73 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -501,6 +501,7 @@ async fn rollback_after_attach_releases_the_actual_initialization_observer() { host: &host, transport: Some(transport.clone()), committed: false, + cause: AdmissionTermination::Canceled, }); assert!(!transport.init_started()); assert!(host.list().is_empty()); @@ -556,3 +557,328 @@ fn configuration_fingerprints_preserve_json_equality_without_normalizing_strings fingerprint(&serde_json::json!(0.01)) ); } + +// Manually polling the public future lets ordinary shutdown commit while the +// loader is suspended; no scheduler race decides which cancellation is tested. +async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: bool) { + use super::super::super::transport::tests::{native_fixture, native_fixture_json}; + use std::future::Future; + let (unused, _, _, barrier) = native_fixture(); + let config = native_fixture_json(&unused); + drop(unused); + let artifact = + std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); + let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); + let (_dir, artifact) = super::super::tests::staged_module(&artifact, &hash); + let host = ModuleHost::new(Broker::new()); + let binding = host.prepare_host_callbacks(vec![endpoint()]).await.unwrap(); + *binding.fixture_controller.lock().unwrap() = Some(barrier.clone()); + let (entered, _) = tokio::sync::mpsc::channel(1); + binding + .serve_at( + endpoint().path, + BlockingCallback { + entered, + release: Arc::new(tokio::sync::Semaphore::new(0)), + published: Arc::new(AtomicUsize::new(0)), + guard: binding.guard(), + }, + ) + .await + .unwrap(); + let mut load = Some(Box::pin( + host.load_file_with_host_callbacks(&artifact, config, &binding), + )); + std::future::poll_fn(|cx| { + assert!( + load.as_mut().unwrap().as_mut().poll(cx).is_pending(), + "loader awaits real C init" + ); + std::task::Poll::Ready(()) + }) + .await; + tokio::time::timeout(Duration::from_secs(2), barrier.entered.notified()) + .await + .unwrap(); + let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + let view = barrier.state.lock().unwrap().0.unwrap(); + barrier.state.lock().unwrap().1 = true; + barrier.changed.notify_all(); + transport.ensure_initialized().await.unwrap(); + // Advance the public future to its real withheld-Ready observation. + std::future::poll_fn(|cx| { + assert!( + load.as_mut().unwrap().as_mut().poll(cx).is_pending(), + "fixture withholds Ready" + ); + std::task::Poll::Ready(()) + }) + .await; + let (finish, receive) = std::sync::mpsc::channel(); + *barrier.finish_gate.lock().unwrap() = Some(receive); + let stopping = transport.clone(); + let stop = tokio::spawn(async move { stopping.stop(Duration::from_secs(2)).await }); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2 + ); + let load_result = if reject_gate { + transport.release_initialization(false); + drop(load.take()); + None + } else if drop_load { + drop(load.take()); + None + } else { + // Real not-ready branch awaits close, whose native return is held. + std::future::poll_fn(|cx| match load.as_mut().unwrap().as_mut().poll(cx) { + std::task::Poll::Pending => std::task::Poll::Ready(None), + std::task::Poll::Ready(result) => std::task::Poll::Ready(Some(result)), + }) + .await + }; + let held_flags = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; + let held_fault = transport.is_faulted(); + finish.send(()).unwrap(); // Always release opaque C before RED assertions. + assert!(stop.await.unwrap().is_ok()); + if let Some(result) = load_result { + assert!(result.is_err()); + drop(load.take()); + } else if let Some(load) = load { + assert!(load.await.is_err()); + } + assert_eq!(held_flags, 2, "ordinary cancellation cannot invent Fault"); + assert!(!held_fault); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 10 + ); + assert!(barrier.state.lock().unwrap().2); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_callback_notready_after_ordinary_stop_preserves_cause() { + native_callback_load_after_ordinary_stop(false, false).await; +} +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_callback_load_drop_after_ordinary_stop_preserves_cause() { + native_callback_load_after_ordinary_stop(true, false).await; +} +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_callback_gate_rejection_after_ordinary_stop_preserves_cause() { + native_callback_load_after_ordinary_stop(false, true).await; +} + +async fn served_native_binding(host: &ModuleHost) -> HostCallbackBinding { + let binding = host.prepare_host_callbacks(vec![endpoint()]).await.unwrap(); + let (entered, _) = tokio::sync::mpsc::channel(1); + binding + .serve_at( + endpoint().path, + BlockingCallback { + entered, + release: Arc::new(tokio::sync::Semaphore::new(0)), + published: Arc::new(AtomicUsize::new(0)), + guard: binding.guard(), + }, + ) + .await + .unwrap(); + binding +} + +async fn native_callback_decision_race(ready: bool, native_fault: bool) { + use super::super::super::transport::tests::{native_fixture, native_fixture_json}; + let (unused, _, _, barrier) = native_fixture(); + let mut config = native_fixture_json(&unused); + config["test_ready"] = serde_json::json!(ready); + drop(unused); + let artifact = + std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); + let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); + let (_dir, artifact) = super::super::tests::staged_module(&artifact, &hash); + let host = Arc::new(ModuleHost::new(Broker::new())); + let binding = Arc::new(served_native_binding(&host).await); + *binding.fixture_controller.lock().unwrap() = Some(barrier.clone()); + let decision = Arc::new(Notify::new()); + let (resume, release) = tokio::sync::oneshot::channel(); + *binding.before_load_decision.lock().unwrap() = Some((decision.clone(), release)); + let loading_host = host.clone(); + let loading_binding = binding.clone(); + let load = tokio::spawn(async move { + loading_host + .load_file_with_host_callbacks(artifact, config, &loading_binding) + .await + }); + // Keep the paused runtime runnable until actual C entry/publication. This + // prevents virtual time from auto-advancing the initializer's deadline. + while barrier.state.lock().unwrap().0.is_none() { + tokio::task::yield_now().await; + } + let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + let view = barrier.state.lock().unwrap().0.unwrap(); + barrier.state.lock().unwrap().1 = true; + barrier.changed.notify_all(); + // Poll until native initialization is genuinely published, without an idle + // paused timer deciding the deadline before the native owner can run. + while !transport.init_started() { + tokio::task::yield_now().await; + } + // init_started precedes actual publication; the observer result is the proof. + let initialized = transport.ensure_initialized(); + tokio::pin!(initialized); + std::future::poll_fn( + |cx| match std::future::Future::poll(initialized.as_mut(), cx) { + std::task::Poll::Ready(result) => std::task::Poll::Ready(result), + std::task::Poll::Pending => { + cx.waker().wake_by_ref(); + std::task::Poll::Pending + } + }, + ) + .await + .unwrap(); + if !ready { + // Real wait_ready deadline, not a fake readiness flag. Let the public + // future install its timer before advancing past the actual bound. + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(6)).await; + } + decision.notified().await; + assert_eq!(transport.is_ready(), ready); + let (finish, receive) = std::sync::mpsc::channel(); + *barrier.finish_gate.lock().unwrap() = Some(receive); + let stopping = transport.clone(); + let stop = tokio::spawn(async move { stopping.stop(Duration::from_secs(2)).await }); + while !barrier.state.lock().unwrap().2 { + tokio::task::yield_now().await; + } + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2 + ); + if native_fault { + unsafe { + (view.fault)(view.host_ctx, std::ptr::null(), 0); + } + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 6 + ); + } + resume.send(()).unwrap(); + // Rollback revokes before awaiting physical close or returning failure. + while !binding.state.lifetime.lock().unwrap().revoked && !load.is_finished() { + tokio::task::yield_now().await; + } + let held = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; + finish.send(()).unwrap(); + // Keep virtual time runnable through the actual opaque return. + while !stop.is_finished() { + tokio::task::yield_now().await; + } + let stopped = stop.await.unwrap(); + assert!(stopped.is_ok(), "actual stop: {stopped:?}"); + assert!(load.await.unwrap().is_err()); + assert_eq!(held, if ready && !native_fault { 2 } else { 6 }); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + if ready && !native_fault { 10 } else { 14 } + ); + assert_eq!(transport.is_faulted(), !ready || native_fault); +} + +#[tokio::test(start_paused = true)] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_ready_commit_cancellation_preserves_ordinary_cause() { + native_callback_decision_race(true, false).await; +} +#[tokio::test(start_paused = true)] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_readiness_deadline_still_faults_after_ordinary_stop() { + native_callback_decision_race(false, false).await; +} +#[tokio::test(start_paused = true)] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_fault_at_ready_commit_dominates_ordinary_stop() { + native_callback_decision_race(true, true).await; +} + +async fn native_callback_setup_failure_race(panic_setup: bool) { + use super::super::super::transport::tests::{native_fixture, native_fixture_json}; + let (unused, _, _, barrier) = native_fixture(); + let config = native_fixture_json(&unused); + drop(unused); + let artifact = + std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); + let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); + let (_dir, artifact) = super::super::tests::staged_module(&artifact, &hash); + let host = Arc::new(ModuleHost::new(Broker::new())); + let binding = Arc::new(served_native_binding(&host).await); + *binding.fixture_controller.lock().unwrap() = Some(barrier.clone()); + let attached = Arc::new(Notify::new()); + let seen = attached.clone(); + let (resume, release) = std::sync::mpsc::channel(); + *binding.after_attach.lock().unwrap() = Some(Box::new(move || { + seen.notify_one(); + release.recv().unwrap(); + assert!( + !panic_setup, + "actual Rust unwind during host callback setup" + ); + })); + let loading_host = host.clone(); + let loading_binding = binding.clone(); + let load = tokio::spawn(async move { + loading_host + .load_file_with_host_callbacks(artifact, config, &loading_binding) + .await + }); + // Admission hashes the real artifact twice before this hook; instrumented + // CI builds can spend several seconds there. This is a fixture setup bound, + // independent of the lifecycle deadline and the race assertions below. + tokio::time::timeout(Duration::from_secs(30), attached.notified()) + .await + .unwrap(); + let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + // Actual ordinary lifecycle ownership, before the callback initialization + // gate opens. This instance proves C never entered, rather than any join. + let _ = transport.clone().stop(Duration::from_secs(2)).await; + assert!(!transport.is_faulted()); + if !panic_setup { + // Public revocation makes real setup return its concurrently-used error. + binding.revoke(); + } + resume.send(()).unwrap(); + let result = load.await; + if panic_setup { + assert!(result.unwrap_err().is_panic()); + } else { + assert!(result.unwrap().is_err()); + } + assert!( + transport.is_faulted(), + "genuine setup failure upgrades ordinary stop" + ); + assert!( + barrier.state.lock().unwrap().0.is_none(), + "C initializer never entered" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_callback_setup_error_still_faults_after_ordinary_stop() { + native_callback_setup_failure_race(false).await; +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_callback_setup_unwind_still_faults_after_ordinary_stop() { + native_callback_setup_failure_race(true).await; +} diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index b030a982..7fd85cae 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -1870,6 +1870,9 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { let _ = ModuleHost::new(Broker::new()).load_search_paths(); let directory = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); + // Files inherit the private DACL, so these assertions exercise content gates. + #[cfg(windows)] + crate::module::cache::secure_release_cache(directory.path(), directory.path()); let module = directory.path().join(if cfg!(windows) { "clock.dll" } else if cfg!(target_os = "macos") { @@ -1929,6 +1932,9 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { #[test] fn an_allowlist_with_a_mismatched_hash_refuses_the_file() { let directory = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); + // Files inherit the private DACL, so these assertions exercise content gates. + #[cfg(windows)] + crate::module::cache::secure_release_cache(directory.path(), directory.path()); let extension = if cfg!(windows) { "dll" } else if cfg!(target_os = "macos") { @@ -2197,7 +2203,7 @@ async fn one_refused_module_does_not_stop_the_others_in_the_directory_from_loadi /// tracing subscriber inside the library, so initializing the same DLL path /// twice shares state even across independent hosts. Windows staging uses the /// release-cache ACL repair so the real admission gate remains in force. -fn staged_module(artifact: &Path, hash: &str) -> (tempfile::TempDir, PathBuf) { +pub(super) fn staged_module(artifact: &Path, hash: &str) -> (tempfile::TempDir, PathBuf) { // Staged beside the artifact rather than in `/tmp`: the loader refuses a // directory another user could write to, and `/tmp` is exactly that. The // artifact's own directory is user-owned (CI builds into @@ -2893,3 +2899,372 @@ impl crate::Interface for NativeBlockingCallback { Ok(serde_json::json!(context.authenticated_sender().unwrap())) } } + +static BLOCKING_STOP_RECEIVER: OnceLock>>> = + OnceLock::new(); +static BLOCKING_STOP_STARTED: AtomicBool = AtomicBool::new(false); +static BLOCKING_STOP_CALLS: AtomicUsize = AtomicUsize::new(0); + +unsafe extern "C" fn blocking_shutdown(_: *mut std::ffi::c_void, _: u64) -> i32 { + BLOCKING_STOP_CALLS.fetch_add(1, Ordering::AcqRel); + BLOCKING_STOP_STARTED.store(true, Ordering::Release); + if let Some(receiver) = BLOCKING_STOP_RECEIVER + .get_or_init(|| StdMutex::new(None)) + .lock() + .expect("H00") + .take() + { + let _ = receiver.recv(); + } + TB_OK +} +unsafe extern "C" fn blocking_shutdown_init( + host: *const crate::module::abi::TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + let code = unsafe { lazy_echo_init(host, out) }; + if code == TB_OK { + unsafe { + (*out).shutdown = blocking_shutdown; + } + } + code +} +struct StopRelease(Option>); +impl Drop for StopRelease { + fn drop(&mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } +} + +#[tokio::test] +async fn ordinary_stop_never_fabricates_faulted_before_or_after_callback_completion() { + let _test_guard = FAKE_MODULE_TEST_LOCK.lock().await; + let bus = MemoryBus::new(); + let broker = Broker::new(); + let broker_task = broker.spawn(bus.clone()); + let host = ModuleHost::new(broker); + let observer = Connection::connect(bus.connect().await.expect("H18")) + .await + .expect("H19"); + let mut state_changes = observer + .add_match( + crate::router::MatchRule::parse( + "type=signal,interface=ai.tinyhumans.tinybus.Bus,member=ModuleStateChanged", + ) + .expect("H20"), + ) + .await + .expect("H21"); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let release = StopRelease(Some(sender)); + *BLOCKING_STOP_RECEIVER + .get_or_init(|| StdMutex::new(None)) + .lock() + .expect("H01") = Some(receiver); + BLOCKING_STOP_STARTED.store(false, Ordering::Release); + BLOCKING_STOP_CALLS.store(0, Ordering::Release); + unsafe { + host.attach_raw( + "clock.so", + TbAbiDescriptor::current("clock", "0.1.0"), + manifest(), + blocking_shutdown_init, + ) + } + .expect("H02"); + tokio::time::timeout(Duration::from_secs(2), async { + while host.list()[0].state != ModuleState::Ready { + tokio::task::yield_now().await; + } + }) + .await + .expect("H03"); + let inner = host.inner.clone(); + let stopping = + tokio::spawn(async move { inner.stop("clock", Duration::from_millis(100)).await }); + tokio::time::timeout(Duration::from_secs(2), async { + while !BLOCKING_STOP_STARTED.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("H04"); + assert!(stopping.await.expect("H05").is_err(), "H06"); + assert_eq!(host.list()[0].state, ModuleState::Ready, "H07"); + let retry = host.inner.stop("clock", Duration::from_millis(10)).await; + assert!(matches!(retry, Err(Error::MethodFailed { .. })), "H08"); + assert_eq!(BLOCKING_STOP_CALLS.load(Ordering::Acquire), 1, "H09"); + // Hold the host observer's lock while the blocking callback returns. The + // real broker cannot commit peer detach until this assertion is finished. + // No await occurs under this lock, and the opaque callback needs no host lock. + { + let loaded = host.inner.loaded.lock().expect("H10"); + drop(release); + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while loaded[0].transport.is_ready() { + assert!(std::time::Instant::now() < deadline, "H11"); + std::thread::yield_now(); + } + assert!(!loaded[0].transport.is_faulted(), "H12"); + // A disposed transport refuses a retry immediately, before any new + // physical task or marker change, even while detach is held back. + let mut retry = std::pin::pin!(loaded[0].transport.clone().stop(Duration::from_secs(1))); + let mut context = std::task::Context::from_waker(std::task::Waker::noop()); + assert!( + matches!( + std::future::Future::poll(retry.as_mut(), &mut context), + std::task::Poll::Ready(Err(StopError::NotStarted(Error::ConnectionClosed))) + ), + "H15" + ); + assert!(loaded[0].transport.stop_task_spawned(), "H16"); + assert_eq!(BLOCKING_STOP_CALLS.load(Ordering::Acquire), 1, "H17"); + assert!( + !matches!( + loaded[0].snapshot().state, + ModuleState::Faulted { .. } | ModuleState::Failed { .. } + ), + "H13" + ); + } + // This is a current-thread runtime. Without an await or yield after dropping + // the loaded lock, the broker cannot process its ready detach task before + // this real Host retry completes synchronously against the closed transport. + { + let mut retry = std::pin::pin!(host.inner.stop("clock", Duration::from_secs(1))); + let mut context = std::task::Context::from_waker(std::task::Waker::noop()); + assert!( + matches!( + std::future::Future::poll(retry.as_mut(), &mut context), + std::task::Poll::Ready(Err(Error::ConnectionClosed)) + ), + "H22" + ); + } + let origin = host.inner.loaded.lock().expect("H23")[0] + .transition_from + .clone(); + assert!(matches!(origin, Some(ModuleState::Ready)), "H24"); + tokio::time::timeout(Duration::from_secs(2), async { + while host.list()[0].state != ModuleState::Stopped { + tokio::task::yield_now().await; + } + }) + .await + .expect("H14"); + let stopped_event = tokio::time::timeout(Duration::from_secs(2), async { + loop { + let message = state_changes.recv().await.expect("H25"); + if message.body.get(2).and_then(serde_json::Value::as_str) == Some("stopped") { + break message; + } + } + }) + .await + .expect("H26"); + assert!( + stopped_event.body == serde_json::json!(["clock", "ready", "stopped", null]), + "H27" + ); + broker_task.abort(); +} + +async fn controlled_native_transport() -> ( + Arc, + crate::module::abi::TbHostVtable, + Arc, + LoadedArtifact, + PathBuf, +) { + let (transport, view, init, barrier) = super::super::transport::tests::native_fixture(); + barrier.state.lock().unwrap().1 = true; + transport.defer_initialize(init, view); + transport.ensure_initialized().await.unwrap(); + barrier.entered.notified().await; + let path = PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE").unwrap()); + let artifact = loader::load(&path, false).unwrap(); + (transport, view, barrier, artifact, path) +} + +fn attach_controlled_native( + host: &ModuleHost, + transport: Arc, + artifact: LoadedArtifact, + path: &Path, +) -> Result { + let mut admitted = host.validate(path, &artifact.descriptor, &artifact.manifest)?; + admitted.state = ModuleState::Initializing; + host.attach_transport( + path, + admitted, + artifact.manifest, + transport, + Activation { + lazy_init: false, + lazy_load: false, + descriptor_info: None, + pinned_sha256: None, + }, + ) +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_host_drop_keeps_terminal_tombstone_and_other_instance_live() { + let (transport, view, barrier, artifact, path) = controlled_native_transport().await; + let (other, other_view, other_barrier, _, _) = controlled_native_transport().await; + let host = ModuleHost::new(Broker::new()); + attach_controlled_native(&host, transport.clone(), artifact, &path).unwrap(); + drop(host); + assert_ne!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 1 + ); + assert_eq!( + unsafe { other_view.native_terminal_state.unwrap()(other_view.host_ctx) }, + 1 + ); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); + let stopped = transport.clone().stop(Duration::from_secs(2)).await; + assert!(matches!(stopped, Err(StopError::NotStarted(_)))); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 10 + ); + drop(transport); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 10 + ); + other.request_terminal(false); + tokio::time::timeout(Duration::from_secs(2), other_barrier.joined.notified()) + .await + .unwrap(); + assert!(barrier.state.lock().unwrap().2); + assert!(other_barrier.state.lock().unwrap().2); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_reservation_rollback_retains_one_cleanup_and_fault_tombstone() { + let (first, first_view, first_barrier, artifact, path) = controlled_native_transport().await; + let (rejected, view, barrier, other_artifact, _) = controlled_native_transport().await; + let host = ModuleHost::new(Broker::new()); + attach_controlled_native(&host, first, artifact, &path).unwrap(); + assert!(attach_controlled_native(&host, rejected.clone(), other_artifact, &path).is_err()); + assert_ne!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 1 + ); + assert_eq!( + unsafe { first_view.native_terminal_state.unwrap()(first_view.host_ctx) }, + 1 + ); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); + let _observed = rejected.clone().stop(Duration::from_secs(2)).await; + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 14 + ); + assert!(rejected.ensure_initialized().await.is_err()); + drop(rejected); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 14 + ); + drop(host); + tokio::time::timeout(Duration::from_secs(2), first_barrier.joined.notified()) + .await + .unwrap(); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_fault_during_ordinary_stop_keeps_fault_state_and_single_join() { + let (transport, view, barrier, artifact, path) = controlled_native_transport().await; + let host = Arc::new(ModuleHost::new(Broker::new())); + let info = attach_controlled_native(&host, transport.clone(), artifact, &path).unwrap(); + let (release, release_rx) = std::sync::mpsc::channel(); + *barrier.finish_gate.lock().unwrap() = Some(release_rx); + let stopping = tokio::spawn({ + let host = host.clone(); + async move { + host.inner + .stop_inner(&info.name, Duration::from_secs(2)) + .await + } + }); + barrier.joined.notified().await; + assert!(transport.stop_task_spawned()); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2, + "joined worker is not yet successful opaque return" + ); + unsafe { + (view.fault)(view.host_ctx, std::ptr::null(), 0); + } + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 6 + ); + release.send(()).unwrap(); + let result = stopping.await.unwrap().unwrap(); + assert!( + matches!(result.state, ModuleState::Faulted { .. }), + "late ordinary completion cannot replace actual fault" + ); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 14 + ); + assert!(barrier.state.lock().unwrap().2); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_rejected_stop_submission_retains_guard_through_host_drop() { + let (transport, view, barrier, artifact, path) = controlled_native_transport().await; + let host = ModuleHost::new(Broker::new()); + attach_controlled_native(&host, transport.clone(), artifact, &path).unwrap(); + transport.reject_worker_spawn.store(true, Ordering::Release); + assert!(matches!( + transport.clone().stop(Duration::from_secs(1)).await, + Err(StopError::Started(_)) + )); + let weak = Arc::downgrade(&transport); + drop(host); + drop(transport); + let retained = weak + .upgrade() + .expect("external anchor retains physical owner through real Host Drop"); + assert!(retained.lifecycle_is_owned_for_test()); + assert!(retained.stop_task_spawned()); + assert!(!retained.is_faulted()); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2 + ); + assert!( + !barrier.state.lock().unwrap().2, + "rejected submission is not physical completion" + ); + retained.reject_worker_spawn.store(false, Ordering::Release); + retained.request_terminal(false); + assert!( + !barrier.state.lock().unwrap().2, + "Host Drop cannot refund and resubmit an uncertain owner" + ); + // This one uncertain physical owner retains its actual controller. + // It makes no process-wide leak or backing-Drop claim. + let controller = Arc::downgrade(&barrier); + drop(barrier); + assert!(controller.upgrade().is_some()); +} diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 7aa409b0..152fab0a 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -2,12 +2,12 @@ use std::collections::VecDeque; use std::ffi::c_void; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex as StdMutex, Weak}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex as StdMutex, Weak}; use std::time::Duration; use async_trait::async_trait; -use tokio::sync::{Mutex, Notify, OnceCell, mpsc}; +use tokio::sync::{Mutex, Notify, mpsc}; use crate::error::{Error, Result}; use crate::message::codec::MAX_FRAME_LEN; @@ -31,16 +31,32 @@ struct HostContext { inbound: StdMutex>>>, wake: Arc, config: StdMutex>, + owner: StdMutex>, + native_terminal_state: AtomicU32, + physical: StdMutex, + physical_changed: Condvar, + physical_notify: Notify, faulted: AtomicBool, init_failed: AtomicBool, + stopped: AtomicBool, + admission_closed: AtomicBool, init_started: AtomicBool, init_notify: Notify, ready: AtomicBool, + ready_gate: StdMutex<()>, broker_routing: AtomicBool, ready_notify: Notify, inflight: AtomicUsize, } +/// Admission errors remain faults; cancellation preserves an already committed +/// ordinary stop. Neither classification can clear a real, sticky Fault. +#[derive(Clone, Copy)] +pub(crate) enum AdmissionTermination { + Failed, + Canceled, +} + /// The broker-facing side of one loaded module. pub(crate) struct ModuleTransport { self_ref: Weak, @@ -49,8 +65,7 @@ pub(crate) struct ModuleTransport { context: &'static HostContext, label: String, initializer: StdMutex>, - init_result: OnceCell>, - pending: Mutex>, + pending: StdMutex>, drain_started: AtomicBool, // Reconfiguration and shutdown both call into opaque module code. The // owned guard moves into the blocking task, so a host-side timeout cannot @@ -58,8 +73,67 @@ pub(crate) struct ModuleTransport { lifecycle: Arc>, native_lifecycle_pending: Arc, stop_task_spawned: Arc, + callbacks: CallbackGate, initialization_gate: StdMutex>, initialization_gate_changed: Notify, + #[cfg(test)] + pub(crate) reject_worker_spawn: AtomicBool, + #[cfg(test)] + before_worker_lock: StdMutex>>, + #[cfg(test)] + before_init_publication: StdMutex>>, + #[cfg(test)] + after_cleanup_publication: StdMutex>>, + // Native fixtures pass a borrowed controller through C config. Retain its + // actual owner with the physical transport, including uncertain jobs. + #[cfg(test)] + pub(crate) fixture_controller: StdMutex>>, +} + +// Reservations cover physical callback execution, including signals. They are +// independent of unanswered method calls and remain owned by the actual caller. +#[derive(Default)] +struct CallbackState { + closed: bool, + active: usize, +} + +#[derive(Default)] +struct CallbackGate { + state: StdMutex, + drained: Condvar, +} + +struct CallbackLease<'a>(&'a CallbackGate); + +impl CallbackGate { + /// Reserve entry atomically with physical stop's admission closure. + fn enter(&self) -> Result> { + let mut state = self.state.lock().expect("module callback lock"); + if state.closed { + return Err(Error::ConnectionClosed); + } + state.active += 1; + Ok(CallbackLease(self)) + } + + /// Retain the shutdown worker until every reserved callback has returned. + fn wait_drained(&self) { + let mut state = self.state.lock().expect("module callback lock"); + while state.active != 0 { + state = self.drained.wait(state).expect("module callback lock"); + } + } +} + +impl Drop for CallbackLease<'_> { + fn drop(&mut self) { + let mut state = self.0.state.lock().expect("module callback lock"); + state.active -= 1; + if state.active == 0 { + self.0.drained.notify_all(); + } + } } // `module_ctx` is opaque and all access to it goes through callbacks whose ABI @@ -75,8 +149,64 @@ struct DeferredInitializer { host: TbHostVtable, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum InitState { + Deferred, + Scheduled, + Entered, + PublishedValid, + PublishedRejected, + CanceledBeforeEntry, + ExternalValid, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CleanupState { + Idle, + Requested, + Executing, + Complete, + Uncertain, + NoCallbackRequired, +} + +struct PhysicalState { + init: InitState, + cleanup: CleanupState, + job: Option>, + result: Option>, + code: Option, +} + +struct PhysicalJob { + transport: Arc, + initializer: StdMutex>, + guard: StdMutex>>, + deadline: Duration, + fault_on_spawn_failure: bool, + submitted: AtomicBool, + started: AtomicBool, +} + +struct InitiatingWaiter<'a> { + transport: &'a ModuleTransport, + armed: bool, +} +impl Drop for InitiatingWaiter<'_> { + fn drop(&mut self) { + if self.armed { + self.transport.abandon_initialization(); + } + } +} + struct SendModuleVtable(TbModuleVtable); +#[derive(Clone)] +enum InitFailure { + Closed, + Failed(String), +} + // Native callbacks outlive canceled public waiters. Clear this marker before // releasing the mutex so an older callback cannot clear its successor's flag. struct NativeLifecyclePending(Arc); @@ -124,11 +254,25 @@ impl ModuleTransport { inbound: StdMutex::new(Some(inbound_tx)), wake: Arc::new(Notify::new()), config: StdMutex::new(config), + owner: StdMutex::new(Weak::new()), + native_terminal_state: AtomicU32::new(crate::module::abi::TB_NATIVE_TERMINAL_LIVE), + physical: StdMutex::new(PhysicalState { + init: InitState::Deferred, + cleanup: CleanupState::Idle, + job: None, + result: None, + code: None, + }), + physical_changed: Condvar::new(), + physical_notify: Notify::new(), faulted: AtomicBool::new(false), init_failed: AtomicBool::new(false), + stopped: AtomicBool::new(false), + admission_closed: AtomicBool::new(false), init_started: AtomicBool::new(false), init_notify: Notify::new(), ready: AtomicBool::new(false), + ready_gate: StdMutex::new(()), broker_routing: AtomicBool::new(false), ready_notify: Notify::new(), inflight: AtomicUsize::new(0), @@ -140,15 +284,26 @@ impl ModuleTransport { context, label, initializer: StdMutex::new(None), - init_result: OnceCell::new(), - pending: Mutex::new(VecDeque::new()), + pending: StdMutex::new(VecDeque::new()), drain_started: AtomicBool::new(false), lifecycle: Arc::new(Mutex::new(())), native_lifecycle_pending: Arc::new(AtomicBool::new(false)), stop_task_spawned: Arc::new(AtomicBool::new(false)), + callbacks: CallbackGate::default(), initialization_gate: StdMutex::new(Some(true)), initialization_gate_changed: Notify::new(), + #[cfg(test)] + reject_worker_spawn: AtomicBool::new(false), + #[cfg(test)] + before_worker_lock: StdMutex::new(None), + #[cfg(test)] + before_init_publication: StdMutex::new(None), + #[cfg(test)] + after_cleanup_publication: StdMutex::new(None), + #[cfg(test)] + fixture_controller: StdMutex::new(None), }); + *context.owner.lock().expect("physical owner lock") = Arc::downgrade(&transport); let config = context.config.lock().expect("module config lock"); let config_slice = crate::module::abi::TbSlice { ptr: config.as_ptr(), @@ -166,6 +321,7 @@ impl ModuleTransport { config: config_slice, ready: host_ready, broker_routing: Some(host_broker_routing), + native_terminal_state: Some(host_native_terminal_state), }; (transport, vtable) } @@ -178,6 +334,11 @@ impl ModuleTransport { return Err(Error::transport("module returned an incomplete vtable")); } *self.module.lock().expect("module vtable lock") = Some(module); + let mut physical = self.context.physical.lock().expect("physical state lock"); + if physical.init == InitState::Deferred { + physical.init = InitState::ExternalValid; + physical.result = Some(Ok(())); + } Ok(()) } @@ -225,15 +386,61 @@ impl ModuleTransport { } } - // Terminal state is distinct from initialization: never fabricate READY or - // init_started to release observers during admission rollback or shutdown. + // Releasing rejected observers is cancellation, which may follow an actual + // ordinary stop. Never fabricate READY or init_started to release them. fn terminate_waiters(&self) { - self.context.faulted.store(true, Ordering::Release); + let fault = self.commit_admission_termination(AdmissionTermination::Canceled); + self.request_terminal(fault); self.context.init_notify.notify_waiters(); self.context.ready_notify.notify_waiters(); self.initialization_gate_changed.notify_waiters(); } + /// Close admission before teardown, without reporting physical completion + /// or inventing a fault. The lifecycle owner alone invokes this operation. + fn close_admission(&self) { + let discarded = { + let _initialization = self + .initialization_gate + .lock() + .expect("initialization gate lock"); + let mut pending = self.pending.lock().expect("module pending lock"); + let mut callbacks = self.callbacks.state.lock().expect("module callback lock"); + callbacks.closed = true; + let _ready = self.context.ready_gate.lock().expect("module ready lock"); + self.context.admission_closed.store(true, Ordering::Release); + std::mem::take(&mut *pending) + }; + // Queued confidential bodies are released outside internal locks. + drop(discarded); + self.context.init_notify.notify_waiters(); + self.context.ready_notify.notify_waiters(); + self.context.wake.notify_waiters(); + self.initialization_gate_changed.notify_waiters(); + } + + /// Publish physical completion only after the opaque shutdown returns. + fn finish_stop(&self) { + self.context.stopped.store(true, Ordering::Release); + self.context + .inbound + .lock() + .expect("host inbound lock") + .take(); + self.context.init_notify.notify_waiters(); + self.context.ready_notify.notify_waiters(); + self.context.wake.notify_waiters(); + self.initialization_gate_changed.notify_waiters(); + } + + pub(crate) fn admission_closed(&self) -> bool { + self.context.admission_closed.load(Ordering::Acquire) + } + + fn is_stopped(&self) -> bool { + self.context.stopped.load(Ordering::Acquire) + } + pub(crate) async fn ensure_initialized(&self) -> Result<()> { loop { let changed = self.initialization_gate_changed.notified(); @@ -242,7 +449,7 @@ impl ModuleTransport { .initialization_gate .lock() .expect("initialization gate lock"); - if self.is_faulted() { + if self.is_faulted() || self.admission_closed() { return Err(Error::ConnectionClosed); } match allowed { @@ -255,48 +462,300 @@ impl ModuleTransport { } } - let result = self - .init_result - .get_or_init(|| async { - self.context.init_started.store(true, Ordering::Release); - self.context.init_notify.notify_waiters(); - let Some(initializer) = self - .initializer + let initiating = self.schedule_initializer()?; + let mut abandonment = InitiatingWaiter { + transport: self, + armed: initiating, + }; + let wait = async { + loop { + let changed = self.context.physical_notify.notified(); + let result = self + .context + .physical .lock() - .expect("module initializer lock") - .take() - else { - return if self.module.lock().expect("module vtable lock").is_some() { - Ok(()) - } else { - Err("module has no initializer".to_string()) - }; - }; - // Module init is opaque synchronous code. Keep it off Tokio's - // worker threads and bound how long the broker waits; a timed - // out blocking task may remain wedged, so its borrowed config - // remains allocated rather than being invalidated underneath it. - let initialized = tokio::time::timeout( - MODULE_INIT_DEADLINE, - tokio::task::spawn_blocking(move || initializer.run()), - ) - .await; - let module = match initialized { - Ok(Ok(Ok(initialized))) => initialized, - Ok(Ok(Err(reason))) => { - self.clear_config(); - return Err(reason); - } - Ok(Err(_)) => return Err("module initialization panicked".to_string()), - Err(_) => return Err("module initialization exceeded its deadline".to_string()), - }; - self.clear_config(); - self.initialize(module.0) - .map_err(|_| "module returned an invalid vtable".to_string())?; + .expect("physical state lock") + .result + .clone(); + if let Some(result) = result { + return result; + } + changed.await; + } + }; + let result = match tokio::time::timeout(MODULE_INIT_DEADLINE, wait).await { + Ok(result) => { + abandonment.armed = false; + result + } + Err(_) => { + if initiating { + self.abandon_initialization(); + } + abandonment.armed = false; + return Err(Error::transport( + "module initialization exceeded its deadline", + )); + } + }; + match result { + Ok(()) if self.admission_closed() || self.is_faulted() => Err(Error::ConnectionClosed), + Ok(()) => Ok(()), + Err(InitFailure::Closed) => Err(Error::ConnectionClosed), + Err(InitFailure::Failed(reason)) => Err(Error::transport(reason)), + } + } + + fn notify_physical(&self) { + self.context.physical_changed.notify_all(); + self.context.physical_notify.notify_waiters(); + self.context.init_notify.notify_waiters(); + } + + // Scheduling is synchronous: the externally anchored owner precedes submission. + fn schedule_initializer(&self) -> Result { + let gate = self + .initialization_gate + .lock() + .expect("initialization gate lock"); + if *gate != Some(true) || self.is_faulted() || self.admission_closed() { + return Err(Error::ConnectionClosed); + } + let mut state = self.context.physical.lock().expect("physical state lock"); + if state.result.is_some() || state.init != InitState::Deferred { + return Ok(false); + } + let Some(initializer) = self + .initializer + .lock() + .expect("module initializer lock") + .take() + else { + state.result = Some(Err(InitFailure::Failed("module has no initializer".into()))); + return Ok(false); + }; + let job = Arc::new(PhysicalJob { + transport: self.self_ref.upgrade().ok_or(Error::ConnectionClosed)?, + initializer: StdMutex::new(Some(initializer)), + guard: StdMutex::new(None), + deadline: MODULE_INIT_DEADLINE, + fault_on_spawn_failure: true, + submitted: AtomicBool::new(false), + started: AtomicBool::new(false), + }); + state.init = InitState::Scheduled; + state.job = Some(job.clone()); + self.context.init_started.store(true, Ordering::Release); + drop(state); + drop(gate); + self.notify_physical(); + Self::submit_job(job); + Ok(true) + } + + fn submit_job(job: Arc) { + // Submission is spent even if the builder drops its closure. The + // external anchor remains; no caller can silently resubmit this job. + if job.submitted.swap(true, Ordering::AcqRel) { + return; + } + let submitted = job.clone(); + #[cfg(test)] + let rejected = job.transport.reject_worker_spawn.load(Ordering::Acquire); + #[cfg(not(test))] + let rejected = false; + let worker = move || submitted.run(); + let spawn = if rejected { + // Model Builder rejecting and dropping the submitted closure: only + // the independent context anchor can preserve its physical owner. + drop(worker); + Err(std::io::Error::other("controlled worker rejection")) + } else { + std::thread::Builder::new() + .name("tinybus-physical".into()) + .spawn(worker) + }; + if spawn.is_err() { + let transport = &job.transport; + transport.commit_terminal(job.fault_on_spawn_failure); + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); + state.cleanup = CleanupState::Uncertain; + state.result.get_or_insert(Err(InitFailure::Failed( + "module physical worker could not start".into(), + ))); + drop(state); + transport.notify_physical(); + } + } + + fn publish_terminal_flags(&self, fault: bool) { + use crate::module::abi::{TB_NATIVE_TERMINAL_FAULT, TB_NATIVE_TERMINAL_STOP_INTENT}; + if fault { + self.context.faulted.store(true, Ordering::Release); + } + let flags = &self.context.native_terminal_state; + let mut old = flags.load(Ordering::Acquire); + loop { + let new = (old & !crate::module::abi::TB_NATIVE_TERMINAL_LIVE) + | TB_NATIVE_TERMINAL_STOP_INTENT + | if fault { TB_NATIVE_TERMINAL_FAULT } else { 0 }; + match flags.compare_exchange_weak(old, new, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => break, + Err(current) => old = current, + } + } + } + + fn abandon_initialization(&self) { + { + let mut state = self.context.physical.lock().expect("physical state lock"); + // Linearize publication versus the initiating waiter's abandonment. + // Atomic-only cause publication does not invert the host lock order. + if state.result.is_some() + || self.context.native_terminal_state.load(Ordering::Acquire) + != crate::module::abi::TB_NATIVE_TERMINAL_LIVE + { + return; + } + self.publish_terminal_flags(true); + if state.cleanup == CleanupState::Idle { + state.cleanup = CleanupState::Requested; + } + } + self.request_terminal(true); + } + + pub(crate) fn commit_admission_termination(&self, cause: AdmissionTermination) -> bool { + let fault = match cause { + AdmissionTermination::Failed => true, + AdmissionTermination::Canceled => { + let flags = self.context.native_terminal_state.load(Ordering::Acquire); + flags & crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT == 0 + || flags & crate::module::abi::TB_NATIVE_TERMINAL_FAULT != 0 + } + }; + self.commit_terminal(fault); + fault + } + + pub(crate) fn commit_terminal(&self, fault: bool) { + self.publish_terminal_flags(fault); + self.close_admission(); + } + + pub(crate) fn request_terminal(&self, fault: bool) { + self.commit_terminal(fault); + let mut state = self.context.physical.lock().expect("physical state lock"); + if state.cleanup == CleanupState::Idle { + state.cleanup = CleanupState::Requested; + } + if state.job.is_some() || state.cleanup != CleanupState::Requested { + return; + } + if state.init == InitState::Deferred { + state.init = InitState::CanceledBeforeEntry; + state.cleanup = CleanupState::NoCallbackRequired; + state.result = Some(Err(InitFailure::Closed)); + drop(state); + self.initializer + .lock() + .expect("module initializer lock") + .take(); + self.clear_config(); + self.finish_stop(); + self.notify_physical(); + return; + } + let Some(transport) = self.self_ref.upgrade() else { + return; + }; + let job = Arc::new(PhysicalJob { + transport, + initializer: StdMutex::new(None), + guard: StdMutex::new(None), + deadline: MODULE_INIT_DEADLINE, + fault_on_spawn_failure: fault, + submitted: AtomicBool::new(false), + started: AtomicBool::new(false), + }); + state.job = Some(job.clone()); + drop(state); + Self::submit_job(job); + } + + pub(crate) fn eager_initialize( + &self, + init: crate::module::abi::TbModuleInit, + host: TbHostVtable, + ) -> Result<()> { + let guard = self.lifecycle.clone().try_lock_owned().map_err(|_| { + self.commit_terminal(true); + self.context + .physical + .lock() + .expect("physical state lock") + .cleanup = CleanupState::Uncertain; + Error::transport("module initialization lifecycle unavailable") + })?; + self.context + .physical + .lock() + .expect("physical state lock") + .init = InitState::Entered; + self.context.init_started.store(true, Ordering::Release); + let mut module = TbModuleVtable::default(); + let code = unsafe { init(&host, &mut module) }; + let result = if code == TB_OK { + self.initialize(module) + } else { + Err(Error::transport("module initialization failed")) + }; + self.clear_config(); + { + let mut state = self.context.physical.lock().expect("physical state lock"); + state.init = if result.is_ok() { + InitState::ExternalValid + } else { + InitState::PublishedRejected + }; + state.result.get_or_insert(if result.is_ok() { Ok(()) - }) - .await; - result.clone().map_err(Error::transport) + } else { + Err(InitFailure::Failed( + "module returned an invalid vtable".into(), + )) + }); + if result.is_err() { + state.cleanup = CleanupState::Uncertain; + if state.job.is_none() { + state.job = self.self_ref.upgrade().map(|transport| { + Arc::new(PhysicalJob { + transport, + initializer: StdMutex::new(None), + guard: StdMutex::new(None), + deadline: MODULE_INIT_DEADLINE, + fault_on_spawn_failure: true, + submitted: AtomicBool::new(false), + started: AtomicBool::new(false), + }) + }); + } + } + } + self.notify_physical(); + drop(guard); + if result.is_err() { + self.request_terminal(true); + return result; + } + if self.is_faulted() || self.admission_closed() { + return Err(Error::ConnectionClosed); + } + result } pub(crate) async fn wait_ready(&self) { @@ -304,6 +763,7 @@ impl ModuleTransport { let notified = self.context.ready_notify.notified(); if self.context.ready.load(Ordering::Acquire) || self.context.faulted.load(Ordering::Acquire) + || self.admission_closed() { return; } @@ -316,7 +776,7 @@ impl ModuleTransport { } pub(crate) fn is_ready(&self) -> bool { - self.context.ready.load(Ordering::Acquire) && !self.is_faulted() + self.context.ready.load(Ordering::Acquire) && !self.is_faulted() && !self.is_stopped() } pub(crate) fn clear_config(&self) { @@ -327,6 +787,9 @@ impl ModuleTransport { } pub(crate) async fn reinitialize(self: &Arc, bytes: crate::Secret) -> Result<()> { + if self.admission_closed() { + return Err(Error::ConnectionClosed); + } let started = std::time::Instant::now(); let lifecycle = tokio::time::timeout(MODULE_INIT_DEADLINE, self.lifecycle.clone().lock_owned()) @@ -336,6 +799,9 @@ impl ModuleTransport { .expect("static member name is valid"), timeout_ms: MODULE_INIT_DEADLINE.as_millis() as u64, })?; + if self.admission_closed() || self.is_faulted() { + return Err(Error::ConnectionClosed); + } let module = self .module .lock() @@ -396,7 +862,7 @@ impl ModuleTransport { pub(crate) async fn wait_initializing(&self) { loop { let notified = self.context.init_notify.notified(); - if self.init_started() || self.is_faulted() { + if self.init_started() || self.is_faulted() || self.admission_closed() { return; } notified.await; @@ -404,7 +870,7 @@ impl ModuleTransport { } async fn deliver_now(&self, message: Message) -> Result<()> { - if self.context.faulted.load(Ordering::Acquire) { + if self.is_faulted() || self.admission_closed() { return Err(Error::ConnectionClosed); } let bytes = serde_json::to_vec(&message)?; @@ -414,14 +880,24 @@ impl ModuleTransport { loop { let notified = self.context.wake.notified(); + if self.is_faulted() || self.admission_closed() { + return Err(Error::ConnectionClosed); + } let module = *self.module.lock().expect("module vtable lock"); let Some(module) = module else { return Err(Error::ConnectionClosed); }; + let lease = self.callbacks.enter()?; + // Stop closes this gate and drains reservations before teardown; + // a caller disappearing cannot refund a running native callback. let code = unsafe { (module.deliver)(module.module_ctx, bytes.as_ptr(), bytes.len()) }; + drop(lease); match code { TB_OK => return Ok(()), TB_BACKPRESSURE => { + if self.admission_closed() { + return Err(Error::ConnectionClosed); + } // A module that stops draining its queue and never calls // `wake` must not park this delivery task — or the caller // behind it — for the process lifetime. Expiry faults the @@ -431,7 +907,13 @@ impl ModuleTransport { .await .is_err() { - self.context.faulted.store(true, Ordering::Release); + let _pending = self.pending.lock().expect("module pending lock"); + if self.admission_closed() { + return Err(Error::ConnectionClosed); + } + self.publish_terminal_flags(true); + drop(_pending); + self.request_terminal(true); self.context.ready_notify.notify_waiters(); self.context .inbound @@ -453,17 +935,22 @@ impl ModuleTransport { async fn drain_pending(self: Arc) { self.wait_ready().await; loop { - while !self.is_faulted() { - let message = self.pending.lock().await.pop_front(); + while !self.is_faulted() && !self.admission_closed() { + let message = self + .pending + .lock() + .expect("module pending lock") + .pop_front(); let Some(message) = message else { break; }; - if self.deliver_now(message.clone()).await.is_err() { - self.fault_pending(message).await; + if let Err(error) = self.deliver_now(message.clone()).await { + self.fault_pending(message, error).await; + self.drain_started.store(false, Ordering::Release); return; } } - if self.is_faulted() { + if self.is_faulted() || self.admission_closed() { self.drain_started.store(false, Ordering::Release); return; } @@ -474,7 +961,7 @@ impl ModuleTransport { // genuinely empty; the lock makes the check-and-clear atomic with a // concurrent push. let restart = { - let pending = self.pending.lock().await; + let pending = self.pending.lock().expect("module pending lock"); let empty = pending.is_empty(); if empty { self.drain_started.store(false, Ordering::Release); @@ -491,9 +978,14 @@ impl ModuleTransport { /// Answer every still-queued call — and the one that just failed to /// deliver — with `ModuleUnavailable`, then fault the module, so callers /// learn the delivery failed instead of waiting out their deadline. - async fn fault_pending(&self, failed: Message) { + async fn fault_pending(&self, failed: Message, error: Error) { let cleared = { - let mut pending = self.pending.lock().await; + let mut pending = self.pending.lock().expect("module pending lock"); + if self.admission_closed() && matches!(error, Error::ConnectionClosed) { + pending.clear(); + return; + } + self.publish_terminal_flags(true); std::mem::take(&mut *pending) }; // Each of these calls incremented `inflight` when it was queued. The @@ -520,7 +1012,7 @@ impl ModuleTransport { } } } - self.context.faulted.store(true, Ordering::Release); + self.request_terminal(true); self.context.ready_notify.notify_waiters(); self.context .inbound @@ -540,8 +1032,22 @@ impl ModuleTransport { } pub(crate) fn stop_sync(&self, deadline: Duration) -> i32 { - self.terminate_waiters(); - let code = self.shutdown_sync(deadline); + if self.is_stopped() { + return TB_CLOSED; + } + let started = std::time::Instant::now(); + self.close_admission(); + // This is the retained physical worker, not the caller's timeout. Never + // tear down code underneath an admitted callback or abort its lease. + self.callbacks.wait_drained(); + let code = self.shutdown_sync(deadline.saturating_sub(started.elapsed())); + if code == TB_OK { + self.context.native_terminal_state.fetch_or( + crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + Ordering::AcqRel, + ); + } + self.finish_stop(); self.context .inbound .lock() @@ -550,6 +1056,11 @@ impl ModuleTransport { code } + #[cfg(test)] + pub(crate) fn lifecycle_is_owned_for_test(&self) -> bool { + self.lifecycle.try_lock().is_err() + } + pub(crate) fn stop_task_spawned(&self) -> bool { self.stop_task_spawned.load(Ordering::Acquire) } @@ -558,7 +1069,9 @@ impl ModuleTransport { self: Arc, deadline: Duration, ) -> std::result::Result { - self.stop_task_spawned.store(false, Ordering::Release); + if self.is_stopped() { + return Err(StopError::NotStarted(Error::ConnectionClosed)); + } let started = std::time::Instant::now(); let lifecycle = tokio::time::timeout(deadline, self.lifecycle.clone().lock_owned()) .await @@ -567,23 +1080,287 @@ impl ModuleTransport { "module lifecycle operation exceeded its deadline", )) })?; + if self.is_stopped() { + return Err(StopError::NotStarted(Error::ConnectionClosed)); + } let remaining = deadline.saturating_sub(started.elapsed()); - self.stop_task_spawned.store(true, Ordering::Release); - let pending = NativeLifecyclePending::begin(&self.native_lifecycle_pending); - let task = tokio::task::spawn_blocking(move || { - let _lifecycle = lifecycle; - let _pending = pending; - self.stop_sync(remaining) - }); - tokio::time::timeout(remaining, task) + self.commit_terminal(false); + let (_existing, job, mut acquired, ordinary_handoff) = { + let mut state = self.context.physical.lock().expect("physical state lock"); + if state.cleanup == CleanupState::Idle { + state.cleanup = CleanupState::Requested; + } + let existing = state.job.is_some(); + let mut acquired = Some(lifecycle); + let job = if existing || state.cleanup != CleanupState::Requested { + None + } else { + let job = Arc::new(PhysicalJob { + transport: self.clone(), + initializer: StdMutex::new(None), + guard: StdMutex::new(acquired.take()), + deadline: remaining, + fault_on_spawn_failure: false, + submitted: AtomicBool::new(false), + started: AtomicBool::new(false), + }); + state.job = Some(job.clone()); + Some(job) + }; + let ordinary_handoff = !self.is_faulted() + && state.cleanup == CleanupState::Requested + && (job.is_some() || (existing && state.init == InitState::Scheduled)); + if ordinary_handoff { + self.stop_task_spawned.store(true, Ordering::Release); + } + (existing, job, acquired, ordinary_handoff) + }; + // A Scheduled initializer must acquire the guard itself to prove no C entry. + drop(acquired.take()); + if let Some(job) = job { + Self::submit_job(job); + } + let classify = |error| { + if ordinary_handoff { + StopError::Started(error) + } else { + StopError::NotStarted(error) + } + }; + let wait = async { + loop { + let changed = self.context.physical_notify.notified(); + let outcome = { + let state = self.context.physical.lock().expect("physical state lock"); + match state.cleanup { + CleanupState::Complete | CleanupState::NoCallbackRequired => { + Some(Ok(state.code.unwrap_or(TB_CLOSED))) + } + CleanupState::Uncertain => Some(match state.code { + Some(code) => Ok(code), + None => Err(classify(Error::transport("module cleanup is uncertain"))), + }), + _ => None, + } + }; + if let Some(outcome) = outcome { + return outcome; + } + changed.await; + } + }; + tokio::time::timeout(remaining, wait) .await - .map_err(|_| { - StopError::Started(Error::failed("module shutdown exceeded its deadline")) - })? - .map_err(|_| StopError::Started(Error::failed("module shutdown task was cancelled"))) + .map_err(|_| classify(Error::failed("module shutdown exceeded its deadline")))? } } +impl PhysicalJob { + fn run(self: Arc) { + self.started.store(true, Ordering::Release); + let transport = &self.transport; + #[cfg(test)] + { + let hook = transport + .before_worker_lock + .lock() + .expect("worker test hook lock") + .take(); + if let Some(hook) = hook { + hook(); + } + } + let initializer = self + .initializer + .lock() + .expect("physical initializer lock") + .take(); + if let Some(initializer) = initializer { + let guard = transport.lifecycle.clone().blocking_lock_owned(); + let entered = { + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); + if transport.admission_closed() || transport.is_faulted() { + state.init = InitState::CanceledBeforeEntry; + state.cleanup = CleanupState::NoCallbackRequired; + state.result = Some(Err(InitFailure::Closed)); + state.code = Some(TB_CLOSED); + state.job = None; + false + } else { + state.init = InitState::Entered; + true + } + }; + if !entered { + drop(initializer); + transport.clear_config(); + transport.finish_stop(); + transport.notify_physical(); + return; + } + let output = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| initializer.run())); + let result = match output { + Ok(Ok(module)) => transport + .initialize(module.0) + .map_err(|_| InitFailure::Failed("module returned an invalid vtable".into())), + Ok(Err(reason)) => Err(InitFailure::Failed(reason)), + Err(_) => Err(InitFailure::Failed("module initialization panicked".into())), + }; + transport.clear_config(); + #[cfg(test)] + { + let hook = transport + .before_init_publication + .lock() + .expect("publication test hook lock") + .take(); + if let Some(hook) = hook { + hook(); + } + } + if result.is_err() { + transport.commit_terminal(true); + } + let cleanup = { + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); + state.init = if result.is_ok() { + InitState::PublishedValid + } else { + InitState::PublishedRejected + }; + state.result.get_or_insert(result.clone()); + if result.is_err() { + state.cleanup = CleanupState::Uncertain; + } + if state.cleanup == CleanupState::Idle { + state.job = None; + } + state.cleanup == CleanupState::Requested + }; + transport.notify_physical(); + drop(guard); + if !cleanup { + return; + } + } else { + // Eager entry publishes before releasing its exclusion. Never hold + // lifecycle while waiting for an initializer which needs that lock. + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); + while matches!(state.init, InitState::Scheduled | InitState::Entered) + && state.result.is_none() + { + state = transport + .context + .physical_changed + .wait(state) + .expect("physical state lock"); + } + } + let guard = self + .guard + .lock() + .expect("physical guard lock") + .take() + .unwrap_or_else(|| transport.lifecycle.clone().blocking_lock_owned()); + { + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); + if state.cleanup != CleanupState::Requested { + return; + } + if state.init == InitState::Deferred || state.init == InitState::CanceledBeforeEntry { + state.init = InitState::CanceledBeforeEntry; + state.cleanup = CleanupState::NoCallbackRequired; + state.result = Some(Err(InitFailure::Closed)); + state.code = Some(TB_CLOSED); + state.job = None; + drop(state); + transport + .initializer + .lock() + .expect("module initializer lock") + .take(); + transport.clear_config(); + transport.finish_stop(); + transport.notify_physical(); + return; + } + if transport + .module + .lock() + .expect("module vtable lock") + .is_none() + { + state.cleanup = CleanupState::Uncertain; + drop(state); + transport.notify_physical(); + return; + } + state.cleanup = CleanupState::Executing; + } + let pending = NativeLifecyclePending::begin(&transport.native_lifecycle_pending); + let code = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + transport.stop_sync(self.deadline) + })); + // Completion is observable before this worker releases its lifecycle + // guard. Clear native activity first so a completed stop cannot leave + // enable/disable reporting that opaque work is still running. + drop(pending); + let mut state = transport + .context + .physical + .lock() + .expect("physical state lock"); + state.code = code.as_ref().ok().copied(); + state.cleanup = if matches!(code, Ok(TB_OK)) { + CleanupState::Complete + } else { + CleanupState::Uncertain + }; + if state.cleanup == CleanupState::Complete { + state.job = None; + } + drop(state); + transport.notify_physical(); + #[cfg(test)] + { + let hook = transport + .after_cleanup_publication + .lock() + .expect("cleanup publication test hook lock") + .take(); + if let Some(hook) = hook { + hook(); + } + } + drop(guard); + } +} + +unsafe extern "C" fn host_native_terminal_state(ctx: *mut c_void) -> u32 { + unsafe { ctx.cast::().as_ref() }.map_or( + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT + | crate::module::abi::TB_NATIVE_TERMINAL_FAULT, + |context| context.native_terminal_state.load(Ordering::Acquire), + ) +} + #[derive(Debug)] pub(crate) enum StopError { NotStarted(Error), @@ -601,9 +1378,18 @@ impl From for Error { #[async_trait] impl Transport for ModuleTransport { async fn send(&self, message: Message) -> Result<()> { - if self.ensure_initialized().await.is_err() { - self.context.init_failed.store(true, Ordering::Release); - self.context.faulted.store(true, Ordering::Release); + if self.admission_closed() { + return Err(Error::ConnectionClosed); + } + if let Err(error) = self.ensure_initialized().await { + { + let _pending = self.pending.lock().expect("module pending lock"); + if self.admission_closed() && matches!(error, Error::ConnectionClosed) { + return Err(Error::ConnectionClosed); + } + self.context.init_failed.store(true, Ordering::Release); + self.publish_terminal_flags(true); + } self.context.ready_notify.notify_waiters(); if message.header.kind == MessageKind::MethodCall { let error = Error::ModuleUnavailable { @@ -623,21 +1409,29 @@ impl Transport for ModuleTransport { let _ = sender.try_send(bytes); } } + self.request_terminal(true); return Ok(()); } + self.request_terminal(true); return Err(Error::ConnectionClosed); } let is_call = message.header.kind == MessageKind::MethodCall; - if is_call { - self.context.inflight.fetch_add(1, Ordering::AcqRel); - } - if message.header.kind == MessageKind::MethodCall && !self.is_ready() { - self.pending.lock().await.push_back(message); - if !self.drain_started.swap(true, Ordering::AcqRel) { - let transport = self.self_ref.upgrade().ok_or(Error::ConnectionClosed)?; - tokio::spawn(transport.drain_pending()); + { + let mut pending = self.pending.lock().expect("module pending lock"); + if self.admission_closed() { + return Err(Error::ConnectionClosed); + } + if is_call { + self.context.inflight.fetch_add(1, Ordering::AcqRel); + } + if is_call && !self.is_ready() { + pending.push_back(message); + if !self.drain_started.swap(true, Ordering::AcqRel) { + let transport = self.self_ref.upgrade().ok_or(Error::ConnectionClosed)?; + tokio::spawn(transport.drain_pending()); + } + return Ok(()); } - return Ok(()); } let result = self.deliver_now(message).await; if result.is_err() && is_call { @@ -754,9 +1548,22 @@ unsafe extern "C" fn host_fault(ctx: *mut c_void, _: *const u8, _: usize) { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { if let Some(context) = unsafe { ctx.cast::().as_ref() } { context.faulted.store(true, Ordering::Release); + context.native_terminal_state.fetch_or( + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT + | crate::module::abi::TB_NATIVE_TERMINAL_FAULT, + Ordering::AcqRel, + ); + context.native_terminal_state.fetch_and( + !crate::module::abi::TB_NATIVE_TERMINAL_LIVE, + Ordering::AcqRel, + ); context.init_notify.notify_waiters(); context.ready_notify.notify_waiters(); context.inbound.lock().expect("host inbound lock").take(); + let owner = context.owner.lock().expect("physical owner lock").upgrade(); + if let Some(owner) = owner { + owner.request_terminal(true); + } } })); } @@ -771,12 +1578,15 @@ unsafe extern "C" fn host_broker_routing(ctx: *mut c_void) -> u32 { unsafe extern "C" fn host_ready(ctx: *mut c_void) { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { if let Some(context) = unsafe { ctx.cast::().as_ref() } { - context.ready.store(true, Ordering::Release); - context.ready_notify.notify_waiters(); + let _ready = context.ready_gate.lock().expect("module ready lock"); + if !context.admission_closed.load(Ordering::Acquire) { + context.ready.store(true, Ordering::Release); + context.ready_notify.notify_waiters(); + } } })); } #[cfg(test)] #[path = "transport_tests.rs"] -mod tests; +pub(super) mod tests; diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index cee6a353..b7c67963 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -5,6 +5,7 @@ use crate::{BusName, InterfaceName, MemberName, ObjectPath}; static DELIVERY_CODE: AtomicI32 = AtomicI32::new(TB_OK); static DELIVERIES: AtomicUsize = AtomicUsize::new(0); +static DELIVERY_BLOCKED: AtomicBool = AtomicBool::new(false); static SHUTDOWN_CODE: AtomicI32 = AtomicI32::new(TB_OK); static SHUTDOWN_CALLS: AtomicUsize = AtomicUsize::new(0); static SHUTDOWN_BLOCKED: AtomicBool = AtomicBool::new(false); @@ -16,6 +17,9 @@ static VTABLE_TEST_LOCK: std::sync::OnceLock> = std::sync unsafe extern "C" fn deliver(_: *mut c_void, _: *const u8, _: usize) -> i32 { DELIVERIES.fetch_add(1, Ordering::AcqRel); + while DELIVERY_BLOCKED.load(Ordering::Acquire) { + std::thread::yield_now(); + } DELIVERY_CODE.load(Ordering::Acquire) } @@ -270,6 +274,7 @@ async fn a_shutdown_task_timeout_is_distinguished_from_a_lifecycle_lock_timeout( )); assert!(SHUTDOWN_ENTERED.load(Ordering::Acquire)); SHUTDOWN_BLOCKED.store(false, Ordering::Release); + wait_physical_cleanup(&transport).await; } #[tokio::test] @@ -498,6 +503,7 @@ async fn delivery_maps_a_faulted_transport_a_missing_module_and_module_refusals( transport.deliver_now(call()).await, Err(Error::ConnectionClosed) )); + wait_physical_cleanup(&transport).await; } #[test] @@ -591,3 +597,1581 @@ async fn ready_followed_by_fault_is_not_successful_readiness() { assert!(transport.is_faulted()); assert!(!transport.is_ready()); } + +// Release opaque callbacks even when a regression assertion fails. +struct ShutdownRelease; +impl Drop for ShutdownRelease { + fn drop(&mut self) { + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + DELIVERY_BLOCKED.store(false, Ordering::Release); + DELIVERY_CODE.store(TB_OK, Ordering::Release); + } +} + +async fn shutdown_entry() { + tokio::time::timeout(Duration::from_secs(2), async { + while !SHUTDOWN_ENTERED.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("S00"); +} + +#[tokio::test] +async fn ordinary_shutdown_deadline_preserves_state_until_callback_returns() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("S01"); + unsafe { + (host.ready)(host.host_ctx); + } + let stopping = tokio::spawn(transport.clone().stop(Duration::from_millis(100))); + shutdown_entry().await; + assert!( + matches!(stopping.await.expect("S02"), Err(StopError::Started(_))), + "S03" + ); + assert!(!transport.is_faulted(), "S04"); + assert!(transport.is_ready(), "S05"); + assert!( + matches!( + transport.clone().stop(Duration::from_millis(10)).await, + Err(StopError::NotStarted(_)) + ), + "S06" + ); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "S07"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()) + .await + .expect("S08"); + assert!(!transport.is_faulted(), "S09"); + unsafe { + (host.ready)(host.host_ctx); + } + assert!(!transport.is_ready(), "S10"); + assert!( + matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), + "S11" + ); + assert!(!transport.init_failed(), "S12"); + assert!( + matches!( + transport.clone().stop(Duration::from_secs(1)).await, + Err(StopError::NotStarted(Error::ConnectionClosed)) + ), + "S68" + ); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "S69"); + assert!( + matches!( + transport + .reinitialize(crate::Secret::new(b"{}".to_vec())) + .await, + Err(Error::ConnectionClosed) + ), + "S70" + ); +} + +#[tokio::test] +async fn cancelling_a_started_shutdown_keeps_physical_exclusion_without_faulting() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("cancel-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("S13"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + shutdown_entry().await; + stopping.abort(); + assert!(stopping.await.expect_err("S14").is_cancelled(), "S15"); + assert!(!transport.is_faulted(), "S16"); + assert!( + matches!( + transport.clone().stop(Duration::from_millis(10)).await, + Err(StopError::NotStarted(_)) + ), + "S17" + ); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "S18"); + unsafe { + (host.fault)(host.host_ctx, std::ptr::null(), 0); + } + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()) + .await + .expect("S19"); + unsafe { + (host.ready)(host.host_ctx); + } + assert!(transport.is_faulted(), "S20"); + assert!(!transport.is_ready(), "S21"); +} + +#[tokio::test] +async fn stopping_initialized_module_before_ready_releases_observers_and_queued_calls() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let (transport, host) = ModuleTransport::new("queued-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.send(call()).await.expect("S22"); + assert!(transport.init_started(), "S23"); + transport.stop_sync(Duration::ZERO); + tokio::time::timeout(Duration::from_secs(2), transport.wait_ready()) + .await + .expect("S24"); + tokio::time::timeout(Duration::from_secs(2), async { + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("S25"); + assert!(transport.pending.lock().expect("S36").is_empty(), "S37"); + assert!(!transport.is_faulted(), "S26"); + assert!(!transport.is_ready(), "S27"); + assert!( + matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), + "S28" + ); + assert!(!transport.init_failed(), "S29"); + let weak = Arc::downgrade(&transport); + drop(transport); + // drain_started and init publication precede the actual task's Arc drop. + // Observe reclamation itself instead of treating either flag as task exit. + tokio::time::timeout(Duration::from_secs(2), async { + while weak.strong_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("S30"); +} + +#[tokio::test] +async fn ordinary_stop_wakes_backpressure_without_turning_closure_into_a_fault() { + let _release = ShutdownRelease; + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + DELIVERY_CODE.store(TB_BACKPRESSURE, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("pressure-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.send(call()).await.expect("S31"); + unsafe { + (host.ready)(host.host_ctx); + } + tokio::time::timeout(Duration::from_secs(2), async { + while DELIVERIES.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("S32"); + transport.stop_sync(Duration::ZERO); + // An explicit legacy wake also must not cause delivery to re-enter disposed code. + unsafe { + (host.wake)(host.host_ctx); + } + tokio::time::timeout(Duration::from_secs(2), async { + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("S33"); + assert!(!transport.is_faulted(), "S34"); + assert_eq!(DELIVERIES.load(Ordering::Acquire), 1, "S35"); + DELIVERY_CODE.store(TB_OK, Ordering::Release); +} + +#[tokio::test] +async fn a_send_waiting_for_initialization_admission_closes_without_an_initializer_fault() { + let (transport, host) = ModuleTransport::new("admission-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.pause_initialization(); + let sending_transport = transport.clone(); + let sending = tokio::spawn(async move { sending_transport.send(call()).await }); + tokio::task::yield_now().await; + transport.stop_sync(Duration::ZERO); + assert!( + matches!( + tokio::time::timeout(Duration::from_secs(2), sending) + .await + .expect("S38") + .expect("S39"), + Err(Error::ConnectionClosed) + ), + "S40" + ); + assert!(!transport.init_started(), "S41"); + assert!(!transport.init_failed(), "S42"); + assert!(!transport.is_faulted(), "S43"); + assert!(transport.pending.lock().expect("S44").is_empty(), "S45"); + assert!(!transport.drain_started.load(Ordering::Acquire), "S46"); +} + +struct InitializerRelease(Option>); +impl Drop for InitializerRelease { + fn drop(&mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } +} + +#[tokio::test] +async fn an_actual_initializer_failure_during_stop_is_retained() { + let (transport, host) = ModuleTransport::new("failed-stop".into(), vec![]); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let release = InitializerRelease(Some(sender)); + let (entered, observed) = tokio::sync::oneshot::channel(); + transport.defer_initializer(host, move |_| { + let _ = entered.send(()); + let _ = receiver.recv(); + Err("module initialization failed".into()) + }); + let sending_transport = transport.clone(); + let sending = tokio::spawn(async move { sending_transport.send(call()).await }); + tokio::time::timeout(Duration::from_secs(2), observed) + .await + .expect("S47") + .expect("S48"); + transport.stop_sync(Duration::ZERO); + drop(release); + tokio::time::timeout(Duration::from_secs(2), sending) + .await + .expect("S49") + .expect("S50") + .expect("S51"); + assert!(transport.init_failed(), "S52"); + assert!(transport.is_faulted(), "S53"); + assert!(!transport.is_ready(), "S54"); +} + +#[tokio::test] +async fn concurrent_pending_sends_cannot_resurrect_a_completed_transport_drainer() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let (transport, host) = ModuleTransport::new("racing-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("S55"); + let barrier = Arc::new(tokio::sync::Barrier::new(17)); + let mut sending = Vec::new(); + for _ in 0..16 { + let sender = transport.clone(); + let barrier = barrier.clone(); + sending.push(tokio::spawn(async move { + barrier.wait().await; + sender.send(call()).await + })); + } + barrier.wait().await; + transport.stop_sync(Duration::ZERO); + for sending in sending { + assert!( + matches!( + sending.await.expect("S56"), + Ok(()) | Err(Error::ConnectionClosed) + ), + "S57" + ); + } + tokio::time::timeout(Duration::from_secs(2), async { + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("S58"); + assert!(transport.pending.lock().expect("S59").is_empty(), "S60"); + assert!(!transport.is_faulted(), "S61"); + assert!(!transport.init_failed(), "S62"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_actual_delivery_failure_overlapping_stop_is_retained() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + DELIVERY_CODE.store(TB_BAD_ARGUMENT, Ordering::Release); + DELIVERY_BLOCKED.store(true, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("delivery-failed-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.send(call()).await.expect("S63"); + unsafe { + (host.ready)(host.host_ctx); + } + tokio::time::timeout(Duration::from_secs(2), async { + while DELIVERIES.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("S64"); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + tokio::time::timeout(Duration::from_secs(2), async { + while !transport.admission_closed() { + tokio::task::yield_now().await; + } + }) + .await + .expect("A57"); + assert!(!SHUTDOWN_ENTERED.load(Ordering::Acquire), "A58"); + DELIVERY_BLOCKED.store(false, Ordering::Release); + stopping.await.expect("A59").expect("A60"); + tokio::time::timeout(Duration::from_secs(2), async { + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("S65"); + assert!(transport.is_faulted(), "S66"); + assert!(!transport.is_ready(), "S67"); +} + +#[tokio::test] +async fn lifecycle_waiters_do_not_reenter_a_transport_disposed_by_the_prior_shutdown() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("lifecycle-stop".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("S71"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_millis(100))); + shutdown_entry().await; + assert!( + matches!(stopping.await.expect("S72"), Err(StopError::Started(_))), + "S73" + ); + let mut retry = std::pin::pin!(transport.clone().stop(Duration::from_secs(2))); + let mut reinitializing = + std::pin::pin!(transport.reinitialize(crate::Secret::new(b"{}".to_vec()))); + let mut context = std::task::Context::from_waker(std::task::Waker::noop()); + assert!( + std::future::Future::poll(retry.as_mut(), &mut context).is_pending(), + "S74" + ); + assert!( + matches!( + std::future::Future::poll(reinitializing.as_mut(), &mut context), + std::task::Poll::Ready(Err(Error::ConnectionClosed)) + ), + "S75" + ); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + assert!( + matches!( + retry.await, + Err(StopError::NotStarted(Error::ConnectionClosed)) + ), + "S76" + ); + + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "S78"); + assert!(transport.stop_task_spawned(), "S79"); + assert!(!transport.is_faulted(), "S80"); +} + +#[tokio::test] +async fn fresh_signal_and_call_cannot_enter_an_actual_running_shutdown() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("admission".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("A01"); + unsafe { + (host.ready)(host.host_ctx); + } + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + shutdown_entry().await; + let mut signal = call(); + signal.header.kind = MessageKind::Signal; + let signal_result = transport.send(signal).await; + let call_result = transport.send(call()).await; + assert!(matches!(signal_result, Err(Error::ConnectionClosed)), "A02"); + assert!(matches!(call_result, Err(Error::ConnectionClosed)), "A03"); + assert_eq!(DELIVERIES.load(Ordering::Acquire), 0, "A04"); + assert!(!transport.is_faulted(), "A05"); + assert!(!transport.init_failed(), "A06"); + assert!(!transport.is_stopped(), "A07"); + assert!(transport.is_ready(), "A08"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + assert_eq!(stopping.await.expect("A09").expect("A10"), TB_OK, "A11"); +} + +#[tokio::test] +async fn queued_readiness_and_backpressure_retire_before_actual_shutdown_returns() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("queued-admission".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.send(call()).await.expect("A12"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + shutdown_entry().await; + tokio::time::timeout(Duration::from_millis(100), transport.wait_ready()) + .await + .expect("A13"); + unsafe { + (host.ready)(host.host_ctx); + (host.wake)(host.host_ctx); + } + tokio::time::timeout(Duration::from_millis(100), async { + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("A14"); + let empty = transport.pending.lock().expect("A15").is_empty(); + assert!(empty, "A16"); + assert_eq!(DELIVERIES.load(Ordering::Acquire), 0, "A17"); + assert!(!transport.is_ready(), "A18"); + assert!(!transport.is_faulted(), "A19"); + assert!(!transport.is_stopped(), "A20"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + stopping.await.expect("A21").expect("A22"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_signal_callback_is_drained_despite_stop_timeout() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + DELIVERY_BLOCKED.store(true, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("callback-drain".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("A23"); + let mut signal = call(); + signal.header.kind = MessageKind::Signal; + let sending_transport = transport.clone(); + let sending = tokio::spawn(async move { sending_transport.send(signal).await }); + tokio::time::timeout(Duration::from_secs(2), async { + while DELIVERIES.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("A24"); + assert_eq!(transport.inflight(), 0, "A25"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_millis(50))); + assert!( + matches!(stopping.await.expect("A26"), Err(StopError::Started(_))), + "A27" + ); + assert!(!SHUTDOWN_ENTERED.load(Ordering::Acquire), "A28"); + assert!(!transport.is_stopped(), "A29"); + assert!(transport.stop_task_spawned(), "A30"); + assert!( + matches!( + transport.clone().stop(Duration::from_millis(10)).await, + Err(StopError::NotStarted(_)) + ), + "A31" + ); + assert!( + matches!(transport.send(call()).await, Err(Error::ConnectionClosed)), + "A32" + ); + DELIVERY_BLOCKED.store(false, Ordering::Release); + sending.await.expect("A33").expect("A34"); + shutdown_entry().await; + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "A35"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()) + .await + .expect("A36"); + assert!(transport.is_stopped(), "A37"); + assert!(!transport.is_faulted(), "A38"); +} + +#[tokio::test] +async fn installed_vtable_without_init_flag_closes_fresh_admission_before_teardown() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("installed-admission".into(), vec![]); + let mut module = TbModuleVtable::default(); + unsafe { + initialize_ok(&host, &mut module); + } + transport.initialize(module).expect("A39"); + assert!(!transport.init_started(), "A40"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + shutdown_entry().await; + let result = transport.send(call()).await; + assert!(matches!(result, Err(Error::ConnectionClosed)), "A41"); + assert!(!transport.init_started(), "A42"); + assert!(!transport.is_stopped(), "A43"); + assert!(!transport.is_faulted(), "A44"); + assert!(!transport.init_failed(), "A45"); + assert_eq!(DELIVERIES.load(Ordering::Acquire), 0, "A46"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + stopping.await.expect("A47").expect("A48"); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "A49"); +} + +#[tokio::test] +async fn backpressure_retry_closes_without_fault_while_actual_shutdown_is_blocked() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + DELIVERY_CODE.store(TB_BACKPRESSURE, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("pressure-admission".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.send(call()).await.expect("A50"); + unsafe { + (host.ready)(host.host_ctx); + } + tokio::time::timeout(Duration::from_secs(2), async { + while DELIVERIES.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("A51"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + shutdown_entry().await; + tokio::time::timeout(Duration::from_millis(100), async { + while transport.drain_started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("A52"); + unsafe { + (host.wake)(host.host_ctx); + } + assert_eq!(DELIVERIES.load(Ordering::Acquire), 1, "A53"); + assert!(!transport.is_faulted(), "A54"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + stopping.await.expect("A55").expect("A56"); +} + +#[tokio::test] +async fn a_reserved_native_entry_cannot_start_after_teardown_even_if_stop_caller_is_cancelled() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(true, Ordering::Release); + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + DELIVERIES.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("reserved-admission".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("A61"); + // Pause at the real check-to-native-entry boundary: the reservation already + // belongs to the callback caller, but native entry has not happened yet. + let lease = transport.callbacks.enter().expect("A62"); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + tokio::time::timeout(Duration::from_secs(2), async { + while !transport.admission_closed() { + tokio::task::yield_now().await; + } + }) + .await + .expect("A63"); + stopping.abort(); + assert!(stopping.await.expect_err("A64").is_cancelled(), "A65"); + assert!(!SHUTDOWN_ENTERED.load(Ordering::Acquire), "A66"); + assert!(!transport.is_stopped(), "A67"); + assert!(transport.callbacks.enter().is_err(), "A68"); + assert!( + matches!( + transport.ensure_initialized().await, + Err(Error::ConnectionClosed) + ), + "A69" + ); + assert!( + matches!( + transport.reinitialize(crate::Secret::new(vec![])).await, + Err(Error::ConnectionClosed) + ), + "A70" + ); + assert!( + matches!( + transport.clone().stop(Duration::from_millis(10)).await, + Err(StopError::NotStarted(_)) + ), + "A71" + ); + // An admitted callback can still enter before teardown; shutdown must wait + // for its physical exit, even after the outer stop future disappears. + let module = *transport.module.lock().expect("A72"); + let module = module.expect("A73"); + let bytes = serde_json::to_vec(&call()).expect("A74"); + assert_eq!( + unsafe { (module.deliver)(module.module_ctx, bytes.as_ptr(), bytes.len()) }, + TB_OK, + "A75" + ); + assert!(!SHUTDOWN_ENTERED.load(Ordering::Acquire), "A76"); + drop(lease); + shutdown_entry().await; + assert_eq!(DELIVERIES.load(Ordering::Acquire), 1, "A77"); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1, "A78"); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let _completed = tokio::time::timeout(Duration::from_secs(2), transport.lifecycle.lock()) + .await + .expect("A79"); + assert!(transport.is_stopped(), "A80"); + assert!(!transport.is_faulted(), "A81"); +} + +#[tokio::test] +async fn unanswered_method_calls_are_not_physical_callback_leases() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let _release = ShutdownRelease; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let (transport, host) = ModuleTransport::new("unanswered".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.expect("A82"); + unsafe { + (host.ready)(host.host_ctx); + } + transport.send(call()).await.expect("A83"); + assert_eq!(transport.inflight(), 1, "A84"); + assert_eq!( + transport + .clone() + .stop(Duration::from_secs(2)) + .await + .expect("A85"), + TB_OK, + "A86" + ); + assert_eq!(transport.inflight(), 1, "A87"); + assert!(transport.is_stopped(), "A88"); +} + +#[tokio::test] +async fn abandoning_initiating_waiter_faults_and_retains_late_initializer_cleanup() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + SHUTDOWN_CODE.store(TB_OK, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("abandoned-init".into(), b"secret".to_vec()); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + transport.defer_initializer(host, move |host| { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + assert_eq!( + unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }, + b"secret" + ); + let mut module = TbModuleVtable::default(); + assert_eq!(unsafe { initialize_ok(&host, &mut module) }, TB_OK); + Ok(module) + }); + let waiter = tokio::spawn({ + let transport = transport.clone(); + async move { transport.ensure_initialized().await } + }); + entered_rx.await.unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + // Always release foreign work even on the unfixed implementation. + let faulted = transport.is_faulted(); + release_tx.send(()).unwrap(); + assert!( + faulted, + "initiating waiter abandonment must publish fault immediately" + ); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if transport.is_stopped() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 1); + assert!(transport.ensure_initialized().await.is_err()); +} + +#[tokio::test] +async fn scheduled_initializer_fault_cancels_before_actual_entry() { + let (transport, host) = ModuleTransport::new("scheduled-fault".into(), vec![]); + let entries = Arc::new(AtomicUsize::new(0)); + transport.defer_initializer(host, { + let entries = entries.clone(); + move |_| { + entries.fetch_add(1, Ordering::AcqRel); + Err("unexpected C entry".into()) + } + }); + let guard = transport.lifecycle.clone().lock_owned().await; + assert!(transport.schedule_initializer().unwrap()); + transport.request_terminal(true); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + assert_eq!( + transport.context.physical.lock().unwrap().init, + InitState::Scheduled + ); + drop(guard); + tokio::time::timeout(Duration::from_secs(2), async { + while !transport.is_stopped() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(entries.load(Ordering::Acquire), 0); + assert_eq!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::NoCallbackRequired + ); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); +} + +#[tokio::test] +async fn scheduled_initializer_is_adopted_by_original_ordinary_stop() { + let (transport, host) = ModuleTransport::new("scheduled-stop".into(), vec![]); + let entries = Arc::new(AtomicUsize::new(0)); + transport.defer_initializer(host, { + let entries = entries.clone(); + move |_| { + entries.fetch_add(1, Ordering::AcqRel); + Err("unexpected C entry".into()) + } + }); + let (worker_started, worker_started_rx) = tokio::sync::oneshot::channel(); + let (worker_release, worker_release_rx) = std::sync::mpsc::channel(); + *transport.before_worker_lock.lock().unwrap() = Some(Box::new(move || { + worker_started.send(()).unwrap(); + worker_release_rx.recv().unwrap(); + })); + let waiter = tokio::spawn({ + let transport = transport.clone(); + async move { transport.ensure_initialized().await } + }); + worker_started_rx.await.unwrap(); + let mut stopping = Box::pin(transport.clone().stop(Duration::from_secs(2))); + tokio::select! { result = &mut stopping => panic!("stop completed before owner: {result:?}"), + _ = async { while !transport.stop_task_spawned() { tokio::task::yield_now().await; } } => {} } + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + assert_eq!( + unsafe { host_native_terminal_state(host.host_ctx) }, + 2, + "ordinary cancellation cannot invent Fault" + ); + worker_release.send(()).unwrap(); + assert_eq!(stopping.await.unwrap(), TB_CLOSED); + assert_eq!(entries.load(Ordering::Acquire), 0); + assert!(!transport.is_faulted()); +} + +#[tokio::test] +async fn rejected_spawn_retains_actual_initializer_owner_and_never_resubmits() { + struct Probe(Arc); + impl Drop for Probe { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } + } + let dropped = Arc::new(AtomicBool::new(false)); + let (transport, host) = ModuleTransport::new("spawn-rejection".into(), vec![]); + transport.reject_worker_spawn.store(true, Ordering::Release); + let probe = Probe(dropped.clone()); + transport.defer_initializer(host, move |_| { + drop(probe); + Err("must never execute".into()) + }); + assert!(transport.ensure_initialized().await.is_err()); + assert!(transport.is_faulted()); + let weak = Arc::downgrade(&transport); + assert_eq!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Uncertain + ); + transport + .reject_worker_spawn + .store(false, Ordering::Release); + assert!(transport.ensure_initialized().await.is_err()); + drop(transport); + assert!( + weak.upgrade().is_some(), + "context anchor retains actual job through host teardown" + ); + assert!( + !dropped.load(Ordering::Acquire), + "closure rejection must not drop controller" + ); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct NativeControls { + ctx: *mut c_void, + entered: unsafe extern "C" fn(*mut c_void, *const TbHostVtable), + joined: unsafe extern "C" fn(*mut c_void), +} +pub(crate) struct NativeBarrier { + pub(crate) state: StdMutex<(Option, bool, bool)>, + pub(crate) changed: Condvar, + pub(crate) entered: tokio::sync::Notify, + pub(crate) joined: tokio::sync::Notify, + pub(crate) finish_gate: StdMutex>>, +} +unsafe impl Send for NativeBarrier {} +unsafe impl Sync for NativeBarrier {} +unsafe extern "C" fn native_entered(ctx: *mut c_void, host: *const TbHostVtable) { + let barrier = unsafe { &*ctx.cast::() }; + let mut state = barrier.state.lock().unwrap(); + state.0 = Some(unsafe { *host }); + barrier.entered.notify_one(); + while !state.1 { + state = barrier.changed.wait(state).unwrap(); + } +} +unsafe extern "C" fn native_joined(ctx: *mut c_void) { + let barrier = unsafe { &*ctx.cast::() }; + let mut state = barrier.state.lock().unwrap(); + assert!(!state.2, "one actual native shutdown/join"); + state.2 = true; + barrier.joined.notify_one(); + drop(state); + let gate = barrier.finish_gate.lock().unwrap().take(); + if let Some(gate) = gate { + gate.recv().unwrap(); + } +} +pub(crate) fn native_fixture() -> ( + Arc, + TbHostVtable, + crate::module::abi::TbModuleInit, + Arc, +) { + let path = std::env::var_os("TINYBUS_TEST_CONTROLLED_INIT_MODULE") + .expect("controlled native fixture path"); + let artifact = crate::module::loader::load(std::path::Path::new(&path), false).unwrap(); + let barrier = Arc::new(NativeBarrier { + state: StdMutex::new((None, false, false)), + changed: Condvar::new(), + entered: Notify::new(), + joined: Notify::new(), + finish_gate: StdMutex::new(None), + }); + let controls = NativeControls { + ctx: Arc::as_ptr(&barrier).cast_mut().cast(), + entered: native_entered, + joined: native_joined, + }; + let config = unsafe { + std::slice::from_raw_parts( + std::ptr::from_ref(&controls).cast::(), + size_of::(), + ) + } + .to_vec(); + let (transport, host) = ModuleTransport::new("actual-native".into(), config); + *transport.fixture_controller.lock().unwrap() = Some(barrier.clone()); + (transport, host, artifact.init, barrier) +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_controller_is_owned_until_shutdown_callback_returns() { + let (transport, host, init, barrier) = native_fixture(); + barrier.state.lock().unwrap().1 = true; + transport.defer_initialize(init, host); + transport.ensure_initialized().await.unwrap(); + barrier.entered.notified().await; + let weak = Arc::downgrade(&barrier); + let (release, gate) = std::sync::mpsc::channel(); + *barrier.finish_gate.lock().unwrap() = Some(gate); + let stopping = tokio::spawn({ + let transport = transport.clone(); + async move { transport.stop(Duration::from_secs(2)).await } + }); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); + // Observe ownership safely while the real native callback still borrows + // the controller. Always release C before a regression assertion can panic. + let retained = Arc::strong_count(&barrier) > 1; + let cleanup_incomplete = + transport.context.physical.lock().unwrap().cleanup != CleanupState::Complete; + release.send(()).unwrap(); + assert_eq!(stopping.await.unwrap().unwrap(), TB_OK); + assert!(cleanup_incomplete); + drop(barrier); + let retained_through_transport = weak.upgrade().is_some(); + drop(transport); + tokio::time::timeout(Duration::from_secs(2), async { + while weak.strong_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("successful fixture owner is reclaimed after actual callback return"); + assert!( + retained, + "physical owner must retain the borrowed native controller" + ); + assert!(retained_through_transport); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_late_init_is_joined_after_initiator_abandonment() { + let (transport, host, init, barrier) = native_fixture(); + transport.defer_initialize(init, host); + let waiter = tokio::spawn({ + let transport = transport.clone(); + async move { transport.ensure_initialized().await } + }); + barrier.entered.notified().await; + let mut observation = Box::pin(transport.ensure_initialized()); + std::future::poll_fn(|cx| { + assert!(std::future::Future::poll(observation.as_mut(), cx).is_pending()); + std::task::Poll::Ready(()) + }) + .await; + drop(observation); + assert_eq!( + unsafe { host_native_terminal_state(host.host_ctx) }, + 1, + "observer abandonment cannot cancel initiating owner" + ); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + let view = barrier.state.lock().unwrap().0.unwrap(); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 6 + ); + assert!(!barrier.state.lock().unwrap().2); + { + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); + } + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); + let _physical_exit = transport.lifecycle.lock().await; + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 14 + ); + assert_eq!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Complete + ); + assert!(transport.ensure_initialized().await.is_err()); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_eager_fault_denies_while_config_remains_borrowed_then_joins() { + let (transport, host, init, barrier) = native_fixture(); + let host = DeferredInitializer { + initialize: Box::new(|_| unreachable!()), + host, + }; + let eager = std::thread::spawn({ + let transport = transport.clone(); + move || { + let owner = host; + transport.eager_initialize(init, owner.host) + } + }); + barrier.entered.notified().await; + let view = barrier.state.lock().unwrap().0.unwrap(); + unsafe { + (view.fault)(view.host_ctx, std::ptr::null(), 0); + } + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 6 + ); + assert_eq!( + transport.context.config.lock().unwrap().len(), + size_of::() + ); + assert!(!barrier.state.lock().unwrap().2); + { + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); + } + assert!(eager.join().unwrap().is_err()); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); + let _physical_exit = transport.lifecycle.lock().await; + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 14 + ); + assert!(transport.context.config.lock().unwrap().is_empty()); +} + +async fn wait_physical_cleanup(transport: &ModuleTransport) { + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let changed = transport.context.physical_notify.notified(); + let terminal = matches!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Complete | CleanupState::Uncertain | CleanupState::NoCallbackRequired + ); + if terminal { + return; + } + changed.await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn observing_fault_owned_job_does_not_create_ordinary_stop_marker() { + let (transport, host) = ModuleTransport::new("fault-owner".into(), vec![]); + transport.reject_worker_spawn.store(true, Ordering::Release); + transport.defer_initialize(initialize_ok, host); + assert!(transport.ensure_initialized().await.is_err()); + assert!(!transport.stop_task_spawned()); + assert!(matches!( + transport.clone().stop(Duration::from_secs(1)).await, + Err(StopError::NotStarted(_)) + )); + assert!( + !transport.stop_task_spawned(), + "fault-owned uncertainty cannot become an ordinary handoff" + ); +} + +#[tokio::test] +async fn ordinary_shutdown_retains_non_ok_return_without_physical_success() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + let (transport, host) = ModuleTransport::new("shutdown-status".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.unwrap(); + SHUTDOWN_CODE.store(TB_TIMEOUT, Ordering::Release); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let result = transport.clone().stop(Duration::from_secs(1)).await; + SHUTDOWN_CODE.store(TB_OK, Ordering::Release); + assert_eq!( + result.unwrap(), + TB_TIMEOUT, + "ordinary public return remains compatible" + ); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 2); + assert_eq!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Uncertain + ); + assert!(transport.stop_task_spawned()); + assert!( + transport.is_stopped(), + "legacy stopped is separate from physical-success proof" + ); + assert!(!transport.is_faulted()); +} + +#[tokio::test] +async fn rejected_module_outputs_never_invoke_untrusted_callbacks() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + SHUTDOWN_CALLS.store(0, Ordering::Release); + for rejection in 0..4 { + let (transport, host) = ModuleTransport::new(format!("rejected-{rejection}"), vec![]); + let init: crate::module::abi::TbModuleInit = match rejection { + 0 => initialize_short_output, + 1 => initialize_null_context_output, + 2 => initialize_missing_declared_reinit, + _ => initialize_non_ok_misleading_output, + }; + transport.defer_initialize(init, host); + assert!(transport.ensure_initialized().await.is_err()); + let physical = transport.context.physical.lock().unwrap(); + assert_eq!(physical.init, InitState::PublishedRejected); + assert_eq!(physical.cleanup, CleanupState::Uncertain); + assert!(physical.job.is_some()); + assert!(transport.module.lock().unwrap().is_none()); + drop(physical); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + } + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 0); +} + +#[tokio::test] +async fn rejected_ordinary_worker_submission_retains_original_lifecycle_guard() { + let (transport, host) = ModuleTransport::new("stop-spawn-rejection".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.unwrap(); + transport.reject_worker_spawn.store(true, Ordering::Release); + assert!(matches!( + transport.clone().stop(Duration::from_secs(1)).await, + Err(StopError::Started(_)) + )); + assert!(transport.stop_task_spawned()); + assert!( + transport.lifecycle.try_lock().is_err(), + "rejected closure cannot drop actual acquired guard" + ); + let weak = Arc::downgrade(&transport); + drop(transport); + let retained = weak + .upgrade() + .expect("actual owner retained beyond host handle"); + assert_eq!( + retained.context.physical.lock().unwrap().cleanup, + CleanupState::Uncertain + ); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 2); + assert!( + !retained.is_faulted(), + "ordinary spawner failure is uncertainty, not module fault" + ); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_deadline_fault_retains_late_cleanup_without_wall_clock_sleep() { + let (transport, host, init, barrier) = native_fixture(); + transport.defer_initialize(init, host); + let waiter = tokio::spawn({ + let transport = transport.clone(); + async move { transport.ensure_initialized().await } + }); + barrier.entered.notified().await; + tokio::time::pause(); + tokio::time::advance(MODULE_INIT_DEADLINE).await; + assert!(waiter.await.unwrap().is_err()); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + assert!(!barrier.state.lock().unwrap().2); + { + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); + } + // Resume the real timer only for the outer stuck-test safety deadline. + tokio::time::resume(); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); + wait_physical_cleanup(&transport).await; + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_fault_between_return_and_publication_uses_same_owner() { + let (transport, host, init, barrier) = native_fixture(); + let (publication_tx, publication_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + *transport.before_init_publication.lock().unwrap() = Some(Box::new(move || { + publication_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + })); + transport.defer_initialize(init, host); + let waiter = tokio::spawn({ + let transport = transport.clone(); + async move { transport.ensure_initialized().await } + }); + barrier.entered.notified().await; + { + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); + } + publication_rx.await.unwrap(); + let job = transport + .context + .physical + .lock() + .unwrap() + .job + .clone() + .unwrap(); + unsafe { + (host.fault)(host.host_ctx, std::ptr::null(), 0); + } + { + let state = transport.context.physical.lock().unwrap(); + assert!(Arc::ptr_eq(&job, state.job.as_ref().unwrap())); + assert_eq!(state.cleanup, CleanupState::Requested); + assert!( + state.result.is_none(), + "cleanup must not claim an empty publication" + ); + } + assert!(!barrier.state.lock().unwrap().2); + release_tx.send(()).unwrap(); + assert!(waiter.await.unwrap().is_err()); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); + wait_physical_cleanup(&transport).await; + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_reinitializer_fault_denies_before_c_returns_and_joins_once() { + let (transport, host, init, barrier) = native_fixture(); + { + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + } + transport.defer_initialize(init, host); + transport.ensure_initialized().await.unwrap(); + barrier.entered.notified().await; + { + let mut state = barrier.state.lock().unwrap(); + state.1 = false; + } + let reinit = tokio::spawn({ + let transport = transport.clone(); + async move { transport.reinitialize(crate::Secret::new(vec![])).await } + }); + barrier.entered.notified().await; + unsafe { + (host.fault)(host.host_ctx, std::ptr::null(), 0); + } + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + assert!(!barrier.state.lock().unwrap().2); + assert!( + transport.lifecycle.try_lock().is_err(), + "actual C reinit owns physical exclusion" + ); + { + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); + } + reinit.await.unwrap().unwrap(); + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) + .await + .unwrap(); + wait_physical_cleanup(&transport).await; + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); + assert!( + transport + .reinitialize(crate::Secret::new(vec![])) + .await + .is_err() + ); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_late_eager_publication_cannot_refund_failed_cleanup_spawn() { + let (transport, host, init, barrier) = native_fixture(); + transport.reject_worker_spawn.store(true, Ordering::Release); + let host_owner = DeferredInitializer { + initialize: Box::new(|_| unreachable!()), + host, + }; + let eager = std::thread::spawn({ + let transport = transport.clone(); + move || { + let owner = host_owner; + transport.eager_initialize(init, owner.host) + } + }); + barrier.entered.notified().await; + unsafe { + (host.fault)(host.host_ctx, std::ptr::null(), 0); + } + let job = transport + .context + .physical + .lock() + .unwrap() + .job + .clone() + .unwrap(); + assert_eq!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Uncertain + ); + { + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); + } + assert!(eager.join().unwrap().is_err()); + transport + .reject_worker_spawn + .store(false, Ordering::Release); + transport.request_terminal(true); + { + let state = transport.context.physical.lock().unwrap(); + assert_eq!(state.init, InitState::ExternalValid); + assert_eq!(state.cleanup, CleanupState::Uncertain); + assert!(Arc::ptr_eq(&job, state.job.as_ref().unwrap())); + } + assert!( + transport.module.lock().unwrap().is_some(), + "valid late output is cleanup access only" + ); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + assert!( + !barrier.state.lock().unwrap().2, + "rejected submission must never be silently retried" + ); + let weak = Arc::downgrade(&transport); + drop(transport); + drop(job); + assert!(weak.upgrade().is_some()); + // The existing uncertain physical owner retains this one controller too. + // No separate leaked controller is needed; this is not global leak-bound proof. + let controller = Arc::downgrade(&barrier); + drop(barrier); + assert!(controller.upgrade().is_some()); +} + +unsafe extern "C" fn initialize_short_output( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + unsafe { + initialize_ok(host, out); + (*out).size = TB_MODULE_VTABLE_BASE_SIZE - 1; + } + TB_OK +} +unsafe extern "C" fn initialize_null_context_output( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + unsafe { + initialize_ok(host, out); + (*out).module_ctx = std::ptr::null_mut(); + } + TB_OK +} +unsafe extern "C" fn initialize_missing_declared_reinit( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + unsafe { + initialize_ok(host, out); + (*out).reinitialize = None; + } + TB_OK +} +unsafe extern "C" fn initialize_non_ok_misleading_output( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + unsafe { + initialize_ok(host, out); + } + TB_PANICKED +} + +#[tokio::test] +async fn eager_rejected_outputs_keep_uncertainty_and_never_call_returned_callbacks() { + let _lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + SHUTDOWN_CALLS.store(0, Ordering::Release); + for init in [ + initialize_short_output as crate::module::abi::TbModuleInit, + initialize_null_context_output, + initialize_missing_declared_reinit, + initialize_non_ok_misleading_output, + ] { + let (transport, host) = ModuleTransport::new("eager-invalid".into(), b"borrowed".to_vec()); + assert!(transport.eager_initialize(init, host).is_err()); + let state = transport.context.physical.lock().unwrap(); + assert_eq!(state.init, InitState::PublishedRejected); + assert_eq!(state.cleanup, CleanupState::Uncertain); + assert!(state.job.is_some()); + assert!(transport.module.lock().unwrap().is_none()); + drop(state); + assert!(transport.context.config.lock().unwrap().is_empty()); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + } + assert_eq!(SHUTDOWN_CALLS.load(Ordering::Acquire), 0); +} + +#[test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +fn actual_native_initializer_owner_outlives_runtime_and_host_handle_drop() { + let (transport, host, init, barrier) = native_fixture(); + transport.defer_initialize(init, host); + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let transport = transport.clone(); + tokio::spawn(async move { + let _ = transport.ensure_initialized().await; + }); + barrier.entered.notified().await; + }); + let weak = Arc::downgrade(&transport); + drop(runtime); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 6); + drop(transport); + let retained = weak + .upgrade() + .expect("real initializer owner survives runtime and host drop"); + assert!(!barrier.state.lock().unwrap().2); + { + let mut state = barrier.state.lock().unwrap(); + state.1 = true; + barrier.changed.notify_all(); + } + // The runtime is gone. Wait on the retained physical predicate/Condvar, + // never on a Tokio task, deadline or notification as completion proof. + let mut state = retained.context.physical.lock().unwrap(); + while state.cleanup != CleanupState::Complete { + let (next, timeout) = retained + .context + .physical_changed + .wait_timeout(state, Duration::from_secs(2)) + .unwrap(); + state = next; + assert!( + !timeout.timed_out(), + "actual retained native cleanup must finish" + ); + } + drop(state); + assert!(barrier.state.lock().unwrap().2); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); + drop(retained); + assert_eq!(unsafe { host_native_terminal_state(host.host_ctx) }, 14); +} + +pub(crate) fn native_fixture_json(transport: &ModuleTransport) -> serde_json::Value { + serde_json::json!({"test_controls": *transport.context.config.lock().unwrap()}) +} + +#[tokio::test] +async fn rejected_observer_after_ordinary_stop_preserves_nonfault_cause() { + let (transport, view) = ModuleTransport::new("ordinary-cancellation".into(), vec![]); + let _ = transport.clone().stop(Duration::from_secs(1)).await; + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2 + ); + transport.release_initialization(false); + assert!(!transport.is_faulted()); + assert_eq!( + unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, + 2 + ); +} + +#[tokio::test] +async fn completed_shutdown_has_already_released_its_native_lifecycle_marker() { + let _guard = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + SHUTDOWN_CODE.store(TB_OK, Ordering::Release); + SHUTDOWN_BLOCKED.store(false, Ordering::Release); + let (transport, host) = ModuleTransport::new("completed-shutdown".into(), vec![]); + transport.eager_initialize(initialize_ok, host).unwrap(); + let published = Arc::new(Notify::new()); + let entered = published.clone(); + let (release, worker) = std::sync::mpsc::channel(); + *transport.after_cleanup_publication.lock().unwrap() = Some(Box::new(move || { + entered.notify_one(); + worker.recv().unwrap(); + })); + let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + published.notified().await; + let result = stopping.await.unwrap(); + let still_pending = transport.native_lifecycle_pending(); + // Always release the worker before asserting, even against the unfixed code. + release.send(()).unwrap(); + assert_eq!(result.unwrap(), TB_OK); + assert!( + !still_pending, + "completed cleanup still reports native work" + ); +} diff --git a/docs/modules/module/abi.md b/docs/modules/module/abi.md index 30b0b2e7..11927a5f 100644 --- a/docs/modules/module/abi.md +++ b/docs/modules/module/abi.md @@ -25,6 +25,39 @@ The final host callback, `ready`, marks completion of asynchronous setup. This keeps a lazy module's reserved name routable during initialization without announcing ownership before its objects are serving. +## Optional host capability tails + +ABI revision 1 and the original host prefix remain frozen. Each optional field +is available only when `size` reaches that field's offset plus its complete +field size. The `broker_routing` callback therefore remains available from an +older routing-capable host even though it lacks the newer +`native_terminal_state: Option u32>` tail. +A null host or a size below the frozen prefix is rejected; absent and partial +optional fields normalize independently to `None`. + +The native query contract admits only exactly Live (`1`). Unsupported queries, +zero, unknown bits, combined values, StopIntent (`2`), Fault (`4`) and +PhysicallyStopped (`8`) deny native admission. Terminal intent is monotonic +and never returns to Live. Query success linearizes admission; a cached result +cannot authorize a later operation, and a query alone grants no publication +authority. Later intent cannot undo an already admitted operation. + +PhysicallyStopped means successful opaque shutdown return. For modules opting +into native terminal hooks it additionally requires explicit finish proving +independent worker join. It proves neither legacy worker completion nor receiver +backing destruction. Faulted or uncertain shutdown must not earn this bit. + +A module copies the normalized callback and context, never borrows the host +vtable or configuration storage. A host supplying the query must keep that +callback and context valid and thread-safe for every retained view use, +including independent workers after runtime shutdown or host destruction. +Host destruction must leave a terminal tombstone; this context must never be +recycled or reopened. Custom unsafe C hosts must truthfully uphold this +contract, which cannot cryptographically authenticate an in-process host. +This tail defines an optional ABI capability; ordinary SDK hosts currently +supply `None`. It introduces no retain/release ABI or aggregate resource bound: +the existing process-lifetime host-context and library retention is unbounded. + ## Symbols Every module exports exactly these revisioned symbols: @@ -64,6 +97,17 @@ separate copy of Tokio's thread-locals, so it cannot borrow the host runtime. The host-to-module `deliver` callback only performs a bounded `try_send` and never blocks a host thread. Queue capacity is reported as backpressure and the host retries after the module's `wake` callback. +Before shutdown, the physical lifecycle owner closes initialization and delivery +admission and discards queued messages. It then waits for every reserved +`deliver` callback to return before entering the opaque shutdown callback. +Each backpressure retry requires a new reservation. This barrier covers actual +callback execution, including signals; it does not wait for unanswered method +calls or detached module work. A caller timeout or cancellation retains the +physical worker, callback reservations, and lifecycle lock. A callback that +never returns prevents teardown rather than allowing shutdown underneath it. +Ordinary admission closure is distinct from a module fault and from physical +shutdown completion. An already reported Ready snapshot remains until physical +completion; late `ready` callbacks cannot reopen admission. The symmetric module-to-host `send` callback applies bounded backpressure by blocking the calling module thread until host queue capacity is available. Modules must not call it from a thread that has to remain responsive. diff --git a/docs/plans/native-terminal-authority.md b/docs/plans/native-terminal-authority.md new file mode 100644 index 00000000..59fd4a9a --- /dev/null +++ b/docs/plans/native-terminal-authority.md @@ -0,0 +1,640 @@ +# Native instance terminal authority implementation plan + +Status: proposed implementation; root reviews before source implementation. +Baseline: 4daf0d4fa1ab3afdc30fcc7c09b6e9d6243cd4f5; canonical comparison 1f25576. +Branch security-bus-7328, existing upstream PR #57. Preserve complete PR scope: +actual callback drain, stop admission, physical lifecycle lock, original stop +task/transition markers and ordinary non-Fault stop semantics. This document is +not TinySec App #263 acceptance or issue #7328 completion. + +Use existing superproject worktree only. All shell commands explicitly cd to +/home/enamakel/work/openhuman/worktrees/security-bus-7328/vendor/tinybus. +No nested worktree, second SDK PR, release, gitlink advance or host workaround. +Auto-commit checkpoints stay separate; never squash. + +## Contract and bounds + +Add optional safe per-instance terminal factory in tinybus-module and a size-gated +ABI-v1 host query. Rust closures/traits/Arc/worker handles stay module-local. +Existing C shutdown owns serialized physical cleanup and actual join. + +Separate terminal intent, physical gate fence, worker join and backing last-Drop. +Callback future drain, Connection closure, host deadline and runtime shutdown +prove no independent worker quiescence. Already admitted Publish may complete; +next phase must deny after terminal intent. Module physical gate checks immutable +owner scope/generation/receiver, local flags, live native query, then local flags +again. Query success is admission linearization; later intent cannot undo an OS +operation already granted. Query alone is not a publication permit. + +Normal stop commits intent only after acquiring actual lifecycle lock, before +handing it to retained physical task. Canceled/unacquired stop makes no commitment. +Actual fault, rollback and Host Drop commit immediately, even behind opaque C init +or reinit. Fault takes public-state precedence; normal stop never becomes Faulted. +Fault cleanup must not create ordinary transition_from or restart terminal modules. + +ModuleTransport::new already leaks HostContext and stores a static reference; +loader never dlcloses. Copy normalized callback plus context, never borrowed +vtable/config storage. New view adds no context allocation/retain-release ABI. +Context is terminal tombstone after Host Drop, never recycled/reopened. Callback +contract covers retained independent worker uses after runtime shutdown. +Custom unsafe C hosts must truthfully uphold this lifetime/issuer contract; the +SDK cannot authenticate a malicious in-process host cryptographically. Payloads, +foreign metadata, serde IDs and cached Live receipts never construct authority. + +Existing process-wide context/RuntimeState/library residue is unbounded. This +addition must qualify that preexisting gap; no bounded aggregate claim follows. +If strict new native admission requires a global bound, separately review finite +process-lifetime context quota reserved before exposure, retained after exposure, +with bounded per-slot dynamic content. No speculative pool rewrite or slot reuse. +One initializer/cleanup per instance is a distinct bound from total load count. +No crypto, confidential backing codec or retirement protocol enters this slice. + +## Proposed safe Rust surface (new names, not existing API) + +New crates/tinybus-module/src/terminal.rs and terminal_tests.rs; lib.rs reexports: + + enum TerminalReason { Stop, Fault, SetupFailed, HostClosed } + enum TerminalCompletion { PhysicallyStopped, Uncertain } + struct NativeInstanceLiveness { inner: Arc } + struct RuntimeHooks { + request: Arc, + finish: Box TerminalCompletion + Send>, + } + +Fields private. RuntimeHooks::new accepts those callback bounds plus static. +NativeInstanceLiveness is Clone, has is_live() -> bool and request_fault(). +No public constructor, serde, raw pointer access or arbitrary Connection enrollment. +request_fault first HostCalls::fault, then local monotonic fault plus bounded +request outside panic hook, registry mutex and module phase gate. Never joins. +Independent worker catch_unwind invokes it after unwinding out of physical gate. +Host-origin faults are observed immediately through query even while local request +is serialized behind opaque lifecycle code. + +New macro arm terminal_setup_factory = factory, config = C, with linked equivalent. +Safe factory takes NativeInstanceLiveness and returns Result<(F, RuntimeHooks)>. +F: Fn(Connection, C) -> Fut + Send + Sync + static; +Fut: Future> + Send + static. +Validate host/output/count/config and capability before factory evaluation. +Create TerminalState before factory; store one immutable controller after return. +Factory constructs controller but launches worker only during initial setup after +registration. Before return, factory owns partial-failure disposal. SDK owns it +after return. Same Arc serves setup/reinit; activation checks view before +publishing owner/service. No hook replacement or terminal reset on reinit. + +Private TerminalState holds copied HostCalls, monotonic AtomicU32 intent, +AtomicBool request/finish claims, mutex Option, atomic completion. +Extract hook under lock, release lock, catch_unwind invoking. Poison recovery +must not create a second panic. Request panic closes/faults, no retry/refund; +finish panic leaves Uncertain. Callback is contractually bounded/no I/O/no join/ +no reentrant SDK calls: arbitrary trusted Rust cannot be forcibly made bounded. +Physical finish joins actual controller-owned worker and retains charged resources +until owner-defined actual destruction. SDK never infers destruction from counters. + +## Task 1: additive independently checked ABI tail + +Files: crates/tinybus/src/module/abi.rs, abi_tests.rs; +crates/tinybus-module/src/lib_tests.rs and adjusted host-vtable literals. + +Write failing tests first: routing_tail_survives_newer_optional_tail, +terminal_tail_requires_its_complete_field, prefix_hosts_normalize_both_tails. +Actual repr(C) prefix and old-routing structs allocate only declared bytes. +Cover null, below prefix, exact prefix, partial routing, complete old routing, +partial new callback, full None/Some, and larger storage. Run focused tests red. + +Append native_terminal_state: Option u32>. +ABI constants Live=1, StopIntent=2, Fault=4, PhysicallyStopped=8. Exactly 1 admits; +zero/unknown/combined/terminal flags deny. Terminal flags monotonic, never Live. +Physical bit means successful opaque shutdown return, and for opted-in modules +explicit finish proved worker join. It proves neither legacy worker completion +nor receiver backing destruction. The generic host sets the bit only after an +accepted opaque callback actually returns TB_OK; StopIntent and Fault remain +monotonic and continue denying admission (a legacy fault cleanup can return +flags 14). This is not opted-in hook acceptance. For new native terminal hooks, +TB_OK additionally requires actual independent worker/SDK resource finish and +no sticky uncertainty or fault, under the separate Task 4 protocol. + +Normalization independently checks offset_of(field) + sizeof(field type) for +broker_routing and new callback; never sizeof enlarged whole vtable for old tail. +Frozen prefix/base size and ABI version stay unchanged. Document stronger callback +lifetime covering every retained view use. Run focused tests green. + +## Task 2: retain initialization and publication independently of waiters + +Files: crates/tinybus/src/module/transport.rs and transport_tests.rs, +host.rs/host_tests.rs, host_callbacks.rs/host_callbacks_tests.rs; declared +module_controlled_init native fixture and its existing CI staging. + +Use one mutex-protected PhysicalState in the existing process-lifetime +HostContext. InitState is Deferred, Scheduled, Entered, PublishedValid, +PublishedRejected, CanceledBeforeEntry or ExternalValid. CleanupState is Idle, +Requested, Executing, Complete, Uncertain or NoCallbackRequired. These are +orthogonal states: Requested does not consume the shutdown invocation, and an +empty vtable slot never proves physical success. + +Before thread submission, install one externally anchored Arc +owning Arc, at most one initializer, and any acquired owned +lifecycle guard. The context anchor survives rejected/dropped spawn closures, +Host Drop and Tokio runtime teardown. Never retry failed submission or reset +Uncertain. This is a finite per-instance ownership bound, not a global leak +bound. Result predicates use Notify and Condvar loops, not notification counts. + +The non-async scheduling helper takes initialization-gate then PhysicalState, +rechecks terminal admission, takes the initializer once and records Scheduled +before awaiting physical completion. OnceCell does not own initializer scheduling +or eventual vtable publication. Only the initiating waiter has an armed Drop +guard: deadline expiry or abandonment before publication immediately commits +Fault and Requested. Observation waiters cannot cancel the physical owner. +Already-committed ordinary cancellation does not invent Fault. + +The retained std thread acquires the actual owned lifecycle guard without +holding host state locks. Before C entry it briefly checks PhysicalState: +terminal Scheduled becomes CanceledBeforeEntry/NoCallbackRequired without any +opaque call; live Scheduled becomes Entered. Entered retains config until real +C return regardless of a host deadline or dropped waiter. After C return, +validate only the existing contract: module size >= base size, non-null context, +and Some(reinitialize) if the full module tail is declared. No fictitious +header/version validation, null testing of mandatory Rust function pointers, +or executable-address authentication is added. Non-OK output is ignored; +structurally rejected output invokes no callbacks and retains Uncertain. + +Store accepted vtables under lifecycle exclusion even after terminal intent, +for cleanup-only access. Clear config after the real borrow ends and publish the +immutable result. Under PhysicalState, Requested keeps the SAME initializer +anchor for cleanup; live publication retires the anchor atomically before +lifecycle release; rejection retains uncertainty. Requested cleanup releases +then reacquires lifecycle, never waits for its own guard or submits a second +cleanup worker. Reinit must recheck terminal after obtaining lifecycle. + +Eager activation uses the same Entered/publication predicates. Its fresh, +unexposed transport is demonstrably unlocked: try_lock_owned before opaque C +entry, fail content-free/uncertain on unexpected contention, retain caller Arc, +config and guard through actual return and validation/publication, then release +exclusion. Cleanup waits both eager and deferred Entered publication before +acquiring lifecycle. Late accepted eager output is cleanup only and cannot +attach or reactivate. Failed cleanup submission remains Uncertain even if +eager output is valid later; no refund or resubmission. + +Prove with controlled channels/Condvar barriers and an actual compiled fixture: +blocked native init followed by initiating deadline/drop and late single join; +fault while Scheduled (C never entered); fault after C return before publication +(same owner, no empty-success claim); actual rejected closure/controller retention; +invalid size/null context/missing declared reinit/non-OK misleading output with +zero callbacks; eager fault while config remains borrowed and late single join. + +## Task 3: actual host flags and retained terminal cleanup + +HostContext supplies the actual AtomicU32 query through the optional ABI tail: +Live=1, StopIntent=2, Fault=4, PhysicallyStopped=8. Only exactly Live permits a +phase. Query performs an Acquire load on the leaked context without locks, +allocation, parsing or product logic. Weak allows host_fault to +upgrade the actual owner without an unconditional strong transport cycle. + +Commit native StopIntent (and actual Fault) immediately before physical cleanup +scheduling. Fault precedence is monotonic. host_fault releases inbound locks +before scheduling and never invokes C recursively, joins or waits. Rollback and +Host Drop commit all flags before revocation, release router/loaded locks and +route through retained cleanup. No lifecycle-bypassing synchronous Drop cleanup. + +Callback-load admission carries an explicit Failed versus Canceled cause. +Actual setup Err/unwind, initialization error and actual readiness deadline are +Failed even when ordinary stop races them. After successful setup, waiter Drop, +admission-gate rejection and terminal-caused not-ready/final-commit cancellation +preserve already committed ordinary StopIntent without adding Fault. Live +abandonment still faults; real Fault remains sticky. Apply the selected cause +before revocation and to retained cleanup, including nested observer release. +The final load liveness check rejects actual admission closure even while the +legacy Ready bit remains set and opaque shutdown has not returned. + +Bounded cfg(test) admission/decision observers and fixture JSON Controls encode +test barriers only, never production authority. The declared native fixture emits +actual Ready during opaque init when requested. Tests hold actual shutdown return +after the independent worker joins, prove ordinary cancellation flags2 and eventual +TB_OK flags10, and distinguish genuine deadline/actual host_fault flags6/14. +A host setup unwind test proves Rust rollback classification, not module catch-time +panic containment or Task4 hooks. Setup rejection before gate release proves C never +entered and makes no physical-join claim. + +Normal stop remains deadline-bound while obtaining the actual owned lifecycle +guard. Before acquisition it owns no ordinary intent, job or stop_task_spawned +marker; cancellation/deadline is NotStarted and remains live absent real fault. +After acquisition commit and install/adopt an actual owner synchronously before +the next await. If an already-submitted initializer is Scheduled, ordinary stop +adopts that SAME job, records the original handoff, releases its acquired guard, +then observes completion; the existing worker obtains lifecycle, sees terminal +and proves CanceledBeforeEntry. Never store the guard where that worker needs it +or create a second worker. A running Entered initializer remains excluded; an +ordinary lock deadline does not cancel it or fabricate Fault. + +Fault-owned jobs are observed rather than claimed as new ordinary handoffs. +Original stop_task_spawned and transition_from.get_or_insert(old) survive retry +and caller cancellation. Preserve Started/NotStarted distinctions and ordinary +non-Fault behavior, including an actual non-OK shutdown return. Failed ordinary +submission retains the actual acquired guard in the external anchor and explicit +Uncertain without inventing module Fault or silently retrying. + +Cleanup waits initialization publication before lifecycle acquisition, then +rechecks terminal/cleanup state, closes the actual callback gate and drains its +physical leases. Only accepted cleanup access can claim Requested -> Executing. +Hold physical exclusion and retained job until the one opaque shutdown returns. +Only actual TB_OK permits Complete and the physical-success bit; failed return +or panic is Uncertain. Legacy public Stopped is separate bookkeeping and never +proves join or backing destruction. Successful cleanup can release the anchor; +uncertainty remains inspectable, with no reset/restart. + +peer_detached checks actual Fault before consuming ordinary transition_from. +Late stop_inner completion preserves Faulted/Failed. Native tests cover blocked +actual reinit fault/query denial before return, ordinary callback lease retention, +Host Drop/rollback tombstones, instance isolation and one real join. Keep every +original #57 callback drain, physical lock, retry and original marker assertion. + +## Task 4: accepted module-local lifecycle protocol + +Baseline for this task: a8a8f9ffb935409da5bc3ead20dcad44f7299292. +The corrections below supersede conflicting earlier proposal text. + +# Task 4 proposed concrete module ownership + +Evidence-only proposal for root preflight. No source/build/pin/ref/commit mutation. +Uses accepted fe080 ABI tail and Task 2/3 host ownership, not another ABI. +Source observations: lib.rs RuntimeState/start_module_runtime/factory-mode/shutdown; +connection.rs::handle_call invokes caught panic handler after dispatch unwinds and +before error enqueue; registry locks are released before user invocation. Existing +transport send faults only after successful send; this cannot remain primary fault +notification for opted-in modules. Existing entrypoints retain their behavior. + +## Proposed public module-local API + +New terminal.rs, terminal_tests.rs and lib_terminal_tests.rs; lib.rs reexports: + + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub enum TerminalReason { Stop, HostClosed, Fault, SetupFailed } + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub enum TerminalCompletion { PhysicallyStopped, Uncertain } + #[derive(Clone)] + pub struct NativeInstanceLiveness { inner: Arc } + pub struct RuntimeHooks { + request: Arc, + finish: Box TerminalCompletion + Send>, + } + impl RuntimeHooks { + pub fn new(request: R, finish: F) -> Self + where R: Fn(TerminalReason) + Send + Sync + 'static, + F: FnOnce() -> TerminalCompletion + Send + 'static; + } + impl NativeInstanceLiveness { + pub fn is_live(&self) -> bool; + pub fn request_fault(&self); + } + +All fields private; no raw pointer, constructor, serde or Connection enrollment. +New terminal_setup_factory macro/linked arm takes factory(view) -> +tinybus::Result<(F, RuntimeHooks)>, F: Fn(Connection, C)->Fut + Send+Sync+'static, +Fut: Future>+Send+'static. Same Arc serves setup/reinit. +New doc-hidden start_terminal_reconfigurable_module_with_factory and linked +equivalent own C argument validation; no change to old setup/setup_factory APIs. + +Normalize/copy HostCalls only through trusted unsafe module-init contract. Check +native_terminal_state exists and returns exactly TB_NATIVE_TERMINAL_LIVE before +factory evaluation; unavailable/terminal returns TB_CLOSED with factory untouched. +Do not infer lifetime from broker_routing or sender JSON. Strong callback lifetime +is already part of accepted unsafe ABI contract; actual HostContext persists. +Every is_live samples local terminal, actual callback, local terminal again. Unknown +flags fail closed, mark observed HostClosed locally; valid Fault upgrades Fault. +StopIntent observes Stop. Observation does not claim physical fence/join/backing Drop. + +Factory constructs controller/resources but launches worker only after returned +hooks are registered, during setup. Before return, factory owns partial Err/panic +cleanup. A factory calling view.request_fault before returning persists terminal +intent; after hooks register replay one bounded request and deny setup. Factory +Err/panic with no hooks cannot give SDK a join handle it never received. + +## Request protocol, cause precedence and non-recursion + +TerminalState holds copied HostCalls; monotonic terminal/fault atomics; mutex +ControlState with request Arc Option, separate finish Option, RequestState +(Unregistered/Idle/Running/Done/Panicked), FinishState +(Idle/Anchored/Running/Complete/Uncertain), startup-publication state; Condvar. +All locks recover poison to terminal/uncertain rather than second panic. Atomic +intent is committed before taking these locks or invoking arbitrary code. + +Cause precedence: real Fault dominates Stop/HostClosed. SetupFailed is diagnostic +fault cause, also sets fault flag; never downgrade fault after once request. Stop +and HostClosed both normal terminal, no HostCalls::fault. Preserve first real fault +cause; reason passed to already-invoked request is immutable, persisted reason may +upgrade later. Request contract is identical atomic admission closure for every +cause, so fault escalation never needs a second request invocation. + +request_fault calls HostCalls::fault FIRST, then records local fault and runs +request_if_registered. Host callback schedules only, never awaits/reenters module +hooks. Host intent closes independent worker phases even if module request cannot +yet run. request_if_registered clones request Arc and marks Running under lock, +releases lock, catches callback unwind, reacquires lock to publish Done/Panicked +and notify. If another caller sees Running it returns without waiting; only +physical finish waits request-done. Request panic commits fault/Uncertain and +notifies BEFORE invoking host.fault for escalation; no retry or recursive request. +Pre-registration request stays pending; registration installs both callbacks under +one lock then invokes request_if_registered outside it. No missed early fault. + +Request MUST be bounded atomic admission-close/cancel signal: no I/O/join/reentrant +SDK API, no waiting physical phase gate or Registry/reinitializer mutex. The SDK +cannot forcibly bound trusted callback code; state docs/tests establish contract. +Already granted Publish may finish; actual next-phase query denies immediately. +Physical gate fence plus actual worker join occurs only in finish. Arbitrary worker +catch invokes request_fault after unwinding out of physical gate, not panic hook. + +## One retained physical finish owner including no RuntimeState paths + +Use private module-global anchor collection of Arc, one entry per +opted-in instance while controller ownership needs retention. Insert BEFORE +factory evaluation/transfer can create SDK-owned hooks and before any spawn. +Each instance ControlState has ONE optional Arc; job owns terminal +state plus startup/runtime cleanup resources. Collection owns state independently +of rejected spawn closure, setup future, service refs and output vtable. No public +fake constructor; no extra HostContext or per-view/per-query allocation. This is +explicit module-local controller/anchor bookkeeping, not an ABI lifetime object. +The collection is inspectable/testable and clears completed entries; uncertain +entries deliberately remain. No global boundedness claim; finite total admission +would require separately approved cap. Do not substitute an untracked Box leak +or self-cycle for this external retained owner. + +finish scheduling under ControlState: install job then mark Anchored exactly once; +clone job into Builder::spawn. Spawn Err drops only clone, retains original anchor, +marks Uncertain, retains resources and closes intent. No implicit retry/claim reset. +Spawner injectable privately. Actual worker waits request-done AND startup ownership +publication before taking finish. Neither request caller nor runtime worker waits. +Startup locks/ControlState/phase/reinitializer locks all released before finish. +Take FnOnce once under lock, then run outside lock/catch_unwind. Physical completion +requires TerminalCompletion::PhysicallyStopped; failure/panic stays Uncertain and +retains job/state/resources conservatively. Cleanup thread never calls host shutdown. + +This additional module cleanup thread exists once per terminal instance, not once +per error; request itself never spawns. C shutdown observes/schedules same job then +waits completion on host's physical blocking worker. A fault during stop cannot +claim second finish. Module startup failure has identical finish owner even when +no valid vtable was ever published and host cannot call shutdown. + +## Startup ownership guard: exact transfer and failure paths + +Create StartupGuard outside catch_unwind after argument/config/capability validation +and TerminalState/anchor creation. Factory Err/panic before returning hooks marks +terminal, removes empty anchor only when SDK owns no controller/resources; factory +partial disposal remains its responsibility. On successful factory return register +hooks immediately under guard, before tracing subscriber/runtime creation. + +Guard owns shared StartupResources (runtime Option, reinitializer slot Option, +connection/inbound resources as created). Every resource is attached to this owned +structure immediately; no fallible gap leaves runtime/closure on unwinding stack +with accidental blocking Drop. Only cleanup worker takes them after publication. + +Refactor opted-in runtime builder path, preserving legacy calls: + +1. tracing set_global_default failure after hook transfer: guard records SetupFailed, + host fault, bounded request, schedules anchored finish; returns TB_CLOSED. +2. runtime Builder failure: same, runtime absent. No direct finish on init callback. +3. panic in reinitializer construction, task spawn, RuntimeState construction or + after ownership transfer: outer catch retains guard/StartupResources, marks fault, + schedules same finish and returns TB_PANICKED. Runtime Drop occurs on cleanup + thread after finish, not unwinding ABI/runtime worker. +4. RuntimeState Box created: guard still owns resources until output publication. + write_module_vtable only uses prevalidated writable capacity. Once complete, + mark PublishedRuntime and disarm guard, keeping anchor until actual finish. +5. Panic before output publication: no TB_OK/usable output promised; guard publishes + StartupFailed/resources to job, no host cleanup assumption. After successful + publication, RuntimeState/shutdown own normal routing and shared finish job. + +Guard Drop never joins/blocks: it marks terminal and transfers to already anchored +resources/job, signals publication; catch path performs bounded request/schedule +outside unwind. Guard is fallback when unwinding, not an arbitrary-callback panic +hook. Startup publication wait prevents fault-triggered finish from overtaking +registration/runtime transfer. No await while holding startup locks. + +## Runtime shutdown and setup result classification + +RuntimeState adds optional terminal state; old entrypoints None unchanged. Opted-in +shutdown first samples actual host flags to select Fault/SetupFailed precedence; +if still Live and actual shutdown entered, commit local Stop. Request once. +Take reinitializer under its mutex, release mutex, drop captured callback outside +lock. Close inbound. Publish cleanup resources; schedule same finish owner. +Wait request/finish completion on host physical shutdown worker, holding no module +runtime/reinitializer/startup/phase locks. Actual Host lifecycle remains serialized +around opaque C callback, as accepted Task 2/3. Runtime.shutdown_timeout runs ONLY +after successful join result, on physical cleanup thread or host worker after +resources ownership is taken once. Uncertain finish returns TB_PANICKED, never TB_OK; +resources remain retained if releasing could discard actual uncertain ownership. + +Setup monitor: actual Ok(Err) or JoinError::is_panic upgrades SetupFailed/Fault even +if ordinary stop races. JoinError::is_cancelled while persisted Stop/HostClosed +shares normal cleanup WITHOUT host.fault; successful setup's intentional pending +task canceled by ordinary runtime shutdown is covered. Unexpected cancellation +while Live is SetupFailed. Fault host flags always dominate local normal cause. +Record cancellation decision before runtime teardown; monitor cannot be sole +controller owner because shutdown can abort it. Setup monitor schedules, never +awaits physical finish; failed callback disposal on physical worker releases its +mutex before joining and never blocks runtime needed by in-flight setup/reinit. +Setup/reinit reject terminal before start and before activation. Real setup +Err/panic must be reported at source catch/result boundary before publication of +normal shutdown success; do not rely solely on a monitor canceled during teardown. + +Caught method handler installed with __set_panic_handler invokes request_fault +before content-free error construction/enqueue. Existing handle_call clones/drops +handler lock; dispatch already unwound Registry locks. Remove opted-in reliance +on successful ModuleTransport::send panic-error handling; old behavior unchanged. +Global panic hook remains location-only, no terminal hook/join/reentrant SDK code. + +## Host init output interaction + +No TB_OK from startup failure, and no host invocation of misleading rejected output. +Host PublishedRejected/NoCallbackRequired concerns usable host callback access; +it DOES NOT certify no module startup controller/backing remains. The module's +independent anchor owns failed-startup finish. If finish pending/Uncertain, host +native query stays terminal without PhysicallyStopped. Successful startup finish +with no valid returned vtable still cannot manufacture a host physical-success bit; +only host accepted-vtable shutdown success sets that ABI bit. No extra ABI needed. +Malformed foreign module output/launch without returned hooks remains unjoinable +uncertainty; this safe API never promises to clean resources it never received. + +## Scoped red/green tests and actual compiled fixtures + +Sibling tests pin unavailable query before factory, retained copied view after +runtime shutdown, independent instance/factory isolation, unknown flags deny; +fault before registration -> exactly one request after registration/no setup; +request races finish with barriers -> finish starts only after request done; +request panic -> host scheduling once/no recursion/no second request; +poison recovery -> terminal uncertain/no double panic; +spawn failure -> real controller/resource Drop stays retained after no-vtable +startup error, Host Drop and runtime teardown; finish FnOnce never repeats. + +Inject tracing-registration conflict, runtime-build error, panic after hook transfer, +panic after runtime creation/before output publication; observe actual finish/join +off init/runtime workers and correct uncertainty on spawn failure. Test normal +initial setup cancel and successful pending-task cancel; race real Err/panic with +stop -> Fault preserved; block outbox/host send -> caught handler still closes +native authority before error enqueue. Reinit terminal never launches/replaces. + +Actual compiled module_terminal fixture registers std worker/controller, holds +Publish lease and opaque reinit/init, exposes init-bound test C release controls. +Host fault during opaque hold denies next worker query before cleanup can enter; +old lease release enables actual fence/join. Deadline/drop retains join, hook +request bounded while gate held, actual worker panic after unwind, startup failure +with no valid vtable still finishes through module anchor, post-HostDrop query +tombstone safe. Real old-prefix host rejects only opted-in factory; legacy works. +Use barriers/events, no sleeps; compiled controls never become production authority. + +No tests infer confidential last-Drop/refund from callback/future/join counters. +No codec/crypto/retirement/global load cap addition. Root preflight must approve +module-global anchor bookkeeping and StartupResources ownership transfer before +Task 4 writer implements; they are explicit proposed internal ownership, not +pretended current APIs or an additional C lifetime contract. + +# Task 4 module-local lifecycle preflight + +Reviewed accepted-task4-proposed.md against unchanged tinybus-module/src/lib.rs and connection.rs; Task 1 additive ABI now exists. Read-only: only this ignored report written. No source/build/ref/pin mutations. No extra C ABI or allocator/lifetime handoff is needed. + +## Important findings and accepted concrete resolutions + +**New global collection needs finite admission.** Proposal:93–104 adds a collection retaining failed-startup/Uncertain instances indefinitely. Existing RuntimeState leakage does not account for this: tracing/build failures return before its allocation (lib.rs:570–579,608–615 versus 684–691). A growing Vec plus a global-bound disclaimer is insufficient. + +Root accepts a fixed **256** private terminal-instance permit/anchor array reserved before factory evaluation/SDK hook ownership. Exhaustion returns TB_CLOSED without invoking factory. Each state owns its SlotLease; external slot owns Arc, state owns one Arc, Job holds Weak, and a submitted worker captures strong State+Job. No state/job strong self-cycle. Successful actual finish plus all SDK-owned resource teardown permits anchor/job removal under bookkeeping locks, with destructors outside locks. The slot permit is released only at actual State last-Drop, so retained old liveness views keep their slot occupied and cannot cause unbounded retired-state accumulation through slot reuse. Uncertain/spawn failure retains slot/state/job/resources permanently; no reset/retry/refund. Factory failure before SDK resource transfer can remove its empty anchor; an escaped retained view still keeps the permit occupied until actual Drop. Fixed metadata does not bound arbitrary trusted closure/resource bytes or all mapped SDK images/host contexts. This is an admission cap for the new opt-in bookkeeping, not a broad legacy loader rewrite. Test capacity exhaustion, retained-view permit retention, actual success retirement and uncertain permanent occupancy. + +**Startup attachment must include all owned captures, not just Runtime.** Proposal:130–133 mentions runtime/slot/connection/inbound resources but factory returns F before runtime construction. Current factory path builds initial_setup and reinitialize_factory on the ABI stack (lib.rs:927–932,970–971); tracing failure can drop those captures before RuntimeState exists. Root accepts immediate attachment of returned Arc, parsed config/initial setup closure, reinitializer factory/slot, transport/connection/inbound resources and actual task handles to preexisting StartupResources before subsequent fallible/unwind boundaries. Successful returned Runtime must be attached via poison-recovering, non-user-code transfer immediately after Builder success at lib.rs:608–615, before channels/reinitializer/task construction at 616–631. RuntimeState and FinishJob share this resource owner rather than transferring exclusive Runtime to a fallible local stack. No Runtime Drop on ABI/runtime worker; startup failure cleanup runs on anchored physical worker. Arbitrary factory partial cleanup/destructors before hook transfer remain trusted factory obligations; the SDK cannot promise to control resources it never received. Tests inject failure after every transfer boundary and observe actual destructor thread and retention, not merely flags. + +**Request failure must not spend physical finish.** Proposal:79–80 says request panic commits Uncertain, but FinishState Uncertain must not prevent the still-uninvoked finish callback from executing. Use orthogonal sticky_uncertainty plus finish invocation state. Request panic/poison publishes fault, marks RequestState Panicked/done and sticky uncertainty, then schedules the same still-eligible physical finish. Never repeat request or consume finish merely because uncertainty was recorded. After actual finish, uncertainty stays visible; no TB_OK/permit refund follows a failed request merely because join succeeded. A failed spawn remains unsubmitted and retained with no automatic retry. Tests cover request panic before job submission and while finish waits request-done. + +## Accepted setup/reinit protocol: settle execution before physical completion + +Current source keeps the successful initial setup task pending forever (lib.rs:646–655), and its monitor owns the exclusive JoinHandle (664–665). Err/panic is reported only after callback disposal and host fault at 669–680. RuntimeState shutdown disposes callback under the reinitializer mutex then destroys runtime (375–400); C reinit holds that mutex while block_on polls setup (404–417,955–960). A new job that waits this mutex or initial setup completion without first arranging cancellation can wait forever for work whose completion needs terminal cleanup. Joining a worker before an initial/reinit setup future is settled also lets that future launch work after the claimed join. + +Use this concrete opted-in-only protocol: + +1. Keep initial setup invocation and its returned future in a real task. Catch both synchronous F(connection, config) construction and asynchronous future polling inline, not only the later monitor. Publish actual Err/panic -> Fault/SetupFailed immediately at that result boundary, before readiness or any task-completion notification. Record cancellation separately, preserving already-committed Stop/HostClosed. Existing non-opted-in behavior is unchanged. +2. Initial task returns after its actual setup result; do not park it forever. The already-persisted connection_slot owns the serving Connection (lib.rs:648–649); StartupResources owns that slot for runtime lifetime. FinishJob/StartupResources owns the task's sole JoinHandle. A monitor may observe shared published outcome, but must not consume the only handle required for real cancellation/completion proof. +3. On terminal request, keep Runtime alive, abort an unfinished initial task, and await its actual JoinHandle on the physical cleanup thread (via retained runtime handle). This proves future cancellation/drop has actually happened. Classify is_cancelled caused by Stop/HostClosed as normal; genuine source-observed Err/panic always upgrades Fault. Do not await that join on a runtime worker, under reinitializer/startup/control/gate locks, or after shutting down its runtime. +4. Opted-in reinit drives its setup invocation under terminal-aware cancellation (a terminal notification/predicate select with inline result/panic recording; poll the result branch first when result and terminal are both ready, so an actual ready Err is not lost). This lets a persisted request close the running reinit future while C still owns its mutex; it then returns a normal terminal code and releases that mutex. A real Err/panic observed while polling still faults before returning. No new setup invocation starts after terminal, and readiness/owner publication rechecks terminal. Cleanup can then take the reinitializer slot, release its mutex and drop captured callbacks outside it. Waiting a reinit that ignores cancellation synchronously retains pending ownership; it is not quiescence. +5. After actual initial-setup settlement and reinitializer disposal, run the one finish callback, keeping Runtime services available for the actual controller worker's fence/join. The request callback must already have performed bounded cancel/wake; finish must not rely on an initial/reinit setup future that has been canceled, nor await host shutdown. This dependency contract belongs in public callback docs. Synchronous never-returning user setup, blocking destructor, nonreturning reinit, or worker dependency that cannot settle remains retained pending/Uncertain with no successful completion. Do not fake a timeout as join. +6. After finish success, teardown SDK-owned runtime/connection/callback/task resources exactly once off locks on the physical worker. Only then publish completion and retire the anchor if there is no sticky uncertainty. Recheck local/actual host fault when answering C shutdown; normal Stop cannot erase genuine Fault/SetupFailed arriving during setup or teardown. Fault/uncertain never earns the host physical-success bit. Successful no-vtable startup cleanup proves only module-local cleanup: host query stays terminal and cannot fabricate accepted-vtable shutdown evidence. + +This avoids the concrete initial-task/reinitializer/runtime dependency cycle without adding a C field. It changes only the opt-in internal setup task lifetime. If implementation chooses another protocol, it must prove the same actual future settlement before finish/physical success and retain a sole actual JoinHandle or equally strong completion owner; a setup monitor canceled by runtime teardown is insufficient. + +## Boundaries already supported by source + +Caught per-connection panic handler runs after registry-free dispatch catches unwind and before error enqueue: connection.rs:1441–1453; object-tree lock ends at 1584–1587 before invocation. Transport writing is a separate writer task (1355–1365). Put host fault first in that handler, before error construction/send. ModuleTransport::send's existing TB_OK-only fault at lib.rs:270–285 cannot be relied on for the opted-in path. Request callback is separate Arc; finish FnOnce is separately taken, callback storage locks released before arbitrary code, request completion notified even on panic. Global panic hook remains location-only. A factory pre-fault is persisted before registration, request replayed once after registration, and setup denied; finish waits the startup-publication predicate to avoid overtaking resource transfer. + +Task 1's actual host query field docs (abi.rs:184–202) cover copied callback/context and post-runtime worker lifetime. Module-local anchors never export Rust traits/Arc or allocations through C and add no new host context. PhysicalStopped means actual supported shutdown/join evidence, never receiver-last-Drop, codec retirement or foreign allocation destruction. + +## Scoped verdict + +**Task 4 IMPLEMENTABLE** with this report's accepted corrections governing the writer. Root has accepted fixed256 actual-last-State-Drop admission, job Weak ownership, immediate StartupResources transfer, the initial-task real JoinHandle protocol, result-first terminal-aware reinit cancellation, actual source Err/panic publication, off-runtime request/initial-task settlement/finish/resource-teardown ordering, and final local/native Fault recheck. This is a concrete module-local protocol with no remaining source contradiction or extra ABI requirement. + +The required orthogonal matrix is: Request Panicked/done -> sticky_uncertainty, while uninvoked Finish remains eligible for its same once job; actual finish return is recorded independently. SpawnFailed retains anchored unsubmitted owner, sticky uncertainty, no retry. Finish panic/Uncertain retains remaining anchored resources and permit. Only actual successful finish AND all SDK-owned teardown with no sticky uncertainty can remove anchor/job; permit itself remains until State last-Drop. Cause Fault is independent of whether resources were actually joined/released and never disappears from terminal state. A successfully cleaned no-vtable startup instance cannot make the host physical-success bit appear. NoCallbackRequired for no returned callable host vtable must never imply that a registered module controller does not exist. + +Arbitrary synchronous nonreturning setup/Drop, blocked opaque reinit or worker dependency can retain pending ownership indefinitely. Callback docs must require bounded cancel/wake request and finish's independent gate/fence/join responsibilities; there is no forced trusted-code cancellation, no arbitrary captured-byte bound, and no confidential receiver destruction proof. Existing legacy entrypoints remain unchanged. Fresh actual compiled failure/cancellation/join/admission proofs remain implementation acceptance gates, not preflight claims. This verdict does not change the active host Task2/3 writer's scope. + + +## Binding panic-payload order addendum + +Root accepts request panic/poison sticky uncertainty independently from the once finish state. The writer must also prevent primary panic payload Drop before the actual fault fence. `catch_unwind(...).is_err()`, `Err(_)`, or temporary matches can dispose the caught `Box` before later fault notification. Use explicit `Err(payload)` ownership in every new catch. Do not format/downcast payloads or override the global hook. + +For once-only module request/finish and the new setup/reinit catches, use fixed private per-instance first-payload cells inside admitted TerminalState, rather than an unbounded vector or a local value whose destructor runs at return. Exact new-source categories: request (once), finish (once), initial setup (first construction/poll panic), reinit (first panic, terminal then denies further invocation), factory (first/only call), startup wrapper (once), and cleanup/SDK teardown wrapper (once). **At most seven first-payload Box owners per admitted instance**, separately named cells, with no arbitrary payload-byte bound. These cells can be omitted where a boundary is already captured by another category, but must not alias categories that can both fail. Initial/reinit construction and polling share their respective first-fault cell; once terminal they cannot be polled/retried to accumulate new primary payloads. This is not a bound on arbitrary panics in unrelated user tasks or legacy/concurrent method calls. + +On primary capture: retain ownership explicitly; publish actual host/native Fault plus local terminal/sticky uncertainty without invoking any foreign destructor, then install the payload in its still-empty fixed cell under poison-recovered control storage. For request panic, set Request Panicked/done and notify completion outside held locks BEFORE any potential payload disposal; finish remains eligible once. For setup/reinit publish the real fault before task/C result completion; for finish/teardown record uncertain completion and notify before return. Atomic native/local intent must not depend on acquiring payload storage. No replacement/take-and-drop of a populated cell on a retry: terminal/once admission forbids that retry. + +The minimal safe disposal policy is **do not dispose captured primary payloads in this slice**. Sticky-fault state remains externally anchored with its permit/controller/resources; the fixed cells retain payloads for the process lifetime. This is accountable finite owner-count retention under fixed256 admitted states, not an untracked per-panic leak, a zeroization guarantee, or a total foreign-byte bound. The same real physical finish is still attempted once after request completion; sticky uncertainty prevents TB_OK, success-bit publication or permit refund even if worker join succeeds. If implementation instead chooses guarded disposal, its secondary-payload owner must also have an explicitly reserved fixed cell and fault/done notifications must already have happened; arbitrary recursive destructor panic/block/abort cannot be promised recoverable. Do not add a generic panic framework or a resubmission loop. + +Captured payload ownership does not make arbitrary user future/capture destruction safe: synchronous blocking Drop or secondary unwind during cancellation can leave the actual task JoinHandle pending or fail. Keep the accepted runtime/job/permit owner alive and do not claim completion. A destructor may abort the process; no generic abort/foreign allocator recovery is promised. Initial-task actual join and reinit exclusion remain required before physical completion; runtime timeout/future cancellation is not their substitute. + +### Actual method-panic source gap requiring a narrow opt-in catch observer + +Source review found a stronger prerequisite than the prior report's handler-only wording. connection.rs:1554–1557 catches a user future poll with `Err(_) => Poll::Ready(Err(()))`. Its foreign payload can be dropped *inside that catch match*, before `handle_call` invokes the registered panic handler at 1451–1453. Therefore installing fault-first logic only in `__set_panic_handler` is too late for a payload that blocks/panics on Drop. + +Add a narrowly private opted-in catch-time terminal observer to the Connection/CatchUnwind path (Rust-local only, no C ABI). Clone its callback outside handler/registry locks. On `Err(payload)` invoke the observer to commit actual host/local Fault, bounded request completion and sticky uncertainty BEFORE payload is dropped; then produce the content-free method error. Existing legacy observer-absent behavior can remain. If observer-triggered request fails, its own fixed request cell/completion protocol applies. Ordinary method dispatch and global panic hooks must not acquire controller joins/gates. + +Do not pretend all concurrent method panic payloads fit the seven once-boundary cells: method calls have no such count bound. The narrow observer guarantees fault-before-disposal for those payloads; it does not promise to retain/recover every arbitrary method destructor. After observer commits sticky fault, a blocking/secondary-panicking method payload can prevent later error delivery/task settlement, but native authority is already denied and the actual external owner/permit remains retained. No physical success/refund follows that method panic. Guarding and repeatedly retaining arbitrary method payloads would require separately bounded admission and is outside this slice. + +**Addendum verdict:** module-local once-boundary retention is implementable within fixed admitted state and without new ABI. Full Task4 fault-before-payloadDrop proof additionally requires the narrow opted-in Connection catch-time observer above; handler-only implementation does not pass. This is a precise source-backed correction, not a promise about all legacy connection panics. Add a panic_any payload with blocking Drop barrier: verify independent native query denies and request-done is observable before the barrier is released; repeat request/finish/initial/reinit boundary cases and assert the actual once finish attempt/retained permit rather than flags alone. + + +Root accepts the narrow opt-in Connection catch-time prerequisite. Its observer is installed only by the actual terminal factory/runtime path, closes private TerminalState plus actual HostCalls::fault first, and catches request panic through the fixed request-cell/done protocol before the primary method payload can drop. No public fake liveness constructor, payload-selected observer authority or connection-wide growing collection is introduced. New setup construction/poll/reinit catches use the same terminal observer ordering. Preserve existing content-free MethodFailed wire schema and legacy observer-absent behavior. + +Runtime resource qualification is binding: `runtime.shutdown_timeout` (current lib.rs:393) can return with opaque blocking tasks/Drop still alive. Its return is not proof those owners were destroyed. Remove successful anchors/release accounting only with actual teardown evidence. Any surviving SDK task/transport/observer must retain its actual TerminalState clone, keeping the State-owned permit occupied through actual task/clone Drop; if that ownership cannot be established, keep the external Uncertain anchor. Never interpret timeout, async Connection closure or callback/future drain as actual method-task Drop, independent-worker join or receiver last-Drop. The controller's proved worker join is a separate fact from SDK-owned resource destruction. User-created detached tasks not received/tracked by SDK remain the trusted controller's lifecycle obligation; no generic task-destruction guarantee follows. + +Exact source touchpoints/tests for this correction: `tinybus/src/connection.rs::CatchUnwind::poll` (1554–1557) explicit payload capture/observer before Drop, `handle_call` (1441–1453) observer cloning outside lock/content-free error after fault, and `tinybus-module/src/lib.rs` caught setup/runtime wrappers plus new terminal.rs request/finish first-payload cells. Controlled `panic_any` payload records Drop entry and may block there: assert actual native Fault observed before Drop entry, actual Request done notification before waiting finish, and outgoing blocked/failed send cannot postpone the fence. Request/finish primary payload is retained in its fixed cell and never reaches Drop in this slice; method payload may drop only after the observer fence. Check actual original once finish attempt, sticky uncertainty and held permit while task/destructor/native join remains pending. Do not infer task destruction from future-count or message-drain acknowledgements. + +**Final scoped Task4 verdict: IMPLEMENTABLE with all binding corrections in this report, including the accepted narrow opt-in catch-time observer and actual SDK-owner/permit retention qualification.** Implementation acceptance still requires fresh native/source proofs. No source, builds, refs, pins, global hook changes or foreign payload formatting were performed for this review. + +## Task 5: actual compiled native proof and compatibility negatives + +New declared cdylibs crates/tinybus/examples/module_terminal.rs and +module_legacy_prefix.rs, explicit examples in crates/tinybus/Cargo.toml. +Legacy fixture uses locally frozen repr(C) prefix, not enlarged current layout. +Extend scripts/check-coverage.sh build/env validation and host_tests.rs loader +tests. Use existing staged_module hash allowlist/private copies; separate mapping +per test avoids module-global tracing collision. Include all fixtures in packages. + +Fixed test interface controls worker phase, held Publish, held reinit/init, +queued jobs, worker panic, failure before/after launch, receiver retention/Drop. +Use std channel/Condvar barriers and event acknowledgements, never sleeps. +After fault bus detaches: test-only exported C control functions, bound from the +actual staged library during init, release barriers and read actual completion. +Do not make production liveness constructors or payload control authority. + +Required red/green actual compiled tests: + +1. Hold opaque C reinit/init; actual host fault denies independent worker's next + physical phase before releasing opaque call. Cleanup stays pending, then once. +2. Old Publish lease survives stop timeout/caller cancellation; new phase denies; + original callback drain/task/lock and join retained. Release proves real join. + Retained receiver last-Drop is separate and never inferred from future drain. +3. Actual method/worker panic and poisoned bounded local gate: one close/finish, + denied queued callers, no blocking panic hook/reopen/refund. +4. Init/factory failure before launch, after registration/launch, rollback and + eventual valid vtable after abandoned waiter preserve real cleanup ownership. +5. Retained actual view after Host Drop/runtime shutdown queries terminal tombstone; + two instance isolation, no stale reuse or terminal reset on reinit/retry. +6. Compiled old module/new host; new terminal module/actual compiled prefix-only + host rejects before factory, ordinary legacy setup still loads. Truncated and + tampered prefix negatives must allocate genuinely frozen layout. JSON Live, + foreign metadata/serialized IDs cannot fabricate query or admission. + +Fixtures prove SDK lifecycle only, not TinySec charge or confidential last-Drop. + +## Task 6: docs, declarations and complete proof gates + +Update docs/modules/module/abi.md and README.md with actual optional tail/lifetime, +factory example, bounded request, physical join, post-runtime tombstone, legacy +denial and preexisting unbounded residue. Public Rust docs match actual API. + +Long gates use absolute superproject scripts/ci-cancel-aware.sh from explicit SDK +cwd. Never export CARGO_TARGET_DIR or send builds into temp directories. +Run focused red/green first and retain commands/full outputs/exit status: + +- cargo fmt --all -- --check +- cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +- cargo test --locked --workspace --all-features; preserve baseline 557 assertions +- cargo check --locked --workspace --all-targets --all-features +- cargo check --locked --package tinybus --no-default-features +- scripts/test-features.sh and Rust 1.85.0 enabled/disabled all-target checks +- cargo test --locked --workspace --doc --all-features +- Build all declared cdylibs, execute native ignored tests with explicit fixture + paths from both SDK cwd and superproject cwd via manifest path; fresh staging +- Pack tinybus/macros/module with compilation verification and real local registry + dependency strategy; --no-verify alone is not package compile proof +- Fresh scripts/check-coverage.sh, at least 90% meaningful compiled file lines, + no new ignore regex. Separate llvm-cov JSON verifies at least 80% actual changed + executable lines versus current canonical and full PR base, including excluded + host integration code in a dedicated report; no global percent substitution +- OpenHuman rust:layout/crate-chain/feature forwarding with native/module ABI off + and on, complete fixture/declaration checks and actual old/new ABI execution + +Review exact final diff and independently verify fresh proof before root updates +existing upstream #57 ready for review. No merge/release/pin/issue closure granted. + +## Source-backed review decisions before implementation + +This is an exact interface/ownership/TDD plan, not a fabricated complete source +replacement. Prove init/normal-stop/fault cleanup-claim handoff, actual method-panic +registry boundary, thread-spawn failure retention and invalid-init-vtable contract. +If source makes these contradictory, escalate exact conflict before green code; +never invent quiescence or silently weaken immediate native fault denial. +Resolve separately whether bounded global native admission is required: leaked +context pointer safety cannot satisfy it. No additional ABI lifetime allocation +is necessary for the approved actual SDK context contract. diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index 2e49d32b..adea2aaf 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." -cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --no-default-features --features modules,macros +cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --no-default-features --features modules,macros set -euo pipefail module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_clock.so' -print -quit) module_two=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_clock_two.so' -print -quit) @@ -9,7 +9,8 @@ wrong_target=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_w context_module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_context.so' -print -quit) context_eager_module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_context_eager.so' -print -quit) host_callback_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) -for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module"; do +controlled_init_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) +for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module"; do if [ -z "$artifact" ] || [ ! -f "$artifact" ]; then echo "a module loader fixture is missing" >&2 exit 1 @@ -33,4 +34,4 @@ printf '"%s" = "%s"\n' "$library" "$digest" > "$private/modules.toml" # process/network branches are not meaningful per-file coverage. The # test-support module is integration-test glue whose platform-specific # dynamic-loader and timeout paths run in the dedicated OS jobs instead. -TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 -- --include-ignored +TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$PWD/$controlled_init_module" cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 -- --include-ignored diff --git a/scripts/test-e2e.ps1 b/scripts/test-e2e.ps1 index 90592413..bbd3f966 100644 --- a/scripts/test-e2e.ps1 +++ b/scripts/test-e2e.ps1 @@ -19,10 +19,11 @@ Copy-Item target/debug/examples/module_clock.dll $private Copy-Item target/debug/examples/module_context.dll $private Copy-Item target/debug/examples/module_context_eager.dll $private Copy-Item target/debug/examples/module_host_callback.dll $private +Copy-Item target/debug/examples/module_controlled_init.dll $private Copy-Item target/debug/examples/module_wrong_target.dll $private # Admit every intentional fixture by its actual built bytes so later # checks, including the wrong-target gate, can exercise their contract. -$fixtureAllowlist = @('module_clock.dll', 'module_context.dll', 'module_context_eager.dll', 'module_host_callback.dll', 'module_wrong_target.dll') | ForEach-Object { +$fixtureAllowlist = @('module_clock.dll', 'module_context.dll', 'module_context_eager.dll', 'module_host_callback.dll', 'module_controlled_init.dll', 'module_wrong_target.dll') | ForEach-Object { $digest = (Get-FileHash -LiteralPath (Join-Path $private $_) -Algorithm SHA256).Hash.ToLowerInvariant() '"' + $_ + '" = "' + $digest + '"' } @@ -45,6 +46,7 @@ $env:TEMP = $private $env:TMP = $private $env:TINYBUS_TEST_CONTEXT_MODULE = (Get-Item "$private/module_context.dll").FullName $env:TINYBUS_TEST_CONTEXT_EAGER_MODULE = (Get-Item "$private/module_context_eager.dll").FullName +$env:TINYBUS_TEST_CONTROLLED_INIT_MODULE = (Get-Item "$private/module_controlled_init.dll").FullName $env:TINYBUS_TEST_HOST_CALLBACK_MODULE = (Get-Item "$private/module_host_callback.dll").FullName $env:TINYBUS_TEST_MODULE = $module $env:TINYBUS_TEST_MODULE_TWO = $moduleTwo diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh index bf97e755..72579373 100755 --- a/scripts/test-e2e.sh +++ b/scripts/test-e2e.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." -cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --no-default-features --features modules,macros +cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --no-default-features --features modules,macros cargo test --locked --no-default-features --features modules,macros,test-support test_support::tests case "${RUNNER_OS:-$(uname -s)}" in Windows|MINGW*|MSYS*|CYGWIN*) pwsh -NoProfile -File scripts/test-e2e.ps1 ;; @@ -13,7 +13,8 @@ wrong_target=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodul context_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_context.so' -o -name 'libmodule_context.dylib' \) -print -quit) context_eager_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_context_eager.so' -o -name 'libmodule_context_eager.dylib' \) -print -quit) host_callback_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) -for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module"; do +controlled_init_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) +for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module"; do if [ -z "$artifact" ] || [ ! -f "$artifact" ]; then echo "a module example artifact is missing; the loader tests would not exercise it" >&2 exit 1 @@ -30,7 +31,7 @@ else digest="$(shasum -a 256 "$private/$library" | awk '{print $1}')" fi printf '"%s" = "%s"\n' "$library" "$digest" > "$private/modules.toml" -TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" cargo test --locked --no-default-features --features modules,macros,test-support -- --ignored +TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$PWD/$controlled_init_module" cargo test --locked --no-default-features --features modules,macros,test-support -- --ignored cargo test --locked --no-default-features --features modules,macros admission_tests -- --include-ignored TINYBUS_TEST_UMASK=1 cargo test --locked --no-default-features --features modules,macros permissive_umask -- --test-threads=1