diff --git a/crates/tinybus-module/Cargo.toml b/crates/tinybus-module/Cargo.toml index 6cf7994d..ea359d0d 100644 --- a/crates/tinybus-module/Cargo.toml +++ b/crates/tinybus-module/Cargo.toml @@ -9,6 +9,9 @@ description = "Module-side runtime and export macro for trusted tinybus cdylibs. [features] static-link = [] +# Private, off-by-default instrumentation for compiled native lifecycle fixtures. +# Not a production API or a source of native authority. +__native-fixture-support = [] [dependencies] async-trait.workspace = true diff --git a/crates/tinybus-module/src/execution_context.rs b/crates/tinybus-module/src/execution_context.rs index 93bede00..5cd40d11 100644 --- a/crates/tinybus-module/src/execution_context.rs +++ b/crates/tinybus-module/src/execution_context.rs @@ -32,6 +32,18 @@ impl ExecutionOwner { live, } } + #[cfg(feature = "__native-fixture-support")] + pub(super) fn is_current_for_fixture(&self) -> bool { + OWNER.with(|slot| { + slot.borrow() + .as_ref() + .is_some_and(|current| Arc::ptr_eq(&self.live, ¤t.live)) + }) + } + #[cfg(feature = "__native-fixture-support")] + pub(super) fn is_stopped_for_fixture(&self) -> bool { + !self.live.load(Ordering::Acquire) + } pub(super) fn enter(&self) -> ExecutionGuard { let previous = OWNER.with(|slot| slot.replace(Some(self.clone()))); let dispatch = tracing::dispatcher::set_default(&self.dispatch); diff --git a/crates/tinybus-module/src/execution_context_tests.rs b/crates/tinybus-module/src/execution_context_tests.rs index 7b0d052a..0acdda77 100644 --- a/crates/tinybus-module/src/execution_context_tests.rs +++ b/crates/tinybus-module/src/execution_context_tests.rs @@ -111,3 +111,138 @@ fn unowned_and_reentrant_panic_contexts_never_select_a_host_callback() { assert_eq!(count.load(Ordering::SeqCst), 0); assert!(owner.panic_location.lock().unwrap().is_none()); } + +static CONFIG_DISPATCH: StdMutex> = StdMutex::new(None); +struct CapturedConfig; +impl<'de> serde::Deserialize<'de> for CapturedConfig { + fn deserialize>( + deserializer: D, + ) -> std::result::Result { + let _ = serde_json::Value::deserialize(deserializer)?; + *CONFIG_DISPATCH.lock().unwrap() = Some(tracing::dispatcher::get_default(Clone::clone)); + Ok(Self) + } +} + +#[test] +fn shutdown_suppresses_the_exact_factory_and_configuration_caller_dispatch() { + for factory in [true, false] { + let (fixture, count) = owner(); + let mut host = fixture.host.0; + host.config = tinybus::module::abi::TbSlice { + ptr: b"{}".as_ptr(), + len: 2, + }; + let captured = Arc::new(StdMutex::new(None)); + let capture = captured.clone(); + let mut module = TbModuleVtable::default(); + let started = if factory { + unsafe { + start_reconfigurable_module_with_factory::( + &host, + &mut module, + 1, + false, + move || { + *capture.lock().unwrap() = + Some(tracing::dispatcher::get_default(Clone::clone)); + |_, _| async { Ok(()) } + }, + ) + } + } else { + unsafe { + start_module_with_config::( + &host, + &mut module, + 1, + false, + |_, _| async { Ok(()) }, + ) + } + }; + let dispatch = if factory { + captured.lock().unwrap().take() + } else { + CONFIG_DISPATCH.lock().unwrap().take() + }; + let before = count.load(Ordering::SeqCst); + if let Some(dispatch) = &dispatch { + tracing::dispatcher::with_default(dispatch, || { + tracing::info!("captured caller before stop") + }); + } + let live = count.load(Ordering::SeqCst); + let stopped = if started == TB_OK { + unsafe { (module.shutdown)(module.module_ctx, 1000) } + } else { + TB_CLOSED + }; + let after_shutdown = count.load(Ordering::SeqCst); + let (replacement_fixture, replacement_count) = owner(); + let mut replacement_host = replacement_fixture.host.0; + replacement_host.config = host.config; + let mut replacement = TbModuleVtable::default(); + let replaced = unsafe { + start_module(&replacement_host, &mut replacement, 1, false, |_| async { + Ok(()) + }) + }; + let replacement_before = replacement_count.load(Ordering::SeqCst); + if let Some(dispatch) = &dispatch { + tracing::dispatcher::with_default(dispatch, || { + tracing::info!("captured caller after actual shutdown") + }); + } + let late = count.load(Ordering::SeqCst); + let replacement_after = replacement_count.load(Ordering::SeqCst); + let replacement_stopped = if replaced == TB_OK { + unsafe { (replacement.shutdown)(replacement.module_ctx, 1000) } + } else { + TB_CLOSED + }; + eprintln!( + "factory={factory}: start={started}, stop={stopped}, before={before}, live={live}, after_shutdown={after_shutdown}, late={late}, replacement={replacement_before}/{replacement_after}, replacement_stop={replacement_stopped}" + ); + assert_eq!(started, TB_OK); + assert!(dispatch.is_some()); + assert!( + live > before, + "real caller dispatch forwards before shutdown" + ); + assert_eq!(stopped, TB_OK); + assert_eq!(replaced, TB_OK); + assert_eq!(replacement_stopped, TB_OK); + assert_eq!( + late, after_shutdown, + "stopped caller dispatch must not forward" + ); + assert_eq!( + replacement_after, replacement_before, + "old dispatch cannot select replacement" + ); + } +} + +unsafe extern "C" fn tracing_native_query(_: *mut c_void) -> u32 { + tracing::info!("terminal delivery query"); + tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE +} +#[test] +fn terminal_deliver_enters_exact_instance_and_restores_nested_caller() { + let (caller, caller_count) = owner(); + let (instance, instance_count) = owner(); + let host = HostCalls(TbHostVtable { + native_terminal_state: Some(tracing_native_query), + ..instance.host.0 + }); + let state = super::super::terminal::TerminalState::admit(host, Some(instance)).unwrap(); + let _outer = caller.enter(); + let byte = 0; + let code = unsafe { super::super::terminal_deliver(state.context(), &byte, 1) }; + tracing::info!("restored caller"); + state.empty_factory_failure(); + assert_eq!(code, TB_CLOSED); // No inbound queue was installed. + assert_eq!(instance_count.load(Ordering::SeqCst), 1); + assert_eq!(caller_count.load(Ordering::SeqCst), 1); +} diff --git a/crates/tinybus-module/src/lib.rs b/crates/tinybus-module/src/lib.rs index 75ba18a0..6f1fafb7 100644 --- a/crates/tinybus-module/src/lib.rs +++ b/crates/tinybus-module/src/lib.rs @@ -26,6 +26,9 @@ use tinybus::module::abi::{ use tinybus::{Connection, Error, Result, Transport}; use tokio::sync::{Mutex, mpsc}; +mod terminal; +pub use terminal::{NativeInstanceLiveness, RuntimeHooks, TerminalCompletion, TerminalReason}; + mod execution_context; use execution_context::ExecutionOwner; @@ -490,6 +493,12 @@ where F: FnOnce(Connection) -> Fut + Send + 'static, Fut: Future> + Send + 'static, { + let Some((_, compatible_host)) = + (unsafe { validate_start_arguments(host, out, worker_threads) }) + else { + return TB_BAD_ARGUMENT; + }; + let owner = Some(ExecutionOwner::new(HostCalls(compatible_host))); unsafe { start_module_runtime( host, @@ -498,7 +507,7 @@ where detach_on_panic, setup, None, - false, + owner, ) } } @@ -528,7 +537,7 @@ where detach_on_panic, setup, None, - true, + None, ) } } @@ -551,6 +560,8 @@ unsafe fn validate_start_arguments( Some((out_capacity, host)) } +// Carry the validated entry caller's exact diagnostic owner into the runtime. +// Captured factory/config dispatch and shutdown must share the same live bit. unsafe fn start_module_runtime( host: *const TbHostVtable, out: *mut TbModuleVtable, @@ -558,7 +569,7 @@ unsafe fn start_module_runtime( detach_on_panic: bool, setup: F, reinitialize_factory: Option, - linked: bool, + owner: Option, ) -> i32 where F: FnOnce(Connection) -> Fut + Send + 'static, @@ -572,7 +583,6 @@ where }; let host = HostCalls(host); - let owner = (!linked).then(|| ExecutionOwner::new(host)); if owner.is_some() { ExecutionOwner::install_panic_hook(); } @@ -739,12 +749,14 @@ where Err(_) => return TB_PANICKED, }; unsafe { - start_module( + start_module_runtime( host, out, worker_threads, detach_on_panic, move |connection| setup(connection, config), + None, + caller_owner, ) } } @@ -970,7 +982,7 @@ where detach_on_panic, move |connection| initial_setup(connection, config), Some(reinitialize_factory), - linked, + caller_owner, ) } } @@ -990,7 +1002,15 @@ where /// state is reused for that instance's reinitializations, allowing admission to /// remain bound without process-global state. Separate instances receive fresh /// state. A factory panic returns `TB_PANICKED` without unwinding into the host. -/// Existing `setup = function` callbacks retain their behavior. +/// Configured modules may instead use `terminal_setup_factory = factory`. Its +/// factory receives [`NativeInstanceLiveness`] and returns `Result<(F, +/// RuntimeHooks)>`, with the same setup callback bounds. A supported, exactly +/// live native query and an available instance slot are required before factory +/// evaluation; otherwise initialization returns `TB_CLOSED`. The factory owns +/// partial-failure cleanup until it returns and must launch independent workers +/// only from setup after hook registration. See [`RuntimeHooks`] for controller +/// obligations. Hooks are registered once and never replaced by reinitialization. +/// Existing `setup = function` and `setup_factory` callbacks retain their behavior. #[macro_export] macro_rules! module_export { (@linked) => { @@ -1128,6 +1148,17 @@ macro_rules! module_export { unsafe { $start(host, out, $threads, true, $setup) } } }; + ( + terminal_setup_factory = $factory:path, + config = $config:ty, + $($rest:tt)* + ) => { + $crate::module_export! { + @configured export = {#[unsafe(no_mangle)]}, + start = $crate::start_terminal_reconfigurable_module_with_factory, + setup = $factory, config = $config, $($rest)* + } + }; ( setup_factory = $factory:path, config = $config:ty, @@ -1166,9 +1197,19 @@ macro_rules! module_export { /// Generate Rust-addressable entry points for linking several modules into a /// single executable. The entry points have no shared C linker symbol names. -/// Both configured `setup` and per-instance `setup_factory` forms are supported. +/// Configured `setup`, per-instance `setup_factory`, and opted-in +/// `terminal_setup_factory` forms are supported; see [`module_export`]. +/// Terminal factories require the supplied host's native query even when linked. #[macro_export] macro_rules! module_export_static { + (terminal_setup_factory = $factory:path, config = $config:ty, $($rest:tt)*) => { + $crate::module_export! { + @configured export = {}, + start = $crate::start_linked_terminal_reconfigurable_module_with_factory, + setup = $factory, config = $config, $($rest)* + } + $crate::module_export! { @linked } + }; (setup_factory = $factory:path, config = $config:ty, $($rest:tt)*) => { $crate::module_export! { @configured export = {}, @@ -1230,3 +1271,637 @@ mod setup_factory_test; #[cfg(test)] #[path = "lib_lifecycle_tests.rs"] mod lifecycle_test; + +/// Initialize an opted-in instance with native admission and physical hooks. +/// +/// The factory returns one shared setup callback and one controller hook pair. +/// It owns partial-failure disposal until return and must defer worker launch +/// until setup, after hook registration. Reinitialization reuses this controller. +/// Admission exhaustion or an unsupported/non-live query returns `TB_CLOSED` +/// before factory evaluation. [`RuntimeHooks`] defines physical finish obligations. +/// +/// # Safety +/// Uphold [`start_module`]'s ABI and output contracts. The compatible native +/// query and copied host context must remain valid for every retained view, +/// including after host destruction. Only the actual native query grants +/// authority; unsupported hosts reject the factory before invoking it. +#[doc(hidden)] +pub unsafe fn start_terminal_reconfigurable_module_with_factory( + host: *const TbHostVtable, + out: *mut TbModuleVtable, + worker_threads: usize, + _detach_on_panic: bool, + factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, +) -> i32 +where + C: serde::de::DeserializeOwned + Send + 'static, + F: Fn(Connection, C) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, +{ + unsafe { start_terminal_factory_mode(host, out, worker_threads, factory, false) } +} + +/// Linked counterpart with explicit native authority from the supplied host. +/// It retains the embedding process's tracing subscriber and panic hook. +/// +/// # Safety +/// Uphold [`start_terminal_reconfigurable_module_with_factory`]'s contracts. +#[doc(hidden)] +pub unsafe fn start_linked_terminal_reconfigurable_module_with_factory( + host: *const TbHostVtable, + out: *mut TbModuleVtable, + worker_threads: usize, + _detach_on_panic: bool, + factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, +) -> i32 +where + C: serde::de::DeserializeOwned + Send + 'static, + F: Fn(Connection, C) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, +{ + unsafe { start_terminal_factory_mode(host, out, worker_threads, factory, true) } +} + +// Legacy transport remains unchanged; only opted-in Connections retain State. +struct TerminalTransport { + inner: ModuleTransport, + _owner: terminal::SdkOwner, +} +#[cfg(any(test, feature = "__native-fixture-support"))] +impl Drop for TerminalTransport { + fn drop(&mut self) { + self._owner.record_transport_drop_for_test(); + } +} +#[async_trait] +impl Transport for TerminalTransport { + async fn send(&self, message: Message) -> Result<()> { + self.inner.send(message).await + } + async fn recv(&self) -> Result> { + self.inner.recv().await + } + async fn close(&self) -> Result<()> { + self.inner.close().await + } + fn describe(&self) -> String { + self.inner.describe() + } +} + +// The existing opaque module_ctx field is a bounded slot/generation token, +// never a dereferenced pointer. Retirement denies old callbacks permanently. +unsafe extern "C" fn terminal_deliver(ctx: *mut c_void, ptr: *const u8, len: usize) -> i32 { + if ptr.is_null() || len > MAX_FRAME_LEN { + return TB_BAD_ARGUMENT; + } + let Some(state) = terminal::TerminalState::lookup(ctx) else { + return TB_CLOSED; + }; + let _scope = state.owner.as_ref().map(ExecutionOwner::enter); + if !state.is_live() { + return TB_CLOSED; + } + let bytes = unsafe { std::slice::from_raw_parts(ptr, len) }.to_vec(); + let brokered = state.host.is_broker_routed(); + let sender = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .inbound + .clone(); + match sender.map(|sender| sender.try_send((bytes, brokered))) { + Some(Ok(())) => TB_OK, + Some(Err(mpsc::error::TrySendError::Full(_))) => TB_BACKPRESSURE, + _ => TB_CLOSED, + } +} +unsafe extern "C" fn terminal_shutdown(ctx: *mut c_void, deadline_ms: u64) -> i32 { + let Some(state) = terminal::TerminalState::lookup(ctx) else { + return TB_CLOSED; + }; + let _scope = state.owner.as_ref().map(ExecutionOwner::enter); + state.observe_host(); // sample actual host Fault before selecting ordinary Stop + state.signal(TerminalReason::Stop); + let completion = state.wait(Duration::from_millis(deadline_ms)); + // Genuine source/native Fault cannot earn accepted-vtable physical success. + state.observe_host(); + if completion != TerminalCompletion::PhysicallyStopped || state.has_fault() { + state.retain_uncertain_anchor(); + return TB_PANICKED; + } + TB_OK +} +unsafe extern "C" fn terminal_reinitialize(ctx: *mut c_void, ptr: *const u8, len: usize) -> i32 { + if len > 1024 * 1024 || (ptr.is_null() && len != 0) { + return TB_BAD_ARGUMENT; + } + let Some(state) = terminal::TerminalState::lookup(ctx) else { + return TB_CLOSED; + }; + if !state.is_live() { + return TB_CLOSED; + } + let _scope = state.owner.as_ref().map(ExecutionOwner::enter); + if !state.begin_reinit() { + return TB_CLOSED; + } + let callback_slot = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reinitialize + .clone(); + // The authoritative Arc remains State-owned until active_reinit settles. + // No callback exclusion is held across foreign invocation or source faults. + let callback = callback_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let outcome = catch_unwind(AssertUnwindSafe(|| { + callback + .as_ref() + .map_or(TB_CLOSED, |callback| callback(ptr, len)) + })); + drop(callback); // non-last: cleanup still waits active_reinit + let code = match outcome { + Ok(code) => code, + Err(payload) => { + state.source_panic(terminal::PayloadKind::Reinit, payload); + TB_PANICKED + } + }; + state.end_reinit(); + code +} + +// The task never owns the foreign future across a yield or on cancellation. +// Each poll takes/polls/reattaches without holding a resource mutex over user +// code. Physical cleanup joins the sole task, then destroys this retained future. +struct TerminalSetupPoll { + slot: terminal::SetupFutureSlot, + state: Arc, + kind: terminal::PayloadKind, +} +impl Future for TerminalSetupPoll { + type Output = Result<()>; + fn poll( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + let mut future = self + .slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let outcome = catch_unwind(AssertUnwindSafe(|| { + future.as_mut().map_or( + std::task::Poll::Ready(Err(Error::ConnectionClosed)), + |future| future.as_mut().poll(cx), + ) + })); + *self + .slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = future; + match outcome { + Ok(std::task::Poll::Ready(Err(error))) => { + self.state.signal(TerminalReason::SetupFailed); + std::task::Poll::Ready(Err(error)) + } + Ok(poll) => poll, + Err(payload) => { + self.state.source_panic(self.kind, payload); + std::task::Poll::Ready(Err(Error::failed("module setup panicked"))) + } + } + } +} +async fn terminal_notification(state: &terminal::TerminalState) { + loop { + let notification = state.terminal_notify.notified(); + tokio::pin!(notification); + notification.as_mut().enable(); + if state.is_terminal() { + return; + } + notification.await; + } +} + +unsafe fn start_terminal_factory_mode( + host: *const TbHostVtable, + out: *mut TbModuleVtable, + worker_threads: usize, + factory: impl FnOnce(NativeInstanceLiveness) -> Result<(F, RuntimeHooks)>, + linked: bool, +) -> i32 +where + C: serde::de::DeserializeOwned + Send + 'static, + F: Fn(Connection, C) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, +{ + let Some((capacity, compatible)) = + (unsafe { validate_start_arguments(host, out, worker_threads) }) + else { + return TB_BAD_ARGUMENT; + }; + // Reject unavailable, unknown and terminal authority before any factory. + let Some(query) = compatible.native_terminal_state else { + return TB_CLOSED; + }; + if unsafe { query(compatible.host_ctx) } != tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE { + return TB_CLOSED; + } + let owner = (!linked).then(|| ExecutionOwner::new(HostCalls(compatible))); + let Some(state) = terminal::TerminalState::admit(HostCalls(compatible), owner) else { + return TB_CLOSED; + }; + if state.owner.is_some() { + ExecutionOwner::install_panic_hook(); + } + let _scope = state.owner.as_ref().map(ExecutionOwner::enter); + let result = catch_unwind(AssertUnwindSafe(|| { + let config = match unsafe { parse_config::(host) } { + Ok(config) => config, + Err(code) => return code, + }; + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config = Some(Box::new(config)); + let (setup, hooks) = match catch_unwind(AssertUnwindSafe(|| factory(state.view()))) { + Ok(Ok(returned)) => returned, + Ok(Err(_error)) => { + state.signal(TerminalReason::SetupFailed); + return TB_CLOSED; + } + Err(payload) => { + state.source_panic(terminal::PayloadKind::Factory, payload); + return TB_PANICKED; + } + }; + let setup = Arc::new(setup); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retained_factory = Some(Box::new(setup.clone())); + state.register(hooks); + #[cfg(any(test, feature = "__native-fixture-support"))] + state.startup_checkpoint_for_test(1); + if !state.is_live() { + return TB_CLOSED; + } + let config = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config + .take(); + let config = *config + .expect("attached config") + .downcast::() + .unwrap_or_else(|_| unreachable!("private config type")); + let initial_setup = setup.clone(); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .setup = Some(Box::new(move |connection| { + Box::pin(initial_setup(connection, config)) + })); + #[cfg(any(test, feature = "__native-fixture-support"))] + state.startup_checkpoint_for_test(2); + let weak = Arc::downgrade(&state); + let reinitializer: ReinitializeFactory = Box::new(move |handle, connection_slot| { + Box::new(move |ptr, len| { + let Some(state) = weak.upgrade() else { + return TB_CLOSED; + }; + if !state.is_live() { + return TB_CLOSED; + } + let connection = connection_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let Some(connection) = connection else { + return TB_CLOSED; + }; + let slot = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reinit_future + .clone(); + let previous = slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + if !state.retire_reinit(previous) { + return TB_CLOSED; + } + if !state.is_live() { + return TB_CLOSED; + } + // Copy ABI input before transferring the single construction operation. + // Only the existing physical worker parses/owns C and calls setup. + let bytes = if len == 0 { + b"{}".to_vec() + } else { + unsafe { std::slice::from_raw_parts(ptr, len) }.to_vec() + }; + let construct_state = Arc::downgrade(&state); + let construct_setup = setup.clone(); + let construct_slot = slot.clone(); + let construct_handle = handle.clone(); + let code = state.construct_reinit(Box::new(move || { + let Some(state) = construct_state.upgrade() else { + return TB_CLOSED; + }; + if !state.is_live() { + return TB_CLOSED; + } + // active_reinit keeps the actual State-owned Runtime alive until + // this worker acknowledges construction and the ABI caller settles. + // Enter its context locally, outside all locks, for synchronous C/F + // construction; the guard never leaves this physical worker. + let _runtime_scope = construct_handle.enter(); + let config = match serde_json::from_slice::(&bytes) { + Ok(config) => config, + Err(_) => return TB_BAD_ARGUMENT, + }; + // If Stop arrived during parsing, keep the received config attached + // until this same worker's terminal resource disposal. + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config = Some(Box::new(config)); + if !state.is_live() { + return TB_CLOSED; + } + let config = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config + .take() + .expect("attached reinit config"); + let config = *config + .downcast::() + .unwrap_or_else(|_| unreachable!("private config type")); + let future = Box::pin(construct_setup(connection, config)); + *construct_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); + TB_OK + })); + if code != TB_OK { + return code; + } + let result = handle.block_on(async { + tokio::select! { biased; + result = TerminalSetupPoll { slot, state: state.clone(), kind: terminal::PayloadKind::Reinit } => result, + () = terminal_notification(&state) => Err(Error::ConnectionClosed), + } + }); + match result { + Ok(()) if state.is_live() => TB_OK, + Ok(()) | Err(_) => TB_CLOSED, + } + }) + }); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reinitializer_factory = Some(reinitializer); + #[cfg(any(test, feature = "__native-fixture-support"))] + state.startup_checkpoint_for_test(3); + let mut builder = tokio::runtime::Builder::new_multi_thread(); + if let Some(owner) = &state.owner { + owner.configure(&mut builder); + } + #[cfg(any(test, feature = "__native-fixture-support"))] + let build_result = if state.reject_runtime_build_for_fixture() { + Err(std::io::Error::other( + "controlled terminal runtime build failure", + )) + } else { + builder.worker_threads(worker_threads).enable_all().build() + }; + #[cfg(not(any(test, feature = "__native-fixture-support")))] + let build_result = builder.worker_threads(worker_threads).enable_all().build(); + let runtime = match build_result { + Ok(runtime) => runtime, + Err(_) => { + state.signal(TerminalReason::SetupFailed); + return TB_CLOSED; + } + }; + // Immediate mechanical transfer: no later startup failure drops Runtime on ABI. + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .runtime = Some(runtime); + #[cfg(any(test, feature = "__native-fixture-support"))] + state.startup_checkpoint_for_test(4); + let (inbound_tx, inbound_rx) = mpsc::channel(MODULE_QUEUE_CAPACITY); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .inbound = Some(inbound_tx); + let transport = Arc::new(TerminalTransport { + inner: ModuleTransport { + host: state.host, + inbound: Mutex::new(inbound_rx), + provenance: Arc::new(AtomicBool::new(false)), + detach_on_panic: false, + }, + _owner: state.sdk_owner(), + }); + // Attach before any later fallible startup boundary. The ABI/task local + // is never the sole owner of the received SDK transport. + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .transport = Some(transport.clone()); + #[cfg(any(test, feature = "__native-fixture-support"))] + state.startup_checkpoint_for_test(5); + let (handle, connection_slot, factory) = { + let mut resources = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ( + resources + .runtime + .as_ref() + .expect("attached runtime") + .handle() + .clone(), + resources.connection.clone(), + resources.reinitializer_factory.take(), + ) + }; + let callback = factory.map(|factory| { + terminal::TerminalReinitialize::from(factory(handle.clone(), connection_slot.clone())) + }); + let callback_slot = state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reinitialize + .clone(); + *callback_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = callback; + #[cfg(any(test, feature = "__native-fixture-support"))] + state.startup_checkpoint_for_test(6); + let task_state = state.clone(); + let task = handle.spawn(async move { + #[cfg(any(test, feature = "__native-fixture-support"))] + if task_state.hold_initial_before_transfer_for_test() { + // Deterministic startup-failure fixture: retain SDK-owned C/F + // before trusted setup consumes them, with a real cancellable task. + terminal_notification(&task_state).await; + return Ok(()); + } + let provenance = transport.inner.provenance.clone(); + let brokered = transport.inner.host.0.broker_routing.is_some(); + let connection = if brokered { + // SAFETY: same private per-frame provenance contract as module_connection. + unsafe { Connection::__attach_module_with_provenance(transport, provenance) } + } else { + Connection::attach(transport) + }; + if let Err(error) = connection.handshake().await { + task_state.signal(TerminalReason::SetupFailed); + return Err(error); + } + connection.__set_panic_handler(Arc::new(|| Error::MethodFailed { + name: MODULE_PANIC_ERROR.to_owned(), + message: "a module method panicked".to_owned(), + })); + let observer_owner = task_state.sdk_owner(); + connection.__set_catch_observer(Arc::new(move || { + observer_owner.caught_panic(); + })); + // Connection is persisted independently of initial task lifetime. + *connection_slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(connection.clone()); + if !task_state.is_live() { + return Ok(()); + } + let setup = task_state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .setup + .take(); + let future = + match catch_unwind(AssertUnwindSafe(|| setup.map(|setup| setup(connection)))) { + Ok(Some(future)) => future, + Ok(None) => return Err(Error::ConnectionClosed), + Err(payload) => { + task_state.source_panic(terminal::PayloadKind::Initial, payload); + return Err(Error::failed("module setup construction panicked")); + } + }; + let slot = task_state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .initial_future + .clone(); + *slot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(future); + let result = TerminalSetupPoll { + slot, + state: task_state.clone(), + kind: terminal::PayloadKind::Initial, + } + .await; + if result.is_ok() && task_state.is_live() { + task_state.host.ready(); + } + result + }); + state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .initial = Some(task); + #[cfg(any(test, feature = "__native-fixture-support"))] + state.startup_checkpoint_for_test(7); + let context = state.context(); + #[cfg(any(test, feature = "__native-fixture-support"))] + state.startup_checkpoint_for_test(8); + state.mark_vtable_published(); + if capacity >= size_of::() as u32 { + unsafe { + out.write(TbModuleVtable { + size: size_of::() as u32, + _reserved: 0, + module_ctx: context, + deliver: terminal_deliver, + shutdown: terminal_shutdown, + reinitialize: Some(terminal_reinitialize), + }); + } + } else { + unsafe { + out.cast::() + .write(TbModuleVtableV1Prefix { + size: TB_MODULE_VTABLE_BASE_SIZE, + _reserved: 0, + module_ctx: context, + deliver: terminal_deliver, + shutdown: terminal_shutdown, + }); + } + } + TB_OK + })); + let code = match result { + Ok(code) => code, + Err(payload) => { + state.source_panic(terminal::PayloadKind::Startup, payload); + TB_PANICKED + } + }; + if code == TB_BAD_ARGUMENT + && state + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config + .is_none() + { + // Parsing failed before any factory/hook/config owner transfer. + state.empty_factory_failure(); + return code; + } + if code != TB_OK { + if let Some(owner) = &state.owner { + owner.stop(); + } + if !state.is_terminal() { + state.signal(TerminalReason::SetupFailed); + } + } + state.publish(); + code +} + +#[cfg(test)] +#[path = "lib_terminal_tests.rs"] +mod terminal_runtime_tests; diff --git a/crates/tinybus-module/src/lib_terminal_tests.rs b/crates/tinybus-module/src/lib_terminal_tests.rs new file mode 100644 index 00000000..796e3368 --- /dev/null +++ b/crates/tinybus-module/src/lib_terminal_tests.rs @@ -0,0 +1,3262 @@ +use super::*; +use std::sync::atomic::AtomicUsize; +use tinybus::module::abi::TbSlice; +unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { + TB_OK +} +unsafe extern "C" fn notify(_: *mut c_void) {} +unsafe extern "C" fn fault(_: *mut c_void, _: *const u8, _: usize) {} +unsafe extern "C" fn log(_: *mut c_void, _: u32, _: *const u8, _: usize) {} +fn host() -> TbHostVtable { + TbHostVtable { + size: size_of::() as u32, + _reserved: 0, + host_ctx: std::ptr::null_mut(), + send, + wake: notify, + log, + ready: notify, + fault, + config: TbSlice { + ptr: std::ptr::null(), + len: 0, + }, + broker_routing: None, + native_terminal_state: None, + } +} +fn output() -> TbModuleVtable { + TbModuleVtable { + size: size_of::() as u32, + _reserved: 0, + module_ctx: std::ptr::null_mut(), + deliver, + shutdown, + reinitialize: None, + } +} +#[test] +fn terminal_callbacks_reject_zero_and_unknown_context_tokens_as_closed() { + let bytes = b"{}"; + for address in [0, 1, 255, usize::MAX] { + let ctx = std::ptr::with_exposed_provenance_mut(address); + let results = unsafe { + [ + ( + "deliver", + terminal_deliver(ctx, bytes.as_ptr(), bytes.len()), + ), + ("shutdown", terminal_shutdown(ctx, 1)), + ( + "reinitialize", + terminal_reinitialize(ctx, bytes.as_ptr(), bytes.len()), + ), + ] + }; + assert_eq!( + results, + [ + ("deliver", TB_CLOSED), + ("shutdown", TB_CLOSED), + ("reinitialize", TB_CLOSED), + ], + "context token {address} with otherwise valid arguments" + ); + } +} + +#[test] +fn legacy_callbacks_keep_null_context_bad_argument() { + let bytes = b"{}"; + let ctx = std::ptr::null_mut(); + let mut out = output(); + let results = unsafe { + [ + ( + "init", + start_module(std::ptr::null(), &mut out, 1, false, |_| async { Ok(()) }), + ), + ("deliver", deliver(ctx, bytes.as_ptr(), bytes.len())), + ("shutdown", shutdown(ctx, 1)), + ( + "reinitialize", + reinitialize(ctx, bytes.as_ptr(), bytes.len()), + ), + ] + }; + assert_eq!( + results, + [ + ("init", TB_BAD_ARGUMENT), + ("deliver", TB_BAD_ARGUMENT), + ("shutdown", TB_BAD_ARGUMENT), + ("reinitialize", TB_BAD_ARGUMENT), + ] + ); +} + +#[test] +fn terminal_callbacks_keep_invalid_frame_and_config_bad_argument() { + let ctx = std::ptr::null_mut(); + let bytes = b"{}"; + let results = unsafe { + [ + terminal_deliver(ctx, std::ptr::null(), 0), + terminal_deliver(ctx, bytes.as_ptr(), MAX_FRAME_LEN + 1), + terminal_reinitialize(ctx, std::ptr::null(), 1), + terminal_reinitialize(ctx, bytes.as_ptr(), 1024 * 1024 + 1), + ] + }; + assert_eq!(results, [TB_BAD_ARGUMENT; 4]); +} + +#[test] +fn native_query_is_required_before_terminal_factory_evaluation() { + use tinybus::module::abi::*; + // Exactly LIVE is the sole admitted value. Missing, empty, terminal, + // combined and unknown flags must not evaluate arbitrary factory code. + for flags in [ + None, + Some(0), + Some(TB_NATIVE_TERMINAL_STOP_INTENT), + Some(TB_NATIVE_TERMINAL_FAULT), + Some(TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED), + Some(TB_NATIVE_TERMINAL_LIVE | TB_NATIVE_TERMINAL_STOP_INTENT), + Some(TB_NATIVE_TERMINAL_LIVE | TB_NATIVE_TERMINAL_FAULT), + Some(TB_NATIVE_TERMINAL_LIVE | (1 << 31)), + Some(u32::MAX), + ] { + let native = std::sync::atomic::AtomicU32::new(flags.unwrap_or(0)); + let mut host = host(); + host.host_ctx = (&native as *const std::sync::atomic::AtomicU32) + .cast_mut() + .cast(); + host.native_terminal_state = + flags.map(|_| native_query as unsafe extern "C" fn(*mut c_void) -> u32); + let calls = AtomicUsize::new(0); + let mut out = output(); + let code = unsafe { + start_terminal_reconfigurable_module_with_factory::( + &host, + &mut out, + 1, + true, + |_| { + calls.fetch_add(1, Ordering::SeqCst); + Ok(( + |_: Connection, _: serde_json::Value| async { Ok(()) }, + RuntimeHooks::new(|_| {}, || TerminalCompletion::PhysicallyStopped), + )) + }, + ) + }; + assert_eq!(code, TB_CLOSED, "query {flags:?}"); + assert_eq!(calls.load(Ordering::SeqCst), 0, "query {flags:?}"); + assert!( + out.module_ctx.is_null(), + "denied init cannot publish output" + ); + } +} + +#[test] +fn invalid_initial_config_rejects_before_factory_or_output_publication() { + let native = std::sync::atomic::AtomicU32::new(tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE); + let bytes = b"{"; + let mut host = host(); + host.host_ctx = (&native as *const std::sync::atomic::AtomicU32) + .cast_mut() + .cast(); + host.native_terminal_state = Some(native_query); + host.config = TbSlice { + ptr: bytes.as_ptr(), + len: bytes.len(), + }; + let calls = AtomicUsize::new(0); + let mut out = output(); + let code = unsafe { + start_terminal_reconfigurable_module_with_factory::( + &host, + &mut out, + 1, + true, + |_| { + calls.fetch_add(1, Ordering::SeqCst); + Ok(( + |_: Connection, _: serde_json::Value| async { Ok(()) }, + RuntimeHooks::new(|_| {}, || TerminalCompletion::PhysicallyStopped), + )) + }, + ) + }; + assert_eq!(code, TB_BAD_ARGUMENT); + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert!(out.module_ctx.is_null()); +} + +unsafe extern "C" fn native_query(ctx: *mut c_void) -> u32 { + unsafe { &*ctx.cast::() }.load(Ordering::Acquire) +} +#[test] +fn shutdown_rechecks_actual_native_fault_after_local_stop_and_physical_finish() { + static FLAGS: std::sync::atomic::AtomicU32 = + std::sync::atomic::AtomicU32::new(tinybus::module::abi::TB_NATIVE_TERMINAL_LIVE); + let flags = &FLAGS; + let mut host = host(); + host.host_ctx = (flags as *const std::sync::atomic::AtomicU32) + .cast_mut() + .cast(); + host.native_terminal_state = Some(native_query); + let state = terminal::TerminalState::admit(HostCalls(host), None).unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let release = Arc::new(std::sync::Barrier::new(2)); + let worker_release = release.clone(); + state.register(RuntimeHooks::new( + |_| {}, + move || { + entered_tx.send(()).unwrap(); + worker_release.wait(); + TerminalCompletion::PhysicallyStopped + }, + )); + state.mark_vtable_published(); + state.publish(); + let ptr = state.context().addr(); + let caller = std::thread::spawn(move || unsafe { terminal_shutdown(ptr as *mut c_void, 3000) }); + entered_rx.recv().unwrap(); + flags.store( + tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, + Ordering::Release, + ); + release.wait(); + let code = caller.join().unwrap(); + assert_eq!( + code, TB_PANICKED, + "actual native Fault cannot disappear behind local Stop" + ); + drop(state); + assert!( + terminal::TerminalState::lookup(std::ptr::with_exposed_provenance_mut(ptr)).is_some(), + "published Fault must retain the exact anchor instead of refunding its permit" + ); +} + +mod controlled { + use super::*; + use std::pin::Pin; + use std::task::{Context, Poll}; + #[derive(serde::Deserialize)] + pub struct Config { + pub mode: u8, + #[serde(default, deserialize_with = "parse_gate")] + #[serde(rename = "parse_gate")] + _parse_gate: bool, + #[serde(default)] + record_drop: bool, + #[serde(skip, default = "config_fixture")] + fixture: std::sync::Weak, + } + static CONFIG_FIXTURE: StdMutex> = + StdMutex::new(std::sync::Weak::new()); + fn config_fixture() -> std::sync::Weak { + CONFIG_FIXTURE.lock().unwrap().clone() + } + fn parse_gate<'de, D: serde::Deserializer<'de>>( + deserializer: D, + ) -> std::result::Result { + let enabled = ::deserialize(deserializer)?; + if enabled { + let fixture = config_fixture().upgrade().unwrap(); + let barrier = fixture.config_parse_control.lock().unwrap().take().unwrap(); + fixture.events.send(Event::ConfigParsing).unwrap(); + barrier.wait(); + } + Ok(enabled) + } + impl Drop for Config { + fn drop(&mut self) { + if self.record_drop { + let Some(fixture) = self.fixture.upgrade() else { + return; + }; + fixture.config_drops.fetch_add(1, Ordering::SeqCst); + let _ = fixture.events.send(Event::ConfigDrop( + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned(), + )); + } + } + } + #[derive(Debug)] + pub enum Event { + Poll(u8), + Request, + FutureDrop(String), + ConfigDrop(String), + ConfigParsing, + TimerConstruct(std::thread::ThreadId, String, tokio::runtime::Id), + TimerDrop(Option), + Finish, + FactoryDrop(String), + } + pub struct Fixture { + pub events: std::sync::mpsc::Sender, + pub view: StdMutex>, + wake: (StdMutex, std::sync::Condvar), + worker: StdMutex>>, + pub requests: AtomicUsize, + pub joins: AtomicUsize, + pub workers_started: AtomicUsize, + pub finishes: AtomicUsize, + pub hook_failure: AtomicUsize, + pub factory_failure: AtomicUsize, + pub startup_failure: AtomicUsize, + pub payload_drops: AtomicUsize, + pub config_drops: AtomicUsize, + pub config_parse_control: StdMutex>>, + pub drop_off_lock: AtomicBool, + pub request_off_lock: AtomicBool, + pub future_drop_control: StdMutex, bool)>>, + } + static NEXT: StdMutex>> = StdMutex::new(None); + pub static TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + pub fn fixture() -> (Arc, std::sync::mpsc::Receiver) { + let (events, receiver) = std::sync::mpsc::channel(); + let fixture = Arc::new(Fixture { + events, + view: StdMutex::new(None), + wake: (StdMutex::new(false), std::sync::Condvar::new()), + worker: StdMutex::new(None), + requests: AtomicUsize::new(0), + joins: AtomicUsize::new(0), + workers_started: AtomicUsize::new(0), + finishes: AtomicUsize::new(0), + hook_failure: AtomicUsize::new(0), + factory_failure: AtomicUsize::new(0), + startup_failure: AtomicUsize::new(0), + payload_drops: AtomicUsize::new(0), + config_drops: AtomicUsize::new(0), + config_parse_control: StdMutex::new(None), + drop_off_lock: AtomicBool::new(true), + request_off_lock: AtomicBool::new(true), + future_drop_control: StdMutex::new(None), + }); + *CONFIG_FIXTURE.lock().unwrap() = Arc::downgrade(&fixture); + *NEXT.lock().unwrap() = Some(fixture.clone()); + (fixture, receiver) + } + struct HookPayload(Arc); + impl Drop for HookPayload { + fn drop(&mut self) { + self.0.payload_drops.fetch_add(1, Ordering::SeqCst); + } + } + impl Fixture { + pub fn settle_rejected_worker(&self) { + *self.wake.0.lock().unwrap() = true; + self.wake.1.notify_all(); + if let Some(worker) = self.worker.lock().unwrap().take() { + let _ = worker.join().unwrap(); + self.joins.fetch_add(1, Ordering::SeqCst); + } + } + } + struct Capture(Arc); + impl Drop for Capture { + fn drop(&mut self) { + let _ = self.0.events.send(Event::FactoryDrop( + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned(), + )); + } + } + struct SetupFuture { + fixture: Arc, + mode: u8, + first: bool, + timer: Option>>, + } + impl Future for SetupFuture { + type Output = Result<()>; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.first { + self.first = false; + self.fixture.events.send(Event::Poll(self.mode)).unwrap(); + } + match self.mode { + 0 => Poll::Ready(Ok(())), + 5 => self + .timer + .as_mut() + .unwrap() + .as_mut() + .poll(cx) + .map(|()| Ok(())), + 2 => panic!("controlled setup poll"), + 3 => Poll::Ready(Err(Error::failed("controlled setup failure"))), + _ => Poll::Pending, + } + } + } + impl Drop for SetupFuture { + fn drop(&mut self) { + if self.mode == 5 { + let _ = self.fixture.events.send(Event::TimerDrop( + tokio::runtime::Handle::try_current() + .ok() + .map(|handle| handle.id()), + )); + } + if let Some(view) = self.fixture.view.lock().unwrap().as_ref() { + let slot = view.inner.resources.lock().unwrap().reinitialize.clone(); + if slot.try_lock().is_err() { + self.fixture.drop_off_lock.store(false, Ordering::Release); + } + } + let _ = self.fixture.events.send(Event::FutureDrop( + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned(), + )); + let control = self.fixture.future_drop_control.lock().unwrap().take(); + if let Some((release, panic)) = control { + release.wait(); + if panic { + panic!("controlled prior reinit future Drop"); + } + } + } + } + type FixtureSetup = Box terminal::SetupFuture + Send + Sync>; + fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> { + let fixture = NEXT.lock().unwrap().take().unwrap(); + *fixture.view.lock().unwrap() = Some(view.clone()); + view.inner + .reject_startup_for_test(fixture.startup_failure.load(Ordering::Acquire) as u8); + match fixture.factory_failure.load(Ordering::Acquire) { + 1 => { + return Err(Error::failed( + "controlled factory failure before hook transfer", + )); + } + 2 => std::panic::panic_any(HookPayload(fixture.clone())), + _ => {} + } + let (request, finish) = (fixture.clone(), fixture.clone()); + let capture = Capture(fixture.clone()); + let setup = move |_: Connection, config: Config| { + let _ = &capture; + let mut worker = fixture.worker.lock().unwrap(); + if worker.is_none() { + let work = fixture.clone(); + let worker_view = view.clone(); + *worker = Some( + std::thread::Builder::new() + .name("controlled-module-worker".into()) + .spawn(move || { + let mut cancelled = work.wake.0.lock().unwrap(); + while !*cancelled { + cancelled = work.wake.1.wait(cancelled).unwrap(); + } + drop(cancelled); + !worker_view.is_live() + }) + .unwrap(), + ); + fixture.workers_started.fetch_add(1, Ordering::Release); + } + drop(worker); + if config.mode == 4 { + panic!("controlled setup construction"); + } + let timer = (config.mode == 5).then(|| { + // Construct synchronously in F, before returning its future. + let timer = Box::pin(tokio::time::sleep(Duration::from_millis(1))); + fixture + .events + .send(Event::TimerConstruct( + std::thread::current().id(), + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned(), + tokio::runtime::Handle::current().id(), + )) + .unwrap(); + timer + }); + Box::pin(SetupFuture { + fixture: fixture.clone(), + mode: config.mode, + first: true, + timer, + }) as Pin> + Send>> + }; + let hooks = RuntimeHooks::new( + move |_| { + if let Some(view) = request.view.lock().unwrap().as_ref() { + let slot = view.inner.resources.lock().unwrap().reinitialize.clone(); + if slot.try_lock().is_err() { + request.request_off_lock.store(false, Ordering::Release); + } + } + request.requests.fetch_add(1, Ordering::SeqCst); + *request.wake.0.lock().unwrap() = true; + request.wake.1.notify_all(); + request.events.send(Event::Request).unwrap(); + if request.hook_failure.load(Ordering::Acquire) == 1 { + std::panic::panic_any(HookPayload(request.clone())); + } + }, + move || { + let worker = finish.worker.lock().unwrap().take(); + if let Some(worker) = worker { + let denied = worker.join().unwrap(); + finish.joins.fetch_add(1, Ordering::SeqCst); + if !denied { + return TerminalCompletion::Uncertain; + } + } + finish.finishes.fetch_add(1, Ordering::SeqCst); + finish.events.send(Event::Finish).unwrap(); + match finish.hook_failure.load(Ordering::Acquire) { + 2 => std::panic::panic_any(HookPayload(finish.clone())), + 3 => TerminalCompletion::Uncertain, + _ => TerminalCompletion::PhysicallyStopped, + } + }, + ); + Ok((Box::new(setup), hooks)) + } + module_export_static! { + terminal_setup_factory = factory, config = Config, worker_threads = 1, + provides = ["ai.tinyhumans.tinybus.Terminal"], methods = [], signals = [], + requires = [], optional = [], lazy = false, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_posttransfer_startup_panics_keep_received_captures_on_physical_worker() { + let _serial = controlled::TEST_LOCK.lock().await; + // Stages correspond to real attach boundaries in start_terminal_factory_mode: + // returned F/hooks, initial setup, reinit factory, Runtime, inbound/transport, + // installed callback, sole JoinHandle, immediately before output publication. + for stage in 1..=8 { + let (fixture, events) = controlled::fixture(); + fixture.startup_failure.store(stage, Ordering::Release); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + let rejected = host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":1,"record_drop":true}), + ); + let view = fixture.view.lock().unwrap().take().unwrap(); + let completion = view.inner.wait(Duration::from_secs(5)); + host.shutdown(Duration::from_secs(3)).await; + let observations: Vec<_> = events.try_iter().collect(); + let native = view.inner.host.0; + let flags = unsafe { native.native_terminal_state.unwrap()(native.host_ctx) }; + // Actual setup execution is aborted/joined before original finish and + // before received F/C disposal; early stages legitimately launch no work. + assert!(rejected.is_err(), "stage {stage}"); + assert_eq!(completion, TerminalCompletion::Uncertain, "stage {stage}"); + assert!(view.inner.request_done_for_test()); + assert_eq!(fixture.requests.load(Ordering::Acquire), 1, "stage {stage}"); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 1, "stage {stage}"); + assert_eq!( + fixture.config_drops.load(Ordering::Acquire), + 1, + "stage {stage}" + ); + assert!( + observations.iter().any(|event| matches!(event, + controlled::Event::FactoryDrop(name) if name == "tinybus-terminal")), + "stage {stage}" + ); + assert!( + observations + .iter() + .filter_map(|event| match event { + controlled::Event::ConfigDrop(name) | controlled::Event::FutureDrop(name) => + Some(name), + _ => None, + }) + .all(|name| name == "tinybus-terminal"), + "stage {stage}" + ); + assert_eq!( + fixture.joins.load(Ordering::Acquire), + fixture.workers_started.load(Ordering::Acquire), + "stage {stage}" + ); + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert!(terminal::TerminalState::lookup(view.inner.context()).is_some()); + if stage == 5 || stage == 6 { + assert_eq!( + view.inner.transport_drop_thread_for_test().as_deref(), + Some("tinybus-terminal"), + "actual startup transport stage {stage}" + ); + } + let resources = view.inner.resources.lock().unwrap(); + assert!(resources.runtime.is_none()); + assert!(resources.initial.is_none()); + assert!(resources.retained_factory.is_none()); + // Sticky Startup primary keeps exactly this bounded State anchored. + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_factory_failure_disposes_received_config_on_physical_worker() { + let _serial = controlled::TEST_LOCK.lock().await; + for mode in [1, 2] { + let (fixture, events) = controlled::fixture(); + fixture.factory_failure.store(mode, Ordering::Release); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + let rejected = host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":1,"record_drop":true}), + ); + let view = fixture.view.lock().unwrap().take().unwrap(); + let completed = view.inner.wait(Duration::from_secs(5)); + host.shutdown(Duration::from_secs(3)).await; + let observations: Vec<_> = events.try_iter().collect(); + let native = view.inner.host.0; + let flags = unsafe { native.native_terminal_state.unwrap()(native.host_ctx) }; + // No controller hooks have transferred, and no setup task was launched. + // Parsed C already belongs to SDK: its actual Drop must occur on the + // original physical job even though no output vtable was returned. + assert!(rejected.is_err()); + assert_eq!( + completed, + if mode == 1 { + TerminalCompletion::PhysicallyStopped + } else { + TerminalCompletion::Uncertain + } + ); + assert_eq!(fixture.config_drops.load(Ordering::Acquire), 1); + assert_eq!(fixture.requests.load(Ordering::Acquire), 0); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 0); + assert_eq!(fixture.joins.load(Ordering::Acquire), 0); + assert_eq!(fixture.payload_drops.load(Ordering::Acquire), 0); + assert!(observations.iter().all(|event| matches!(event, + controlled::Event::ConfigDrop(name) if name == "tinybus-terminal"))); + assert_eq!(observations.len(), 1); + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0, + "SDK failed-init cleanup cannot fabricate host physical completion" + ); + assert!(!view.is_live()); + if mode == 1 { + assert!(terminal::TerminalState::lookup(view.inner.context()).is_none()); + } else { + assert!(terminal::TerminalState::lookup(view.inner.context()).is_some()); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_hook_panics_and_uncertain_finish_preserve_once_join_and_anchor() { + let _serial = controlled::TEST_LOCK.lock().await; + for mode in [1, 2, 3] { + let (fixture, events) = controlled::fixture(); + fixture.hook_failure.store(mode, Ordering::Release); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":1}), + ) + .unwrap(); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let context = view.inner.context(); + // Real setup has launched the independent controller worker. Actual + // request closes/wakes it, and original finish really joins it even + // when request panic has already made uncertainty sticky. + view.inner.signal(TerminalReason::Stop); + host.shutdown(Duration::from_secs(5)).await; + let completed = view.inner.wait(Duration::from_secs(5)); + let observations: Vec<_> = events.try_iter().collect(); + let native = view.inner.host.0; + let flags = unsafe { native.native_terminal_state.unwrap()(native.host_ctx) }; + // Controlled work and actual original finish/join have settled before + // assertions. Uncertain primary payload/resource ownership stays anchored. + assert!(matches!(entered, controlled::Event::Poll(1))); + assert_eq!(completed, TerminalCompletion::Uncertain, "hook mode {mode}"); + assert!(view.inner.request_done_for_test()); + assert_eq!(fixture.requests.load(Ordering::Acquire), 1); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 1); + assert_eq!(fixture.joins.load(Ordering::Acquire), 1); + assert_eq!(fixture.payload_drops.load(Ordering::Acquire), 0); + assert!(!view.is_live()); + assert!(terminal::TerminalState::lookup(context).is_some()); + if mode != 3 { + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + } + let future_drop = observations + .iter() + .position(|event| { + matches!(event, + controlled::Event::FutureDrop(name) if name == "tinybus-terminal") + }) + .unwrap(); + let finish = observations + .iter() + .position(|event| matches!(event, controlled::Event::Finish)) + .unwrap(); + assert!(future_drop < finish); + // Neither a later cause upgrade nor shutdown retry spends hooks again. + view.request_fault(); + host.shutdown(Duration::from_millis(20)).await; + assert_eq!(fixture.requests.load(Ordering::Acquire), 1); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 1); + assert_eq!(fixture.payload_drops.load(Ordering::Acquire), 0); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_rejected_physical_spawn_retains_received_resources_without_retry() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":1}), + ) + .unwrap(); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let context = view.inner.context(); + view.inner.reject_worker_spawn_for_test(); + view.request_fault(); + host.shutdown(Duration::from_secs(3)).await; + // Submission failure cannot execute original finish. The fixture alone + // releases/joins its independent worker; this is not SDK finish evidence. + fixture.settle_rejected_worker(); + let completed = view.inner.wait(Duration::from_secs(1)); + let observations: Vec<_> = events.try_iter().collect(); + let native = view.inner.host.0; + let flags = unsafe { native.native_terminal_state.unwrap()(native.host_ctx) }; + assert!(matches!(entered, controlled::Event::Poll(1))); + assert_eq!(completed, TerminalCompletion::Uncertain); + assert!(view.inner.request_done_for_test()); + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!(fixture.requests.load(Ordering::Acquire), 1); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 0); + assert_eq!(fixture.joins.load(Ordering::Acquire), 1); + assert!(!observations.iter().any(|event| matches!( + event, + controlled::Event::FactoryDrop(_) + | controlled::Event::FutureDrop(_) + | controlled::Event::Finish + ))); + assert!(!view.is_live()); + drop(host); + view.request_fault(); + assert_eq!( + view.inner.wait(Duration::from_millis(20)), + TerminalCompletion::Uncertain + ); + assert_eq!(fixture.requests.load(Ordering::Acquire), 1); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 0); + let anchored = terminal::TerminalState::lookup(context).unwrap(); + let resources = anchored.resources.lock().unwrap(); + assert!(resources.retained_factory.is_some()); + assert!(resources.runtime.is_some()); + assert!(resources.initial.is_some()); + assert!(resources.initial_future.lock().unwrap().is_some()); + // One deliberately retained, bounded original State/job/Runtime remains. +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_initial_cancellation_drops_future_and_joins_worker_on_physical_thread() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":1}), + ) + .unwrap(); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let shutdown = host.shutdown(Duration::from_secs(5)); + shutdown.await; + let remainder: Vec<_> = events.try_iter().collect(); + let request_count = fixture.requests.load(Ordering::SeqCst); + let joins = fixture.joins.load(Ordering::SeqCst); + // All actual workers/futures settled before asserting behavior. + assert!(matches!(entered, controlled::Event::Poll(1))); + assert_eq!(request_count, 1); + assert_eq!(joins, 1); + assert!(!view.is_live()); + let future_drop = remainder.iter().position(|event| matches!(event, controlled::Event::FutureDrop(name) if name == "tinybus-terminal")).unwrap(); + let finish = remainder + .iter() + .position(|event| matches!(event, controlled::Event::Finish)) + .unwrap(); + assert!(future_drop < finish); + assert!(remainder.iter().any( + |event| matches!(event, controlled::Event::FactoryDrop(name) if name == "tinybus-terminal") + )); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_setup_construction_and_poll_panics_still_finish_real_worker_once() { + let _serial = controlled::TEST_LOCK.lock().await; + for mode in [2, 4] { + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":mode}), + ) + .unwrap(); + // Consume through actual join notification before assertions or host teardown. + let mut seen = Vec::new(); + loop { + let event = events.recv_timeout(Duration::from_secs(5)).unwrap(); + let finished = matches!(event, controlled::Event::Finish); + seen.push(event); + if finished { + break; + } + } + host.shutdown(Duration::from_secs(5)).await; + let view = fixture.view.lock().unwrap().take().unwrap(); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + assert_eq!(fixture.requests.load(Ordering::SeqCst), 1); + assert!(!view.is_live()); + if mode == 2 { + assert!(seen.iter().any(|event| matches!(event, controlled::Event::FutureDrop(name) if name == "tinybus-terminal"))); + } + } +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct NativeControls { + ctx: *mut c_void, + event: unsafe extern "C" fn(*mut c_void, u32, u32), +} +unsafe extern "C" fn native_event(ctx: *mut c_void, event: u32, value: u32) { + let sender = unsafe { &*ctx.cast::>() }; + let _ = sender.send((event, value)); +} +// Only fixture symbols are looked up; TinyBus itself verifies/adopts the pinned +// private artifact through the actual ModuleHost loader first. It remains mapped. +#[cfg(unix)] +unsafe fn terminal_fixture_symbols( + path: &std::path::Path, +) -> ( + extern "C" fn() -> usize, + extern "C" fn(usize) -> i32, + extern "C" fn() -> usize, + unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), + extern "C" fn() -> u32, +) { + use std::os::unix::ffi::OsStrExt; + #[cfg_attr(target_os = "linux", link(name = "dl"))] + unsafe extern "C" { + fn dlopen(path: *const std::ffi::c_char, flags: i32) -> *mut c_void; + fn dlsym(handle: *mut c_void, name: *const std::ffi::c_char) -> *mut c_void; + fn dlclose(handle: *mut c_void) -> i32; + } + let path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + let handle = unsafe { dlopen(path.as_ptr(), 2) }; + assert!(!handle.is_null()); + let context = unsafe { dlsym(handle, c"terminal_test_context".as_ptr()) }; + let probe = unsafe { dlsym(handle, c"terminal_test_old_context".as_ptr()) }; + let drops = unsafe { dlsym(handle, c"terminal_test_drops".as_ptr()) }; + let early = unsafe { dlsym(handle, c"terminal_test_early".as_ptr()) }; + let flags = unsafe { dlsym(handle, c"terminal_test_native_flags".as_ptr()) }; + assert!( + !context.is_null() + && !probe.is_null() + && !drops.is_null() + && !early.is_null() + && !flags.is_null() + ); + unsafe { + dlclose(handle); + } + unsafe { + ( + std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), + std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), + std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops), + std::mem::transmute::< + *mut c_void, + unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), + >(early), + std::mem::transmute::<*mut c_void, extern "C" fn() -> u32>(flags), + ) + } +} +#[cfg(windows)] +unsafe fn terminal_fixture_symbols( + path: &std::path::Path, +) -> ( + extern "C" fn() -> usize, + extern "C" fn(usize) -> i32, + extern "C" fn() -> usize, + unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), + extern "C" fn() -> u32, +) { + use std::os::windows::ffi::OsStrExt; + #[link(name = "kernel32")] + unsafe extern "system" { + fn LoadLibraryW(path: *const u16) -> *mut c_void; + fn GetProcAddress(handle: *mut c_void, name: *const u8) -> *mut c_void; + fn FreeLibrary(handle: *mut c_void) -> i32; + } + let path: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let handle = unsafe { LoadLibraryW(path.as_ptr()) }; + assert!(!handle.is_null()); + let context = unsafe { GetProcAddress(handle, c"terminal_test_context".as_ptr().cast()) }; + let probe = unsafe { GetProcAddress(handle, c"terminal_test_old_context".as_ptr().cast()) }; + let drops = unsafe { GetProcAddress(handle, c"terminal_test_drops".as_ptr().cast()) }; + let early = unsafe { GetProcAddress(handle, c"terminal_test_early".as_ptr().cast()) }; + let flags = unsafe { GetProcAddress(handle, c"terminal_test_native_flags".as_ptr().cast()) }; + assert!( + !context.is_null() + && !probe.is_null() + && !drops.is_null() + && !early.is_null() + && !flags.is_null() + ); + unsafe { + FreeLibrary(handle); + } + unsafe { + ( + std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(context), + std::mem::transmute::<*mut c_void, extern "C" fn(usize) -> i32>(probe), + std::mem::transmute::<*mut c_void, extern "C" fn() -> usize>(drops), + std::mem::transmute::< + *mut c_void, + unsafe extern "C" fn(*mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)), + >(early), + std::mem::transmute::<*mut c_void, extern "C" fn() -> u32>(flags), + ) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a privately pinned TINYBUS_TEST_TERMINAL_MODULE compiled cdylib"] +async fn compiled_native_terminal_owns_initial_cancellation_panics_join_and_stale_context() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let digest = tinybus::module::sha256_file(&path).unwrap(); + let name = path.file_name().unwrap().to_str().unwrap(); + let pins = std::fs::read_to_string(path.parent().unwrap().join("modules.toml")).unwrap(); + assert!(pins.contains(&format!("\"{name}\" = \"{digest}\""))); + let mut old_context = None; + let mut dropped_total = 0; + for mode in [1u8, 0, 2, 4] { + let (events, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + // If a controlled failure cannot prove teardown, conservatively retain + // this one fixture callback owner; never free live foreign context. + let events = Box::into_raw(Box::new(events)); + let controls = NativeControls { + ctx: events.cast(), + event: native_event, + }; + let encoded = unsafe { + std::slice::from_raw_parts( + (&controls as *const NativeControls).cast::(), + size_of::(), + ) + } + .to_vec(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.load_file_with_config( + &path, + serde_json::json!({"mode":mode,"test_controls":encoded}), + ) + .unwrap(); + let (context, old_probe, drops, _, _) = unsafe { terminal_fixture_symbols(&path) }; + let dropped_before = dropped_total; + let actual_context = context(); + let mut seen = Vec::new(); + let actual_entered = if mode == 4 { 16 } else { 2 }; + loop { + let event = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); + seen.push(event); + if event.0 == actual_entered { + break; + } + } + let stale_result = old_context.map(|token| old_probe(token)); + host.shutdown(Duration::from_secs(5)).await; + while !seen.iter().any(|(event, _)| *event == 6) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == dropped_before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + dropped_total = drops(); + unsafe { + drop(Box::from_raw(events)); + } + // All actual controller workers and SDK captures are destroyed before assertions. + if let Some(code) = stale_result { + assert_eq!( + code, TB_CLOSED, + "retired successful context must not act on replacement mode {mode}" + ); + } + // The instrumented fixture compares Arc identity internally. Only + // booleans cross its controls; neither TLS nor these events authorize work. + assert!( + seen.iter() + .any(|(event, bits)| *event == 10 && bits & 1 != 0) + ); + assert!( + seen.iter() + .any(|(event, bits)| *event == 12 && bits & 1 != 0) + ); + assert!( + seen.iter() + .any(|(event, bits)| *event == 13 && bits & 7 == 7) + ); + assert!( + seen.iter() + .any(|(event, bits)| *event == 15 && bits & 7 == 7) + ); + if mode != 4 { + assert!( + seen.iter() + .any(|(event, bits)| *event == 11 && bits & 9 == 9) + ); + assert!( + seen.iter() + .any(|(event, bits)| *event == 14 && bits & 1 != 0) + ); + } + assert_eq!(seen.iter().filter(|(event, _)| *event == 3).count(), 1); + assert_eq!( + seen.iter() + .filter(|(event, value)| *event == 5 && *value == 1) + .count(), + 1 + ); + assert_eq!( + seen.iter() + .filter(|(event, value)| *event == 7 && *value == 1) + .count(), + 1 + ); + assert!(seen.iter().any(|(event, value)| *event == 6 && *value == 1)); + if mode != 4 { + let drop = seen + .iter() + .position(|(event, value)| *event == 4 && *value == 1) + .unwrap(); + let finish = seen.iter().position(|(event, _)| *event == 5).unwrap(); + assert!(drop < finish); + } + if mode == 1 { + old_context = Some(actual_context); + } + } + + // The image was admitted by the real loader above. Symbol lookup releases + // its temporary OS handle: only the canonical loader pins the code now. + let (_, _, drops, early, native_flags) = unsafe { terminal_fixture_symbols(&path) }; + let (sender, receiver) = std::sync::mpsc::channel(); + let controls = Box::into_raw(Box::new(HeldNativeControls { + sender, + release: (std::sync::Mutex::new(false), std::sync::Condvar::new()), + })); + unsafe { + early(controls.cast(), held_native_event); + } + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = tinybus::module::ModuleHost::new(broker); + let result = + host.load_file_with_config(&path, serde_json::json!({"mode":0,"test_controls":[]})); + let mut seen = Vec::new(); + let entered = loop { + match receiver.recv_timeout(Duration::from_secs(5)) { + Ok(event) => { + seen.push(event); + if event.0 == 8 { + break true; + } + } + Err(_) => break false, + } + }; + let flags_before_drop = native_flags(); + drop(host); + let flags_after_host_drop = native_flags(); + // Release the actual physical callback before any assertion, even if + // rejected admission or host bookkeeping disagrees with the contract. + unsafe { + *(*controls) + .release + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = true; + (*controls).release.1.notify_all(); + } + while !seen.iter().any(|event| event.0 == 6) { + match receiver.recv_timeout(Duration::from_secs(5)) { + Ok(event) => seen.push(event), + Err(_) => break, + } + } + let settled = tokio::time::timeout(Duration::from_secs(5), async { + while drops() == dropped_total { + tokio::task::yield_now().await; + } + }) + .await + .is_ok(); + let final_flags = native_flags(); + // On an unresolved controlled failure retain this one callback context. + // Otherwise the Acquire fence proves its final foreign use returned. + if settled { + unsafe { + drop(Box::from_raw(controls)); + } + } + assert!( + result.is_err(), + "source Fault before vtable publication must reject init" + ); + assert!( + entered && settled, + "actual SDK finish and resource Drop must settle independently" + ); + for flags in [flags_before_drop, flags_after_host_drop, final_flags] { + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0, + "no accepted vtable means no host physical-worker evidence" + ); + } + assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); + assert!(seen.contains(&(5, 1))); + assert!(seen.contains(&(6, 1))); +} + +struct HeldNativeControls { + sender: std::sync::mpsc::Sender<(u32, u32)>, + release: (std::sync::Mutex, std::sync::Condvar), +} +unsafe extern "C" fn held_native_event(ctx: *mut c_void, event: u32, value: u32) { + let controls = unsafe { &*ctx.cast::() }; + let _ = controls.sender.send((event, value)); + if event == 8 { + let mut release = controls + .release + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + while !*release { + release = controls + .release + .1 + .wait(release) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_linked_reinit_terminal_cancellation_settles_before_worker_join() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); + let initial = events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let reinit_host = host.clone(); + let reinit = tokio::spawn(async move { + reinit_host + .reinitialize("tinybus-module", serde_json::json!({"mode":1})) + .await + }); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + view.request_fault(); + let reinit_result = reinit.await.unwrap(); + host.shutdown(Duration::from_secs(5)).await; + let mut remainder = Vec::new(); + while !remainder + .iter() + .any(|event| matches!(event, controlled::Event::FactoryDrop(_))) + { + remainder.push(events.recv_timeout(Duration::from_secs(5)).unwrap()); + } + assert!(matches!(initial, controlled::Event::Poll(0))); + assert!(matches!(entered, controlled::Event::Poll(1))); + assert!(reinit_result.is_err()); + assert_eq!(fixture.requests.load(Ordering::SeqCst), 1); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + assert!(!view.is_live()); + let drops: Vec<_> = remainder + .iter() + .enumerate() + .filter_map(|(index, event)| { + matches!(event, controlled::Event::FutureDrop(name) if name == "tinybus-terminal") + .then_some(index) + }) + .collect(); + let finish = remainder + .iter() + .position(|event| matches!(event, controlled::Event::Finish)) + .unwrap(); + assert_eq!(drops.len(), 2); + assert!(drops.into_iter().all(|index| index < finish)); +} + +struct MethodPayloadGate { + state: std::sync::Weak, + entered: std::sync::mpsc::Sender<(u32, bool, usize, tokio::runtime::Id)>, + release: Arc, + settled: std::sync::mpsc::Sender<()>, +} +struct BlockingMethodPayload(Arc); +impl Drop for BlockingMethodPayload { + fn drop(&mut self) { + let state = self.0.state.upgrade().unwrap(); + let host = state.host.0; + let flags = unsafe { host.native_terminal_state.unwrap()(host.host_ctx) }; + // This sample is the first foreign payload-disposal action. It reads + // actual copied native authority and the real registered request state. + self.0 + .entered + .send(( + flags, + state.request_done_for_test(), + state.sdk_owner_count_for_test(), + tokio::runtime::Handle::current().id(), + )) + .unwrap(); + drop(state); + self.0.release.wait(); + self.0.settled.send(()).unwrap(); + } +} +struct MethodPayloadObject(Arc); +#[async_trait] +impl tinybus::Interface for MethodPayloadObject { + fn name(&self) -> tinybus::InterfaceName { + "ai.tinyhumans.tinybus.Payload".parse().unwrap() + } + fn members(&self) -> Vec { + vec!["Panic".parse().unwrap()] + } + async fn call( + &self, + _: &tinybus::MemberName, + _: serde_json::Value, + ) -> Result { + std::panic::panic_any(BlockingMethodPayload(self.0.clone())); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_method_observer_fences_native_fault_and_request_done_before_blocked_payload_drop() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + let bus = tinybus::transport::memory::MemoryBus::new(); + broker.clone().spawn(bus.clone()); + let client = Connection::connect(bus.connect().await.unwrap()) + .await + .unwrap(); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (settled_tx, settled_rx) = std::sync::mpsc::channel(); + let release = Arc::new(std::sync::Barrier::new(2)); + let gate = Arc::new(MethodPayloadGate { + state: Arc::downgrade(&view.inner), + entered: entered_tx, + release: release.clone(), + settled: settled_tx, + }); + let (connection, module_runtime) = { + let resources = view.inner.resources.lock().unwrap(); + let connection = resources.connection.lock().unwrap().clone().unwrap(); + ( + connection, + resources.runtime.as_ref().unwrap().handle().id(), + ) + }; + let destination = connection.unique_name().unwrap().to_string(); + connection + .serve_at("/payload".parse().unwrap(), MethodPayloadObject(gate)) + .await + .unwrap(); + drop(connection); + let call = tokio::spawn(async move { + client + .proxy(&destination, "/payload", "ai.tinyhumans.tinybus.Payload") + .unwrap() + .with_timeout(Duration::from_secs(8)) + .call::("Panic", ()) + .await + }); + let sample = entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + // Actual method task is held in foreign payload Drop. Its Connection owner + // survives runtime shutdown_timeout and therefore denies SDK completion. + host.shutdown(Duration::from_secs(8)).await; + let completion = view.inner.wait(Duration::ZERO); + let anchored = terminal::TerminalState::lookup(view.inner.context()).is_some(); + let retained_sdk = view.inner.sdk_owner_count_for_test(); + let native = view.inner.host.0; + let flags = unsafe { native.native_terminal_state.unwrap()(native.host_ctx) }; + release.wait(); + settled_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + let call_result = call.await.unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while view.inner.sdk_owner_count_for_test() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + // All controlled payload work, actual method caller and controller worker + // have settled before assertions; the conservative original anchor remains. + assert_ne!(sample.0 & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert!( + sample.1, + "installed observer must finish bounded request before foreign Drop" + ); + assert!( + sample.2 > 0, + "actual method Connection/task must retain SDK State owner" + ); + assert_eq!(sample.3, module_runtime); + assert_eq!(completion, TerminalCompletion::Uncertain); + assert!(anchored && retained_sdk > 0); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert_eq!(fixture.requests.load(Ordering::Acquire), 1); + assert_eq!(fixture.finishes.load(Ordering::Acquire), 1); + assert_eq!(fixture.joins.load(Ordering::Acquire), 1); + assert!(call_result.is_err()); + assert!(terminal::TerminalState::lookup(view.inner.context()).is_some()); +} + +// A real served object, rather than a synthetic transport owner: Inner destroys +// its transport before its ObjectTree, so the old count can reach zero here. +struct TeardownObject { + entered: std::sync::mpsc::Sender<()>, + release: Arc, + settled: std::sync::mpsc::Sender<()>, + panic: bool, +} +#[async_trait] +impl tinybus::Interface for TeardownObject { + fn name(&self) -> tinybus::InterfaceName { + "ai.tinyhumans.tinybus.Teardown".parse().unwrap() + } + fn members(&self) -> Vec { + Vec::new() + } + async fn call( + &self, + _: &tinybus::MemberName, + _: serde_json::Value, + ) -> Result { + Ok(serde_json::Value::Null) + } +} +impl Drop for TeardownObject { + fn drop(&mut self) { + self.entered.send(()).unwrap(); + self.release.wait(); + self.settled.send(()).unwrap(); + if self.panic { + panic!("controlled ObjectTree teardown"); + } + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_connection_object_drop_after_runtime_timeout_denies_physical_completion() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (settled_tx, settled_rx) = std::sync::mpsc::channel(); + let release = Arc::new(std::sync::Barrier::new(2)); + let connection = view + .inner + .resources + .lock() + .unwrap() + .connection + .lock() + .unwrap() + .clone() + .unwrap(); + connection + .serve_at( + "/teardown".parse().unwrap(), + TeardownObject { + entered: entered_tx, + release: release.clone(), + settled: settled_tx, + panic: false, + }, + ) + .await + .unwrap(); + drop(connection); + let stopping_host = host.clone(); + let stopping = tokio::spawn(async move { + stopping_host.shutdown(Duration::from_secs(8)).await; + }); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + stopping.await.unwrap(); // The SDK's real shutdown_timeout has elapsed. + let completion = view.inner.wait(Duration::ZERO); + let flags = + unsafe { (view.inner.host.0.native_terminal_state.unwrap())(view.inner.host.0.host_ctx) }; + let still_anchored = terminal::TerminalState::lookup(view.inner.context()).is_some(); + // Release and settle actual foreign work before reporting a failed proof. + release.wait(); + settled_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + // Only the retained registry anchor and this test's view may remain. + // This waits for actual SDK Connection/task owners to finish dropping, + // rather than treating the object's last event as whole-Inner settlement. + tokio::time::timeout(Duration::from_secs(3), async { + while Arc::strong_count(&view.inner) > 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(completion, TerminalCompletion::Uncertain); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert!( + still_anchored, + "blocked whole-Connection teardown cannot refund admission" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_connection_object_drop_panic_fences_fault_after_owner_suppression() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().take().unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (settled_tx, settled_rx) = std::sync::mpsc::channel(); + let release = Arc::new(std::sync::Barrier::new(2)); + let connection = view + .inner + .resources + .lock() + .unwrap() + .connection + .lock() + .unwrap() + .clone() + .unwrap(); + connection + .serve_at( + "/teardown".parse().unwrap(), + TeardownObject { + entered: entered_tx, + release: release.clone(), + settled: settled_tx, + panic: true, + }, + ) + .await + .unwrap(); + drop(connection); + let stopping_host = host.clone(); + let stopping = tokio::spawn(async move { + stopping_host.shutdown(Duration::from_secs(8)).await; + }); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + release.wait(); + settled_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + stopping.await.unwrap(); // Runtime teardown settles the real panicking destructor. + let completion = view.inner.wait(Duration::ZERO); + let flags = + unsafe { (view.inner.host.0.native_terminal_state.unwrap())(view.inner.host.0.host_ctx) }; + let still_anchored = terminal::TerminalState::lookup(view.inner.context()).is_some(); + + // Only the retained registry anchor and this test's view may remain. + // This waits for actual SDK Connection/task owners to finish dropping, + // rather than treating the object's last event as whole-Inner settlement. + tokio::time::timeout(Duration::from_secs(3), async { + while Arc::strong_count(&view.inner) > 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(completion, TerminalCompletion::Uncertain); + assert_eq!( + flags & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert!( + still_anchored, + "panicking Connection teardown cannot refund admission" + ); + assert_ne!(flags & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn successful_repeated_reinit_disposes_prior_future_on_same_physical_worker_off_lock() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let first = host + .reinitialize("tinybus-module", serde_json::json!({"mode":0})) + .await; + let second = host + .reinitialize("tinybus-module", serde_json::json!({"mode":0})) + .await; + host.shutdown(Duration::from_secs(8)).await; + let drops: Vec<_> = events + .try_iter() + .filter_map(|event| { + if let controlled::Event::FutureDrop(name) = event { + Some(name) + } else { + None + } + }) + .collect(); + assert!(first.is_ok() && second.is_ok()); + assert_eq!(drops.len(), 3); + assert!( + drops.iter().all(|name| name == "tinybus-terminal"), + "all completed futures must be destroyed by the sole physical worker: {drops:?}" + ); + assert!(fixture.drop_off_lock.load(Ordering::Acquire)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_reinit_source_error_and_panic_request_outside_callback_exclusion() { + let _serial = controlled::TEST_LOCK.lock().await; + for mode in [3, 2, 4] { + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let result = host + .reinitialize("tinybus-module", serde_json::json!({"mode":mode})) + .await; + host.shutdown(Duration::from_secs(8)).await; + loop { + if matches!( + events.recv_timeout(Duration::from_secs(5)).unwrap(), + controlled::Event::Finish + ) { + break; + } + } + assert!(result.is_err()); + assert_eq!(fixture.requests.load(Ordering::SeqCst), 1); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + assert!( + fixture.request_off_lock.load(Ordering::Acquire), + "source fault request held reinitializer mutex in mode {mode}" + ); + } +} + +async fn controlled_prior_reinit_drop(panic: bool) { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + host.reinitialize("tinybus-module", serde_json::json!({"mode":0})) + .await + .unwrap(); + let _ = events.try_iter().count(); + let release = Arc::new(std::sync::Barrier::new(2)); + *fixture.future_drop_control.lock().unwrap() = Some((release.clone(), panic)); + let caller_host = host.clone(); + let caller = tokio::spawn(async move { + caller_host + .reinitialize( + "tinybus-module", + serde_json::json!({"mode":0,"record_drop":true}), + ) + .await + }); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().as_ref().unwrap().clone(); + view.inner.signal(TerminalReason::Stop); + let not_finished_while_blocked = fixture.joins.load(Ordering::SeqCst) == 0; + let no_next_setup = !events + .try_iter() + .any(|event| matches!(event, controlled::Event::Poll(_))); + release.wait(); + let result = caller.await.unwrap(); + loop { + if matches!( + events.recv_timeout(Duration::from_secs(5)).unwrap(), + controlled::Event::Finish + ) { + break; + } + } + host.shutdown(Duration::from_secs(8)).await; + let completion = view.inner.wait(Duration::from_secs(5)); + assert_eq!( + fixture.config_drops.load(Ordering::SeqCst), + 0, + "rejected reinit must not receive a config before prior retirement and LIVE admission" + ); + assert!(matches!(entered, controlled::Event::FutureDrop(name) if name == "tinybus-terminal")); + assert!(not_finished_while_blocked && no_next_setup); + assert!(result.is_err()); + assert_eq!(fixture.requests.load(Ordering::SeqCst), 1); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + assert!(fixture.drop_off_lock.load(Ordering::Acquire)); + assert!(fixture.request_off_lock.load(Ordering::Acquire)); + if panic { + assert_eq!(completion, TerminalCompletion::Uncertain); + assert!(view.inner.has_fault()); + assert!(terminal::TerminalState::lookup(view.inner.context()).is_some()); + assert!( + !events + .try_iter() + .any(|event| matches!(event, controlled::Event::FactoryDrop(_))), + "remaining foreign captures must stay anchored after first cleanup panic" + ); + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_prior_reinit_drop_blocks_next_setup_then_stop_settles_same_job() { + controlled_prior_reinit_drop(false).await; +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_prior_reinit_drop_panic_retains_resources_but_attempts_finish_once() { + controlled_prior_reinit_drop(true).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn accepted_reinit_config_is_destroyed_on_physical_worker_including_constructor_panic() { + let _serial = controlled::TEST_LOCK.lock().await; + for mode in [0, 4] { + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let result = host + .reinitialize( + "tinybus-module", + serde_json::json!({"mode":mode,"record_drop":true}), + ) + .await; + host.shutdown(Duration::from_secs(8)).await; + let mut names = Vec::new(); + loop { + match events.recv_timeout(Duration::from_secs(5)).unwrap() { + controlled::Event::ConfigDrop(name) => names.push(name), + controlled::Event::Finish => break, + _ => {} + } + } + let view = fixture.view.lock().unwrap().as_ref().unwrap().clone(); + let _completion = view.inner.wait(Duration::from_secs(5)); + assert_eq!(result.is_ok(), mode == 0); + assert_eq!(fixture.config_drops.load(Ordering::SeqCst), 1); + assert_eq!( + names, + ["tinybus-terminal"], + "accepted generic config Drop must run on the one physical worker" + ); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stop_during_reinit_parse_retains_received_config_until_physical_cleanup() { + let _serial = controlled::TEST_LOCK.lock().await; + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":0}), + ) + .unwrap(); + events.recv_timeout(Duration::from_secs(3)).unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let release = Arc::new(std::sync::Barrier::new(2)); + *fixture.config_parse_control.lock().unwrap() = Some(release.clone()); + let caller_host = host.clone(); + let caller = tokio::spawn(async move { + caller_host + .reinitialize( + "tinybus-module", + serde_json::json!({"mode":0,"record_drop":true,"parse_gate":true}), + ) + .await + }); + let entered = events.recv_timeout(Duration::from_secs(3)).unwrap(); + let view = fixture.view.lock().unwrap().as_ref().unwrap().clone(); + view.inner.signal(TerminalReason::Stop); + let drops_while_blocked = fixture.config_drops.load(Ordering::SeqCst); + let joins_while_blocked = fixture.joins.load(Ordering::SeqCst); + release.wait(); + let result = caller.await.unwrap(); + loop { + if matches!( + events.recv_timeout(Duration::from_secs(5)).unwrap(), + controlled::Event::Finish + ) { + break; + } + } + host.shutdown(Duration::from_secs(8)).await; + let completion = view.inner.wait(Duration::from_secs(5)); + let remainder: Vec<_> = events.try_iter().collect(); + assert!(matches!(entered, controlled::Event::ConfigParsing)); + assert_eq!(drops_while_blocked, 0); + assert_eq!(joins_while_blocked, 0); + assert!(result.is_err()); + assert_eq!(completion, TerminalCompletion::PhysicallyStopped); + assert_eq!(fixture.config_drops.load(Ordering::SeqCst), 1); + assert!(remainder.iter().any( + |event| matches!(event, controlled::Event::ConfigDrop(name) if name == "tinybus-terminal") + )); + assert!( + !remainder + .iter() + .any(|event| matches!(event, controlled::Event::Poll(_))) + ); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn linked_reinit_synchronous_timer_uses_retained_runtime_on_same_worker() { + let _serial = controlled::TEST_LOCK.lock().await; + let caller_runtime = tokio::runtime::Handle::current().id(); + let (fixture, events) = controlled::fixture(); + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + host.attach_linked_with_config( + controlled::linked_module().unwrap(), + serde_json::json!({"mode":5}), + ) + .unwrap(); + let mut observed = vec![events.recv_timeout(Duration::from_secs(3)).unwrap()]; + tokio::time::timeout(Duration::from_secs(3), async { + while !host + .list() + .iter() + .any(|info| matches!(info.state, tinybus::module::ModuleState::Ready)) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let first = host + .reinitialize("tinybus-module", serde_json::json!({"mode":5})) + .await; + let second = host + .reinitialize("tinybus-module", serde_json::json!({"mode":5})) + .await; + let caller_restored = tokio::runtime::Handle::current().id() == caller_runtime; + let view = fixture.view.lock().unwrap().as_ref().unwrap().clone(); + host.shutdown(Duration::from_secs(8)).await; + let completion = view.inner.wait(Duration::from_secs(5)); + let after_stop = host + .reinitialize("tinybus-module", serde_json::json!({"mode":5})) + .await; + observed.extend(events.try_iter()); + let constructors: Vec<_> = observed + .iter() + .filter_map(|event| match event { + controlled::Event::TimerConstruct(thread, name, runtime) => { + Some((*thread, name.as_str(), *runtime)) + } + _ => None, + }) + .collect(); + // Actual caller, controller worker and SDK cleanup have settled before assertions, + // including the unfixed constructor-panic path. + assert!( + first.is_ok() && second.is_ok(), + "synchronous timer reinit failed: {first:?}, {second:?}" + ); + assert_eq!(completion, TerminalCompletion::PhysicallyStopped); + assert_eq!(fixture.joins.load(Ordering::SeqCst), 1); + assert_eq!(constructors.len(), 3); + assert_eq!(constructors[1].1, "tinybus-terminal"); + assert_eq!(constructors[2].1, "tinybus-terminal"); + assert_eq!(constructors[1].0, constructors[2].0); + assert!( + constructors + .iter() + .all(|entry| entry.2 == constructors[0].2) + ); + assert_ne!(constructors[0].2, caller_runtime); + assert!(caller_restored); + assert!(after_stop.is_err()); + let cleanup_contexts: Vec<_> = observed + .iter() + .filter_map(|event| match event { + controlled::Event::TimerDrop(context) => Some(context), + _ => None, + }) + .collect(); + assert_eq!(cleanup_contexts.len(), 3); + assert!( + cleanup_contexts.iter().all(|context| context.is_none()), + "constructor runtime guard must be restored before later physical-worker cleanup" + ); + assert_eq!( + observed + .iter() + .filter(|event| matches!(event, controlled::Event::Poll(5))) + .count(), + 3 + ); +} + +// The genuine ModuleHost loader must have pinned the artifact before this helper. +// Temporary OS symbol handles are released; controls never grant product authority. +unsafe fn native_fixture_symbol(path: &std::path::Path, symbol: &std::ffi::CStr) -> T { + #[cfg(unix)] + let raw = { + use std::os::unix::ffi::OsStrExt; + #[cfg_attr(target_os = "linux", link(name = "dl"))] + unsafe extern "C" { + fn dlopen(path: *const std::ffi::c_char, flags: i32) -> *mut c_void; + fn dlsym(handle: *mut c_void, name: *const std::ffi::c_char) -> *mut c_void; + fn dlclose(handle: *mut c_void) -> i32; + } + let path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + let handle = unsafe { dlopen(path.as_ptr(), 2) }; + assert!(!handle.is_null()); + let raw = unsafe { dlsym(handle, symbol.as_ptr()) }; + unsafe { + dlclose(handle); + } + raw + }; + #[cfg(windows)] + let raw = { + use std::os::windows::ffi::OsStrExt; + #[link(name = "kernel32")] + unsafe extern "system" { + fn LoadLibraryW(path: *const u16) -> *mut c_void; + fn GetProcAddress(handle: *mut c_void, name: *const u8) -> *mut c_void; + fn FreeLibrary(handle: *mut c_void) -> i32; + } + let path: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let handle = unsafe { LoadLibraryW(path.as_ptr()) }; + assert!(!handle.is_null()); + let raw = unsafe { GetProcAddress(handle, symbol.as_ptr().cast()) }; + unsafe { + FreeLibrary(handle); + } + raw + }; + assert!(!raw.is_null()); + assert_eq!(size_of::(), size_of::<*mut c_void>()); + unsafe { std::mem::transmute_copy(&raw) } +} +fn native_controls_config(mode: u8, controls: &NativeControls) -> serde_json::Value { + let bytes = unsafe { + std::slice::from_raw_parts( + (controls as *const NativeControls).cast::(), + size_of::(), + ) + }; + serde_json::json!({"mode": mode, "test_controls": bytes}) +} +fn native_fixture_host() -> Arc { + let broker = tinybus::broker::Broker::new(); + broker + .clone() + .spawn(tinybus::transport::memory::MemoryBus::new()); + Arc::new(tinybus::module::ModuleHost::new(broker)) +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires the privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_factory_and_hook_failures_preserve_original_finish_and_fault() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + let host = native_fixture_host(); + host.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let (_, _, drops, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + let before = drops(); + host.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + unsafe { + drop(Box::from_raw(sender)); + } + let configure: unsafe extern "C" fn( + usize, + *mut c_void, + unsafe extern "C" fn(*mut c_void, u32, u32), + ) = unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + for mode in [1usize, 2, 10, 11, 12] { + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + // Uncertain primary/control resources intentionally retain this bounded + // fixture context; a manual join is never substituted for SDK completion. + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + unsafe { + configure(mode, sender.cast(), native_event); + } + let host = native_fixture_host(); + let result = + host.load_file_with_config(&path, native_controls_config(mode as u8, &controls)); + let mut seen = Vec::new(); + let target = if mode < 3 { 19 } else { 2 }; + loop { + let event = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); + seen.push(event); + if event.0 == target { + break; + } + } + if mode >= 10 { + host.shutdown(Duration::from_secs(5)).await; + while !seen.iter().any(|event| event.0 == 5) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + drop(host); + let native = flags(); + seen.extend(receiver.try_iter()); + // All blocking controller work is joined by the original SDK finish + // before these assertions. Factory failures never launch a controller. + if mode < 3 { + assert!(result.is_err()); + assert!(seen.contains(&(19, 1))); + assert!(!seen.iter().any(|event| matches!(event.0, 2 | 3 | 5 | 7))); + } else { + assert!(result.is_ok()); + assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (7, 1)).count(), 1); + assert!(seen.iter().any(|event| event.0 == 13 && event.1 & 7 == 7)); + } + assert!( + !seen.iter().any(|event| event.0 == 17), + "primary payload must remain retained" + ); + assert_eq!( + native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + if mode != 12 { + assert_ne!(native & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + } + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_retained_views_bound_256_and_release_only_at_actual_last_drop() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + let bootstrap = native_fixture_host(); + bootstrap + .load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let (context, probe, drops, _, _) = unsafe { terminal_fixture_symbols(&path) }; + let before = drops(); + bootstrap.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + unsafe { + drop(Box::from_raw(sender)); + } + drop(bootstrap); + let configure: unsafe extern "C" fn( + usize, + *mut c_void, + unsafe extern "C" fn(*mut c_void, u32, u32), + ) = unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let factories: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_factories") }; + let retained_live: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_retained_live") }; + let release_one: extern "C" fn() -> bool = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_one_view") }; + let release_all: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_views") }; + let mut first_token = None; + let mut admitted = 0; + let mut all_settled = true; + for _ in 0..256 { + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + unsafe { + configure(3, sender.cast(), native_event); + } + let host = native_fixture_host(); + let result = host.load_file_with_config(&path, native_controls_config(0, &controls)); + if result.is_err() { + unsafe { + drop(Box::from_raw(sender)); + } + break; + } + first_token.get_or_insert(context()); + admitted += 1; + let before = drops(); + host.shutdown(Duration::from_secs(5)).await; + let mut settled = false; + while let Ok(event) = receiver.recv_timeout(Duration::from_secs(5)) { + if event.0 == 6 { + settled = true; + break; + } + } + settled &= tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .is_ok(); + all_settled &= settled; + drop(host); + if settled { + unsafe { + drop(Box::from_raw(sender)); + } + } else { + break; + } + } + let before_factory = factories(); + let denied_host = native_fixture_host(); + let denied = + denied_host.load_file_with_config(&path, serde_json::json!({"mode":0,"test_controls":[]})); + drop(denied_host); + let factories_after_denial = factories(); + let live_after_host_drops = retained_live(); + let released_one = release_one(); + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + unsafe { + configure(3, sender.cast(), native_event); + } + let replacement = native_fixture_host(); + let replacement_result = + replacement.load_file_with_config(&path, native_controls_config(0, &controls)); + let new_token = context(); + let stale = first_token.map(|token| probe(token)); + let before = drops(); + replacement.shutdown(Duration::from_secs(5)).await; + let mut replacement_settled = false; + if replacement_result.is_ok() { + while let Ok(event) = receiver.recv_timeout(Duration::from_secs(5)) { + if event.0 == 6 { + replacement_settled = true; + break; + } + } + replacement_settled &= tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .is_ok(); + } + drop(replacement); + if replacement_settled { + unsafe { + drop(Box::from_raw(sender)); + } + } + let released_rest = release_all(); + // All actual controller joins and SDK capture destruction precede assertions. + // No uncertain anchor was reset; this filter starts in a fresh process. + assert!(all_settled && replacement_settled); + assert_eq!(admitted, 256); + assert!(denied.is_err()); + assert_eq!(factories_after_denial, before_factory); + assert_eq!(live_after_host_drops, 0); + assert!(released_one && replacement_result.is_ok()); + assert_eq!( + new_token & 255, + first_token.unwrap() & 255, + "the exact freed slot must be reusable" + ); + assert_ne!(new_token, first_token.unwrap()); + assert_eq!(stale, Some(TB_CLOSED)); + assert_eq!(released_rest, 256); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_spawn_rejection_retains_received_runtime_task_and_original_finish() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + // Keep the library loaded while resolving its fixture controls. + let warm = native_fixture_host(); + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let configure: unsafe extern "C" fn( + usize, + *mut c_void, + unsafe extern "C" fn(*mut c_void, u32, u32), + ) = unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let resources: extern "C" fn() -> u32 = + unsafe { native_fixture_symbol(&path, c"terminal_test_resources") }; + let join: extern "C" fn() -> bool = + unsafe { native_fixture_symbol(&path, c"terminal_test_external_join") }; + let (_, _, _, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + // The following control context is deliberately retained with the uncertain + // original SDK owner. The warm context is independent. + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + unsafe { + configure(20, sender.cast(), native_event); + } + let host = native_fixture_host(); + host.load_file_with_config(&path, native_controls_config(1, &controls)) + .unwrap(); + let mut seen = Vec::new(); + loop { + let event = receiver.recv_timeout(Duration::from_secs(5)).unwrap(); + seen.push(event); + if event.0 == 2 { + break; + } + } + host.shutdown(Duration::from_millis(30)).await; + // Physical spawn failed. Only this fixture cleanup joins its independently + // received controller; original finish and SDK settlement never run. + let externally_joined = join(); + let held = resources(); + host.shutdown(Duration::from_millis(30)).await; + drop(host); + let after = resources(); + seen.extend(receiver.try_iter()); + assert!(externally_joined); + assert_eq!(seen.iter().filter(|event| event.0 == 21).count(), 1); + assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); + assert!(!seen.iter().any(|event| matches!(event.0, 4 | 5 | 6 | 20))); + // Actual retained F, Runtime, transport, reinit callback, sole initial + // JoinHandle and already-polled owned future. No timeout/refund inference. + let expected = 1 | 8 | 16 | 32 | 64 | 256; + assert_eq!(held & expected, expected); + assert_eq!(after & expected, expected); + assert_ne!(flags() & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!( + flags() & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_startup_transfers_and_runtime_build_error_dispose_on_physical_worker() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let warm = native_fixture_host(); + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let configure: unsafe extern "C" fn( + usize, + *mut c_void, + unsafe extern "C" fn(*mut c_void, u32, u32), + ) = unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let resources: extern "C" fn() -> u32 = + unsafe { native_fixture_symbol(&path, c"terminal_test_resources") }; + let transport_drop: extern "C" fn() -> bool = + unsafe { native_fixture_symbol(&path, c"terminal_test_transport_drop") }; + let release: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_views") }; + let (_, _, drops, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + for (stage, expected) in [ + (1, 129u32), + (2, 3), + (3, 7), + (4, 15), + (5, 31), + (6, 59), + (7, 123), + (8, 123), + (9, 7), + ] { + let (sender, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let sender = Box::into_raw(Box::new(sender)); + let controls = NativeControls { + ctx: sender.cast(), + event: native_event, + }; + unsafe { + configure(20 + stage, sender.cast(), native_event); + } + let before = drops(); + let host = native_fixture_host(); + let result = host.load_file_with_config(&path, native_controls_config(1, &controls)); + let mut seen = Vec::new(); + // Both real F and SDK-owned C destruction plus controller last Drop + // settle before assertions. No task has entered trusted F at stages7/8. + while ![6, 19, 20] + .iter() + .all(|kind| seen.iter().any(|event: &(u32, u32)| event.0 == *kind)) + { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before || resources() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let dropped_transport = transport_drop(); + host.shutdown(Duration::from_secs(5)).await; + drop(host); + let native = flags(); + release(); + seen.extend(receiver.try_iter()); + unsafe { + drop(Box::from_raw(sender)); + } + assert!(result.is_err(), "stage {stage}"); + assert!( + seen.contains(&(22, expected)), + "received stage {stage}: {seen:?}" + ); + assert_eq!(seen.iter().filter(|event| event.0 == 3).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert!(seen.contains(&(6, 1))); + assert!(seen.contains(&(19, 1))); + assert!(seen.contains(&(20, 1))); + assert!(!seen.iter().any(|event| event.0 == 2)); + assert_eq!(dropped_transport, (5..=8).contains(&stage), "stage {stage}"); + assert_ne!(native & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!( + native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + } +} + +struct NativeBarrierControls { + events: std::sync::mpsc::Sender<(u32, u32)>, + blocked_event: u32, + release: (StdMutex, std::sync::Condvar), +} +impl NativeBarrierControls { + fn release(&self) { + *self.release.0.lock().unwrap() = true; + self.release.1.notify_all(); + } +} +unsafe extern "C" fn native_barrier_event(ctx: *mut c_void, event: u32, value: u32) { + let controls = unsafe { &*ctx.cast::() }; + let _ = controls.events.send((event, value)); + if event == controls.blocked_event { + let mut released = controls.release.0.lock().unwrap(); + while !*released { + released = controls.release.1.wait(released).unwrap(); + } + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_entered_initial_and_reinit_work_settle_before_original_finish() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + // Modes40/41 block actual synchronous construction/poll; mode42 blocks + // actual retained future Drop. Reinit43 returns a genuine ready Err only + // after the same poll has observed an already-committed real host Stop. + for (reinitializing, mode) in [ + (false, 40u8), + (false, 41), + (false, 42), + (false, 44), + (false, 45), + (true, 40), + (true, 41), + (true, 42), + (true, 43), + (true, 44), + (true, 45), + ] { + let (events, receiver) = std::sync::mpsc::channel(); + let barrier = Box::into_raw(Box::new(NativeBarrierControls { + events, + blocked_event: match mode { + 40 | 44 => 23, + 42 => 25, + _ => 24, + }, + release: (StdMutex::new(false), std::sync::Condvar::new()), + })); + let controls = NativeControls { + ctx: barrier.cast(), + event: native_barrier_event, + }; + let host = native_fixture_host(); + host.load_file_with_config( + &path, + native_controls_config(if reinitializing { 0 } else { mode }, &controls), + ) + .unwrap(); + let (context, _, drops, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + let reinit: unsafe extern "C" fn(usize, *const u8, usize) -> i32 = + unsafe { native_fixture_symbol(&path, c"terminal_test_reinit") }; + let before = drops(); + let mut seen = Vec::new(); + if reinitializing { + while !seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + let call = reinitializing.then(|| { + let token = context(); + let bytes = serde_json::to_vec(&native_controls_config(mode, &controls)).unwrap(); + std::thread::spawn(move || unsafe { reinit(token, bytes.as_ptr(), bytes.len()) }) + }); + // Drop42 is first entered on actual physical cleanup after shutdown. + if mode != 42 { + while !seen + .iter() + .any(|event| event.0 == unsafe { &*barrier }.blocked_event) + { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } else if !reinitializing { + while !seen.iter().any(|event| event.0 == 2) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } else { + while seen.iter().filter(|event| event.0 == 2).count() != 2 { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + host.shutdown(Duration::from_millis(30)).await; + if mode == 42 { + while !seen.iter().any(|event| event.0 == 25) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + let blocked_native = flags(); + seen.extend(receiver.try_iter()); + let finish_before_release = seen.iter().any(|event| event.0 == 5); + let drop_before_release = drops(); + // Release before any assertion; join the real ABI caller, actual + // initial task/future and original independent controller below. + unsafe { &*barrier }.release(); + let reinit_result = call.map(|call| call.join().unwrap()); + host.shutdown(Duration::from_secs(5)).await; + while !seen.iter().any(|event| event.0 == 6) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let settled_native = flags(); + seen.extend(receiver.try_iter()); + drop(host); + unsafe { + drop(Box::from_raw(barrier)); + } + assert!( + !finish_before_release, + "entered mode {mode}, reinit {reinitializing}" + ); + assert_eq!(drop_before_release, before); + assert_ne!( + blocked_native & tinybus::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT, + 0 + ); + assert_eq!( + blocked_native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (7, 1)).count(), 1); + assert!(seen.iter().any(|event| event.0 == 13 && event.1 & 7 == 7)); + assert!(seen.iter().any(|event| event.0 == 16 && event.1 & 1 == 1)); + assert!( + seen.iter() + .filter(|event| event.0 == 26) + .all(|event| event.1 == 1) + ); + if let Some(result) = reinit_result { + assert_eq!(result, if mode == 44 { TB_PANICKED } else { TB_CLOSED }); + } + if mode == 43 || mode == 44 || mode == 45 { + assert_ne!( + settled_native & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, + 0, + "genuine source error/panic must dominate simultaneous ready Stop" + ); + assert_eq!( + settled_native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert!(!seen.iter().any(|event| event.0 == 17)); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_method_payload_and_whole_connection_drop_retain_actual_sdk_owners() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let warm = native_fixture_host(); + let (events, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let events = Box::into_raw(Box::new(events)); + let controls = NativeControls { + ctx: events.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let configure: unsafe extern "C" fn( + usize, + *mut c_void, + unsafe extern "C" fn(*mut c_void, u32, u32), + ) = unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let owners: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_sdk_owners") }; + let release_views: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_views") }; + let (_, _, drops, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + for mode in [50u8, 51, 52] { + let (events, receiver) = std::sync::mpsc::channel(); + let barrier = Box::into_raw(Box::new(NativeBarrierControls { + events, + blocked_event: if mode == 50 { 30 } else { 32 }, + release: (StdMutex::new(false), std::sync::Condvar::new()), + })); + let controls = NativeControls { + ctx: barrier.cast(), + event: native_barrier_event, + }; + unsafe { + configure(46, barrier.cast(), native_barrier_event); + } + let broker = tinybus::broker::Broker::new(); + let bus = tinybus::transport::memory::MemoryBus::new(); + broker.clone().spawn(bus.clone()); + let client = Connection::connect(bus.connect().await.unwrap()) + .await + .unwrap(); + let host = Arc::new(tinybus::module::ModuleHost::new(broker)); + let before = drops(); + host.load_file_with_config(&path, native_controls_config(mode, &controls)) + .unwrap(); + let mut seen = Vec::new(); + while !seen.iter().any(|event: &(u32, u32)| event.0 == 31) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + let call = (mode == 50).then(|| { + tokio::spawn(async move { + client + .proxy( + "ai.tinyhumans.TerminalFixture", + "/terminal", + "ai.tinyhumans.TerminalFixture", + ) + .unwrap() + .with_timeout(Duration::from_secs(10)) + .call::("Panic", ()) + .await + }) + }); + if mode == 50 { + while !seen.iter().any(|event| event.0 == 30) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + let stopping_host = host.clone(); + let stopping = tokio::spawn(async move { + stopping_host.shutdown(Duration::from_secs(8)).await; + }); + if mode != 50 { + while !seen.iter().any(|event| event.0 == 32) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + } + stopping.await.unwrap(); + // Host shutdown has a deadline and can return before the physical + // worker finishes. Observe the original finish/join directly while + // the SDK-owned destructor is still held by the fixture barrier. + while !seen.contains(&(5, 1)) || !seen.contains(&(7, 1)) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + let held_owners = owners(); + let timed_out_native = flags(); + seen.extend(receiver.try_iter()); + let actual_finish = seen.iter().filter(|event| **event == (5, 1)).count(); + let actual_join = seen.iter().filter(|event| **event == (7, 1)).count(); + let retained_controller = drops() == before; + // Always settle controlled work before asserting: release real foreign + // Drop, await actual call and actual counted last Connection observer. + unsafe { &*barrier }.release(); + let call_result = match call { + Some(call) => Some(call.await.unwrap()), + None => None, + }; + while !seen.iter().any(|event| event.0 == 6) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + tokio::time::timeout(Duration::from_secs(5), async { + while owners() != 0 || drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let settled_native = flags(); + release_views(); + seen.extend(receiver.try_iter()); + drop(host); + unsafe { + drop(Box::from_raw(barrier)); + } + assert!( + held_owners > 0, + "whole SDK-owned Connection task/Drop mode {mode}" + ); + assert!(retained_controller); + assert_eq!( + actual_finish, 1, + "actual original controller finish is separate" + ); + assert_eq!(actual_join, 1); + assert_eq!( + timed_out_native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert_eq!( + settled_native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + if mode == 50 { + assert!( + seen.contains(&(29, 7)), + "Fault/request-done/actualSDKowner before first payload Drop" + ); + assert!(call_result.unwrap().is_err()); + assert_ne!( + timed_out_native & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, + 0 + ); + } else { + assert!( + seen.iter().any(|event| event.0 == 35 && event.1 & 6 == 6), + "owner already stopped, bounded request done before whole Connection Drop" + ); + if mode == 52 { + assert_ne!( + settled_native & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, + 0, + "explicit resource-drop observer fences Fault even after owner suppression" + ); + } + } + } +} + +type NativeConfigure = + unsafe extern "C" fn(usize, *mut c_void, unsafe extern "C" fn(*mut c_void, u32, u32)); +type NativeReinit = unsafe extern "C" fn(usize, *const u8, usize) -> i32; +struct NativeNestedControls { + events: std::sync::mpsc::Sender<(u32, u32)>, + path: std::path::PathBuf, + host: Arc, + other: NativeControls, + configure: NativeConfigure, + reinit: NativeReinit, + context: extern "C" fn() -> usize, + token: std::sync::atomic::AtomicUsize, + phase: std::sync::atomic::AtomicUsize, +} +unsafe extern "C" fn native_nested_event(ctx: *mut c_void, event: u32, value: u32) { + let controls = unsafe { &*ctx.cast::() }; + let _ = controls.events.send((event, value)); + if event == 10 { + unsafe { + (controls.configure)(46, controls.other.ctx, controls.other.event); + } + let loaded = controls + .host + .load_file_with_config(&controls.path, native_controls_config(0, &controls.other)); + controls + .token + .store((controls.context)(), Ordering::Release); + let _ = controls.events.send((36, u32::from(loaded.is_ok()))); + } + if event == 26 && controls.phase.load(Ordering::Acquire) == 1 { + let bytes = serde_json::to_vec(&native_controls_config(0, &controls.other)).unwrap(); + let result = unsafe { + (controls.reinit)( + controls.token.load(Ordering::Acquire), + bytes.as_ptr(), + bytes.len(), + ) + }; + let _ = controls.events.send((37, u32::from(result == TB_OK))); + } +} +struct NativeLogCount(Arc); +impl tracing::Subscriber for NativeLogCount { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + if event.metadata().target() == "tinybus_module" { + self.0.fetch_add(1, Ordering::Release); + } + } + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_nested_factory_reinit_restore_exact_owner_and_suppress_captured_dispatch() +{ + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let warm = native_fixture_host(); + let (events, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let events = Box::into_raw(Box::new(events)); + let controls = NativeControls { + ctx: events.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let configure: NativeConfigure = + unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let reinit: NativeReinit = unsafe { native_fixture_symbol(&path, c"terminal_test_reinit") }; + let dispatch: extern "C" fn(usize) -> bool = + unsafe { native_fixture_symbol(&path, c"terminal_test_dispatch") }; + let unowned: extern "C" fn() -> bool = + unsafe { native_fixture_symbol(&path, c"terminal_test_unowned") }; + let release_views: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_views") }; + let retained_live: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_retained_live") }; + let (context, old_probe, drops, _, _) = unsafe { terminal_fixture_symbols(&path) }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + let (b_events, b_receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let b_events = Box::into_raw(Box::new(b_events)); + let b_controls = NativeControls { + ctx: b_events.cast(), + event: native_event, + }; + let host_b = native_fixture_host(); + let (a_events, a_receiver) = std::sync::mpsc::channel(); + let nested = Box::into_raw(Box::new(NativeNestedControls { + events: a_events, + path: path.clone(), + host: host_b.clone(), + other: b_controls, + configure, + reinit, + context, + token: std::sync::atomic::AtomicUsize::new(0), + phase: std::sync::atomic::AtomicUsize::new(0), + })); + let a_controls = NativeControls { + ctx: nested.cast(), + event: native_nested_event, + }; + unsafe { + configure(46, nested.cast(), native_nested_event); + } + let host_a = native_fixture_host(); + host_a + .load_file_with_config(&path, native_controls_config(0, &a_controls)) + .unwrap(); + let token_a = context(); + let token_b = unsafe { &*nested }.token.load(Ordering::Acquire); + let mut a_seen = Vec::new(); + let mut b_seen = Vec::new(); + while !a_seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + a_seen.push(a_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + while !b_seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + b_seen.push(b_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + unsafe { &*nested }.phase.store(1, Ordering::Release); + let bytes = serde_json::to_vec(&native_controls_config(0, &a_controls)).unwrap(); + let first_reinit = unsafe { reinit(token_a, bytes.as_ptr(), bytes.len()) }; + let restored_external = unowned(); + let logs = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let probe = |index| { + let before = logs.load(Ordering::Acquire); + let enabled = + tracing::subscriber::with_default(NativeLogCount(logs.clone()), || dispatch(index)); + (enabled, logs.load(Ordering::Acquire) - before) + }; + let a_live_probe = probe(0); + let b_live_probe = probe(1); + let before = drops(); + host_a.shutdown(Duration::from_secs(5)).await; + while !a_seen.iter().any(|event| event.0 == 6) { + a_seen.push(a_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let a_stopped_probe = probe(0); + let b_after_a_probe = probe(1); + let live_after_a = retained_live(); + let b_bytes = serde_json::to_vec(&native_controls_config(0, &b_controls)).unwrap(); + let b_after_a_reinit = unsafe { reinit(token_b, b_bytes.as_ptr(), b_bytes.len()) }; + host_b.shutdown(Duration::from_secs(5)).await; + while !b_seen.iter().any(|event| event.0 == 6) { + b_seen.push(b_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + let b_stopped_probe = probe(1); + // A newly loaded third native instance has its own exact owner; retired + // opaque tokens cannot select it, and old factory dispatchers remain quiet. + let (c_events, c_receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let c_events = Box::into_raw(Box::new(c_events)); + let c_controls = NativeControls { + ctx: c_events.cast(), + event: native_event, + }; + unsafe { + configure(46, c_events.cast(), native_event); + } + let host_c = native_fixture_host(); + host_c + .load_file_with_config(&path, native_controls_config(0, &c_controls)) + .unwrap(); + let mut c_seen = Vec::new(); + while !c_seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + c_seen.push(c_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + let c_live_probe = probe(2); + let old_a = old_probe(token_a); + let old_b = old_probe(token_b); + let old_dispatch_after_reload = [probe(0), probe(1)]; + let before = drops(); + host_c.shutdown(Duration::from_secs(5)).await; + while !c_seen.iter().any(|event| event.0 == 6) { + c_seen.push(c_receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + a_seen.extend(a_receiver.try_iter()); + b_seen.extend(b_receiver.try_iter()); + c_seen.extend(c_receiver.try_iter()); + let finally_unowned = unowned(); + release_views(); + drop(host_a); + drop(host_b); + drop(host_c); + // All real original finishes/joins and F/controller captures settled. + unsafe { + drop(Box::from_raw(nested)); + drop(Box::from_raw(b_events)); + drop(Box::from_raw(c_events)); + } + assert_eq!(first_reinit, TB_OK); + assert_eq!(b_after_a_reinit, TB_OK); + assert!(restored_external && finally_unowned); + assert!(a_seen.contains(&(36, 1)) && a_seen.contains(&(37, 1))); + assert!( + a_seen.iter().any(|event| event.0 == 27 && event.1 & 3 == 1), + "factory B scope restores actual factory A" + ); + assert!( + a_seen + .iter() + .filter(|event| event.0 == 28) + .all(|event| event.1 & 3 == 1), + "nested B reinit restores actual worker A" + ); + assert_eq!(a_live_probe, (true, 1)); + assert_eq!(b_live_probe, (true, 1)); + assert_eq!(a_stopped_probe, (false, 0)); + assert_eq!(b_after_a_probe, (true, 1)); + assert_eq!(b_stopped_probe, (false, 0)); + assert_eq!(c_live_probe, (true, 1)); + assert_eq!(old_dispatch_after_reload, [(false, 0), (false, 0)]); + assert_eq!(live_after_a, 1); + assert_eq!((old_a, old_b), (TB_CLOSED, TB_CLOSED)); + for seen in [a_seen, b_seen, c_seen] { + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (7, 1)).count(), 1); + assert!(seen.iter().any(|event| event.0 == 13 && event.1 & 7 == 7)); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_first_foreign_cleanup_panic_keeps_later_resources_and_attempts_original_finish() + { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let warm = native_fixture_host(); + let (warm_events, warm_receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let warm_events = Box::into_raw(Box::new(warm_events)); + let warm_controls = NativeControls { + ctx: warm_events.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &warm_controls)) + .unwrap(); + let configure: NativeConfigure = + unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let resources: extern "C" fn() -> u32 = + unsafe { native_fixture_symbol(&path, c"terminal_test_resources") }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!( + warm_receiver.recv_timeout(Duration::from_secs(5)), + Ok((6, _)) + ) {} + let (events, receiver) = std::sync::mpsc::channel(); + // This callback owner deliberately stays retained with the SDK's uncertain + // untouched later F/Runtime/callback resources after the first Drop failure. + let barrier = Box::into_raw(Box::new(NativeBarrierControls { + events, + blocked_event: 25, + release: (StdMutex::new(false), std::sync::Condvar::new()), + })); + let controls = NativeControls { + ctx: barrier.cast(), + event: native_barrier_event, + }; + unsafe { + configure(46, barrier.cast(), native_barrier_event); + } + let host = native_fixture_host(); + host.load_file_with_config(&path, native_controls_config(47, &controls)) + .unwrap(); + let (_, _, _, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + let mut seen = Vec::new(); + while !seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + host.shutdown(Duration::from_millis(30)).await; + while !seen.iter().any(|event| event.0 == 25) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + let before_release = flags(); + seen.extend(receiver.try_iter()); + let finished_early = seen.iter().any(|event| event.0 == 5); + unsafe { &*barrier }.release(); + while !seen.iter().any(|event| event.0 == 5) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + host.shutdown(Duration::from_secs(5)).await; + host.shutdown(Duration::from_millis(30)).await; + drop(host); + let after = flags(); + let held_resources = resources(); + seen.extend(receiver.try_iter()); + assert!(!finished_early); + assert_eq!(held_resources & (1 | 8 | 16 | 32), 1 | 8 | 16 | 32); + assert_eq!( + before_release & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (7, 1)).count(), 1); + assert!(seen.iter().any(|event| event.0 == 13 && event.1 & 7 == 7)); + assert!(!seen.iter().any(|event| matches!(event.0, 6 | 17 | 20))); + assert_ne!(after & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!( + after & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires a fresh-process privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_prior_reinit_future_drop_restores_owner_and_retains_settlement() { + let path = std::path::PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + let warm = native_fixture_host(); + let (events, receiver) = std::sync::mpsc::channel::<(u32, u32)>(); + let events = Box::into_raw(Box::new(events)); + let controls = NativeControls { + ctx: events.cast(), + event: native_event, + }; + warm.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let configure: NativeConfigure = + unsafe { native_fixture_symbol(&path, c"terminal_test_factory_mode") }; + let reinit: NativeReinit = unsafe { native_fixture_symbol(&path, c"terminal_test_reinit") }; + let resources: extern "C" fn() -> u32 = + unsafe { native_fixture_symbol(&path, c"terminal_test_resources") }; + let release_views: extern "C" fn() -> usize = + unsafe { native_fixture_symbol(&path, c"terminal_test_release_views") }; + let (context, _, drops, _, flags) = unsafe { terminal_fixture_symbols(&path) }; + warm.shutdown(Duration::from_secs(5)).await; + while !matches!(receiver.recv_timeout(Duration::from_secs(5)), Ok((6, _))) {} + for mode in [48u8, 49] { + let (events, receiver) = std::sync::mpsc::channel(); + let barrier = Box::into_raw(Box::new(NativeBarrierControls { + events, + blocked_event: 25, + release: (StdMutex::new(false), std::sync::Condvar::new()), + })); + let controls = NativeControls { + ctx: barrier.cast(), + event: native_barrier_event, + }; + unsafe { + configure(46, barrier.cast(), native_barrier_event); + } + let host = native_fixture_host(); + let before = drops(); + host.load_file_with_config(&path, native_controls_config(0, &controls)) + .unwrap(); + let mut seen = Vec::new(); + while !seen.iter().any(|event: &(u32, u32)| event.0 == 2) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + let token = context(); + let bytes = serde_json::to_vec(&native_controls_config(mode, &controls)).unwrap(); + let first_result = unsafe { reinit(token, bytes.as_ptr(), bytes.len()) }; + let next_bytes = serde_json::to_vec(&native_controls_config(0, &controls)).unwrap(); + let next = std::thread::spawn(move || unsafe { + reinit(token, next_bytes.as_ptr(), next_bytes.len()) + }); + while !seen.iter().any(|event| event.0 == 25) { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + host.shutdown(Duration::from_millis(30)).await; + let held_runtime = resources() & 8 != 0; + let held_native = flags(); + seen.extend(receiver.try_iter()); + let finish_before_release = seen.iter().any(|event| event.0 == 5); + // Real previous-future Drop owns the sole pending retirement operation. + // Release it and join real second ABI caller before asserting context. + unsafe { &*barrier }.release(); + let second_result = next.join().unwrap(); + host.shutdown(Duration::from_secs(5)).await; + while !seen + .iter() + .any(|event| event.0 == if mode == 48 { 6 } else { 5 }) + { + seen.push(receiver.recv_timeout(Duration::from_secs(5)).unwrap()); + } + if mode == 48 { + tokio::time::timeout(Duration::from_secs(5), async { + while drops() == before { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } + let after = flags(); + let later_resources = resources(); + seen.extend(receiver.try_iter()); + drop(host); + release_views(); + if mode == 48 { + unsafe { + drop(Box::from_raw(barrier)); + } + } + // Mode49 context stays conservatively retained with later resources. + assert_eq!(first_result, TB_OK); + assert_eq!(second_result, TB_CLOSED); + assert!(held_runtime && !finish_before_release); + assert_ne!( + held_native & tinybus::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT, + 0 + ); + assert_eq!( + held_native & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + assert_eq!(seen.iter().filter(|event| **event == (5, 1)).count(), 1); + assert_eq!(seen.iter().filter(|event| **event == (7, 1)).count(), 1); + assert!( + seen.contains(&(38, 0)), + "construction Handle guard ends before its ack and later foreign Drop: {seen:?}" + ); + assert!(seen.iter().any(|event| event.0 == 39 && event.1 & 1 == 1)); + assert!( + seen.contains(&(25, 1)), + "actual previous Drop on the same physical worker" + ); + if mode == 49 { + assert_eq!(later_resources & (1 | 8 | 16 | 32), 1 | 8 | 16 | 32); + assert!(!seen.iter().any(|event| matches!(event.0, 6 | 17 | 20))); + assert_ne!(after & tinybus::module::abi::TB_NATIVE_TERMINAL_FAULT, 0); + assert_eq!( + after & tinybus::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, + 0 + ); + } + } +} diff --git a/crates/tinybus-module/src/terminal.rs b/crates/tinybus-module/src/terminal.rs new file mode 100644 index 00000000..849df1ec --- /dev/null +++ b/crates/tinybus-module/src/terminal.rs @@ -0,0 +1,1057 @@ +//! Opted-in instance terminal authority and bounded physical cleanup ownership. +use super::*; +use std::any::Any; +use std::pin::Pin; +use std::sync::atomic::{AtomicU8, AtomicUsize}; +use std::sync::{Condvar, Weak}; +use tinybus::module::abi::{ + TB_NATIVE_TERMINAL_FAULT, TB_NATIVE_TERMINAL_LIVE, TB_NATIVE_TERMINAL_STOP_INTENT, +}; + +/// Cause of irreversible instance admission closure. Genuine fault takes +/// precedence over ordinary stop or host disappearance. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TerminalReason { + /// Ordinary stop intent; it does not assert worker completion. + Stop, + /// The native query is unavailable or no longer grants live admission. + HostClosed, + /// A genuine native or module fault, including a caught panic. + Fault, + /// The factory or setup returned an error; native fault is also reported. + SetupFailed, +} +/// Actual controller cleanup outcome, independent of native admission. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TerminalCompletion { + /// The controller has fenced admission and joined its actual workers. + /// This assertion alone proves neither SDK teardown nor backing last-Drop. + PhysicallyStopped, + /// Controller or SDK physical cleanup could not be proved. Even joined + /// workers can leave uncertain SDK resources. The SDK retains the owner; + /// a deadline cannot convert this into successful cleanup. + Uncertain, +} +/// Controller callbacks supplied by the terminal factory. +/// +/// The factory owns partial-failure cleanup until it returns these hooks. It +/// must launch independent workers only during setup, after hook registration. +/// The same controller remains responsible through every reinitialization. +pub struct RuntimeHooks { + request: Arc, + finish: Box TerminalCompletion + Send>, +} +impl RuntimeHooks { + /// `request` must only close admission and signal bounded cancel/wake. It + /// must not join, perform I/O, reenter SDK APIs, or wait for a gate. `finish` + /// independently fences and joins the controller after SDK setup cancellation + /// has settled; it must not depend on cancelled futures or host shutdown. + /// Returning `PhysicallyStopped` asserts actual fence/join, not a timeout. + /// The SDK additionally requires its own resource teardown without sticky + /// uncertainty before accepting physical completion. Trusted callbacks must + /// uphold these obligations; the SDK cannot force arbitrary Rust to finish. + pub fn new(request: R, finish: F) -> Self + where + R: Fn(TerminalReason) + Send + Sync + 'static, + F: FnOnce() -> TerminalCompletion + Send + 'static, + { + Self { + request: Arc::new(request), + finish: Box::new(finish), + } + } +} +/// A retained view of one actual admitted native instance. Its lifetime retains +/// the instance's admission permit even after successful physical cleanup. +/// Each SDK image admits at most 256 retained instances; views keep their slot +/// occupied until actual last-Drop. This does not bound controller resource bytes +/// or the host's existing context and mapped-library retention. +#[derive(Clone)] +pub struct NativeInstanceLiveness { + pub(super) inner: Arc, +} +impl NativeInstanceLiveness { + /// Only a supported, exactly-live native query allows new admission. + /// Observed terminal state never reopens. This is not a publication lease. + /// A controller must check its immutable instance/receiver binding and local + /// terminal flags, then this view, then local flags again under its physical + /// phase gate. A cached `true` cannot authorize a later phase; a later stop + /// cannot undo an operation whose admission already linearized. + pub fn is_live(&self) -> bool { + self.inner.is_live() + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_owner_is_current(&self) -> bool { + self.inner + .owner + .as_ref() + .is_some_and(ExecutionOwner::is_current_for_fixture) + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_owner_is_stopped(&self) -> bool { + self.inner + .owner + .as_ref() + .is_some_and(ExecutionOwner::is_stopped_for_fixture) + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_request_done(&self) -> bool { + self.inner.request_done_for_test() + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_sdk_owners(&self) -> usize { + self.inner.sdk_owner_count_for_test() + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_reject_worker_spawn(&self) { + self.inner.reject_worker_spawn_for_test(); + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_reject_startup(&self, stage: u8) { + self.inner.reject_startup_for_test(stage); + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_resources(&self) -> u32 { + self.inner.resources_for_fixture() + } + #[doc(hidden)] + #[cfg(feature = "__native-fixture-support")] + pub fn __fixture_transport_dropped_on_physical_worker(&self) -> bool { + self.inner.transport_drop_thread.lock().unwrap().as_deref() == Some("tinybus-terminal") + } + /// Persist genuine fault and close independent native admission first. + /// Signals the bounded request hook without joining. Call this after + /// unwinding out of the controller's physical phase gate. + pub fn request_fault(&self) { + self.inner.signal(TerminalReason::Fault); + } +} +const INSTANCE_LIMIT: usize = 256; +#[derive(Default)] +struct Slot { + reserved: bool, + exhausted: bool, + generation: usize, + anchor: Option>, +} +struct Admission { + slots: StdMutex<[Slot; INSTANCE_LIMIT]>, +} +impl Admission { + fn new() -> Arc { + Arc::new(Self { + slots: StdMutex::new(std::array::from_fn(|_| Slot::default())), + }) + } +} +static ADMISSION: OnceLock> = OnceLock::new(); +struct SlotLease { + admission: Weak, + index: usize, +} +impl Drop for SlotLease { + fn drop(&mut self) { + if let Some(admission) = self.admission.upgrade() { + let mut slots = admission + .slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + // The permit ends only at actual State last-Drop, never at anchor retirement. + slots[self.index].reserved = false; + } + } +} +#[derive(Clone, Copy, Eq, PartialEq)] +enum RequestState { + Unregistered, + Idle, + Running, + Done, + Panicked, +} +#[derive(Clone, Copy, Eq, PartialEq)] +enum FinishState { + Idle, + Submitted, + Running, + Complete, + SpawnFailed, +} +struct FinishJob { + state: Weak, +} +impl FinishJob { + fn run(self: Arc, state: Arc) { + state.run_finish(); + } +} +type Payload = Box; +#[derive(Default)] +struct PrimaryPayloads { + request: Option, + finish: Option, + initial: Option, + reinit: Option, + factory: Option, + startup: Option, + cleanup: Option, +} +#[derive(Clone, Copy)] +pub(super) enum PayloadKind { + Request, + Finish, + Initial, + Reinit, + Factory, + Startup, + Cleanup, +} +impl PrimaryPayloads { + fn cell(&mut self, kind: PayloadKind) -> &mut Option { + match kind { + PayloadKind::Request => &mut self.request, + PayloadKind::Finish => &mut self.finish, + PayloadKind::Initial => &mut self.initial, + PayloadKind::Reinit => &mut self.reinit, + PayloadKind::Factory => &mut self.factory, + PayloadKind::Startup => &mut self.startup, + PayloadKind::Cleanup => &mut self.cleanup, + } + } +} +enum ReinitOperation { + Retire(SetupFuture), + Construct(Box i32 + Send>), +} +struct Control { + request: Option>, + finish: Option TerminalCompletion + Send>>, + request_state: RequestState, + finish_state: FinishState, + published: bool, + completion: Option, + job: Option>, + payloads: PrimaryPayloads, + active_reinit: bool, + reinit_operation: Option, + construction_code: i32, + prior_pending: bool, + retirement_failed: bool, +} +pub(super) type Setup = + Box Pin> + Send>> + Send>; +pub(super) type SetupFuture = Pin> + Send>>; +pub(super) type SetupFutureSlot = Arc>>; +pub(super) type TerminalReinitialize = Arc i32 + Send + Sync>; + +/// Resources are attached here before the next fallible startup boundary. +/// Taking values is mechanical; all foreign destruction occurs off this lock. +pub(super) struct StartupResources { + pub runtime: Option, + pub setup: Option, + pub retained_factory: Option>, + pub config: Option>, + pub reinitializer_factory: Option, + pub reinitialize: Arc>>, + pub connection: Arc>>, + pub inbound: Option>, + pub transport: Option>, + pub initial: Option>>, + pub initial_future: SetupFutureSlot, + pub reinit_future: SetupFutureSlot, +} +impl Default for StartupResources { + fn default() -> Self { + Self { + runtime: None, + setup: None, + retained_factory: None, + config: None, + reinitializer_factory: None, + reinitialize: Arc::new(StdMutex::new(None)), + connection: Arc::new(StdMutex::new(None)), + inbound: None, + transport: None, + initial: None, + initial_future: Arc::new(StdMutex::new(None)), + reinit_future: Arc::new(StdMutex::new(None)), + } + } +} +pub(super) struct TerminalState { + pub host: HostCalls, + pub owner: Option, + reason: AtomicU8, + uncertain: AtomicBool, + control: StdMutex, + changed: Condvar, + pub terminal_notify: tokio::sync::Notify, + pub resources: StdMutex, + sdk_owners: AtomicUsize, + vtable_published: AtomicBool, + #[cfg(any(test, feature = "__native-fixture-support"))] + reject_worker_spawn: AtomicBool, + #[cfg(any(test, feature = "__native-fixture-support"))] + startup_failure_stage: AtomicU8, + #[cfg(any(test, feature = "__native-fixture-support"))] + transport_drop_thread: StdMutex>, + lease: SlotLease, + token: usize, +} +/// Every surviving SDK transport/Connection/task retains this actual State. +/// A pending runtime teardown may therefore never recycle its slot. +pub(super) struct SdkOwner { + state: Arc, +} +impl SdkOwner { + #[cfg(any(test, feature = "__native-fixture-support"))] + pub(super) fn record_transport_drop_for_test(&self) { + *self.state.transport_drop_thread.lock().unwrap() = Some( + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned(), + ); + } + pub fn caught_panic(&self) { + self.state.mark_uncertain(); + self.state.signal(TerminalReason::Fault); + } +} +impl Drop for SdkOwner { + fn drop(&mut self) { + // A last-field guard also runs when a preceding foreign destructor + // unwinds. Counting that as clean disposal would fabricate completion. + // The diagnostic owner may already be stopped; explicitly fence native + // authority and preserve the anchor before releasing the resource count. + if std::thread::panicking() { + self.caught_panic(); + } + self.state.sdk_owners.fetch_sub(1, Ordering::AcqRel); + } +} +impl TerminalState { + pub fn admit(host: HostCalls, owner: Option) -> Option> { + Self::admit_in(ADMISSION.get_or_init(Admission::new), host, owner) + } + fn admit_in( + admission: &Arc, + host: HostCalls, + owner: Option, + ) -> Option> { + let mut slots = admission + .slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let index = slots.iter_mut().enumerate().find_map(|(index, slot)| { + if slot.reserved || slot.exhausted { + return None; + } + let generation = slot + .generation + .checked_add(1) + .filter(|generation| *generation <= usize::MAX >> 8); + match generation { + Some(generation) => { + slot.generation = generation; + Some(index) + } + None => { + slot.exhausted = true; + None + } + } + })?; + slots[index].reserved = true; + let token = (slots[index].generation << 8) | index; + let state = Arc::new(Self { + host, + owner, + reason: AtomicU8::new(0), + uncertain: AtomicBool::new(false), + control: StdMutex::new(Control { + request: None, + finish: None, + request_state: RequestState::Unregistered, + finish_state: FinishState::Idle, + published: false, + completion: None, + job: None, + payloads: PrimaryPayloads::default(), + active_reinit: false, + reinit_operation: None, + construction_code: TB_CLOSED, + prior_pending: false, + retirement_failed: false, + }), + changed: Condvar::new(), + terminal_notify: tokio::sync::Notify::new(), + resources: StdMutex::new(StartupResources::default()), + sdk_owners: AtomicUsize::new(0), + vtable_published: AtomicBool::new(false), + #[cfg(any(test, feature = "__native-fixture-support"))] + reject_worker_spawn: AtomicBool::new(false), + #[cfg(any(test, feature = "__native-fixture-support"))] + startup_failure_stage: AtomicU8::new(0), + #[cfg(any(test, feature = "__native-fixture-support"))] + transport_drop_thread: StdMutex::new(None), + lease: SlotLease { + admission: Arc::downgrade(admission), + index, + }, + token, + }); + slots[index].anchor = Some(state.clone()); + Some(state) + } + /// Opaque address token only; no allocation or dereference occurs. + pub fn context(&self) -> *mut c_void { + std::ptr::with_exposed_provenance_mut(self.token) + } + pub fn lookup(context: *mut c_void) -> Option> { + Self::lookup_in(ADMISSION.get()?, context.addr()) + } + fn lookup_in(admission: &Arc, token: usize) -> Option> { + let generation = token >> 8; + if generation == 0 { + return None; + } + let slots = admission + .slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let slot = &slots[token & (INSTANCE_LIMIT - 1)]; + if slot.generation != generation { + return None; + } + // Strong exact State is acquired before releasing registry exclusion. + slot.anchor + .as_ref() + .filter(|state| state.token == token) + .cloned() + } + pub fn mark_vtable_published(&self) { + self.vtable_published.store(true, Ordering::Release); + } + pub fn retain_uncertain_anchor(self: &Arc) { + self.uncertain.store(true, Ordering::Release); + if let Some(admission) = self.lease.admission.upgrade() { + let mut slots = admission + .slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let slot = &mut slots[self.lease.index]; + // This State's still-live lease forbids reuse while a C callback or + // retained view performs the final genuine-fault recheck. + if slot.generation == self.token >> 8 && slot.anchor.is_none() { + slot.anchor = Some(self.clone()); + } + } + } + pub fn view(self: &Arc) -> NativeInstanceLiveness { + NativeInstanceLiveness { + inner: self.clone(), + } + } + pub fn sdk_owner(self: &Arc) -> SdkOwner { + self.sdk_owners.fetch_add(1, Ordering::AcqRel); + SdkOwner { + state: self.clone(), + } + } + pub fn is_terminal(&self) -> bool { + self.reason.load(Ordering::Acquire) != 0 + } + pub fn is_live(self: &Arc) -> bool { + if self.is_terminal() { + return false; + } + self.observe_host(); + !self.is_terminal() + } + // Cleanup must sample even when local Stop already closed admission: a real + // later native Fault is orthogonal to that local admission predicate. + pub fn observe_host(self: &Arc) { + let flags = self + .host + .0 + .native_terminal_state + .map(|query| unsafe { query(self.host.0.host_ctx) }); + if flags != Some(TB_NATIVE_TERMINAL_LIVE) { + let reason = if flags.is_some_and(|flags| flags & TB_NATIVE_TERMINAL_FAULT != 0) { + TerminalReason::Fault + } else if flags.is_some_and(|flags| flags & TB_NATIVE_TERMINAL_STOP_INTENT != 0) { + TerminalReason::Stop + } else { + TerminalReason::HostClosed + }; + self.signal(reason); + } + } + fn reason(&self) -> TerminalReason { + match self.reason.load(Ordering::Acquire) { + 1 => TerminalReason::Stop, + 2 => TerminalReason::HostClosed, + 4 => TerminalReason::SetupFailed, + _ => TerminalReason::Fault, + } + } + fn commit(&self, reason: TerminalReason) { + let next = match reason { + TerminalReason::Stop => 1, + TerminalReason::HostClosed => 2, + TerminalReason::Fault => 3, + TerminalReason::SetupFailed => 4, + }; + let mut old = self.reason.load(Ordering::Acquire); + while old == 0 || (old < 3 && next >= 3) { + match self + .reason + .compare_exchange_weak(old, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => break, + Err(actual) => old = actual, + } + } + self.terminal_notify.notify_waiters(); + } + pub fn signal(self: &Arc, reason: TerminalReason) { + if matches!(reason, TerminalReason::Fault | TerminalReason::SetupFailed) { + self.host.fault(); + } + self.commit(reason); + self.run_request(); + self.schedule(); + } + pub fn source_panic(self: &Arc, kind: PayloadKind, payload: Payload) { + // Atomic authority fence must precede payload-cell locks and foreign Drop. + self.host.fault(); + self.commit(TerminalReason::Fault); + self.uncertain.store(true, Ordering::Release); + self.keep_payload(kind, payload); + self.run_request(); + self.schedule(); + } + fn lock(&self) -> std::sync::MutexGuard<'_, Control> { + self.control.lock().unwrap_or_else(|poison| { + self.host.fault(); + self.commit(TerminalReason::Fault); + self.uncertain.store(true, Ordering::Release); + poison.into_inner() + }) + } + fn keep_payload(&self, kind: PayloadKind, payload: Payload) { + let mut control = self.lock(); + let cell = control.payloads.cell(kind); + // Once/terminal admission prevents a second primary in this category. + // A violated internal invariant cannot discard a foreign payload here. + // Call sites admit this category once and deny all later invocation. + // No assertion/unwind, replacement retry, or payload disposal path. + *cell = Some(payload); + } + pub fn register(self: &Arc, hooks: RuntimeHooks) { + { + let mut control = self.lock(); + control.request = Some(hooks.request); + control.finish = Some(hooks.finish); + control.request_state = RequestState::Idle; + } + if self.is_terminal() { + self.run_request(); + self.schedule(); + } + } + fn run_request(self: &Arc) { + let request = { + let mut control = self.lock(); + if !self.is_terminal() || control.request_state != RequestState::Idle { + return; + } + control.request_state = RequestState::Running; + control.request.clone() + }; + let _scope = self.owner.as_ref().map(ExecutionOwner::enter); + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Some(request) = request { + request(self.reason()); + } + })); + match outcome { + Ok(()) => { + self.lock().request_state = RequestState::Done; + self.changed.notify_all(); + } + Err(payload) => { + self.host.fault(); + self.commit(TerminalReason::Fault); + self.uncertain.store(true, Ordering::Release); + self.lock().request_state = RequestState::Panicked; + self.changed.notify_all(); + self.keep_payload(PayloadKind::Request, payload); + } + } + } + pub fn publish(self: &Arc) { + self.lock().published = true; + self.changed.notify_all(); + if self.is_terminal() { + self.run_request(); + self.schedule(); + } + } + pub fn begin_reinit(&self) -> bool { + let mut control = self.lock(); + if self.is_terminal() || control.active_reinit { + return false; + } + control.active_reinit = true; + true + } + pub fn end_reinit(&self) { + self.lock().active_reinit = false; + self.changed.notify_all(); + } + pub fn retire_reinit(self: &Arc, future: Option) -> bool { + let Some(future) = future else { + return true; + }; + { + let mut control = self.lock(); + // active_reinit admits one caller. It cannot construct the next + // future until the worker has acknowledged this exact disposal. + control.reinit_operation = Some(ReinitOperation::Retire(future)); + control.prior_pending = true; + } + self.schedule(); + self.changed.notify_all(); + let mut control = self.lock(); + while control.prior_pending && control.finish_state != FinishState::SpawnFailed { + control = self + .changed + .wait(control) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + !control.retirement_failed && control.finish_state != FinishState::SpawnFailed + } + pub fn construct_reinit(self: &Arc, construct: Box i32 + Send>) -> i32 { + { + let mut control = self.lock(); + // The same sole cell owns construction captures before scheduling. + // Spawn failure retains them; no ABI caller receives generic C. + control.reinit_operation = Some(ReinitOperation::Construct(construct)); + control.prior_pending = true; + } + self.schedule(); + self.changed.notify_all(); + let mut control = self.lock(); + while control.prior_pending && control.finish_state != FinishState::SpawnFailed { + control = self + .changed + .wait(control) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + if control.finish_state == FinishState::SpawnFailed { + TB_CLOSED + } else { + control.construction_code + } + } + #[cfg(test)] + pub(super) fn transport_drop_thread_for_test(&self) -> Option { + self.transport_drop_thread.lock().unwrap().clone() + } + #[cfg(feature = "__native-fixture-support")] + fn resources_for_fixture(&self) -> u32 { + let resources = self.resources.lock().unwrap(); + let flags = [ + resources.retained_factory.is_some(), + resources.setup.is_some(), + resources.reinitializer_factory.is_some(), + resources.runtime.is_some(), + resources.transport.is_some(), + resources.reinitialize.lock().unwrap().is_some(), + resources.initial.is_some(), + resources.config.is_some(), + resources.initial_future.lock().unwrap().is_some(), + resources.reinit_future.lock().unwrap().is_some(), + ]; + flags + .iter() + .enumerate() + .fold(0, |mask, (index, held)| mask | (u32::from(*held) << index)) + } + #[cfg(any(test, feature = "__native-fixture-support"))] + pub(super) fn reject_runtime_build_for_fixture(&self) -> bool { + self.startup_failure_stage.load(Ordering::Acquire) == 9 + } + #[cfg(any(test, feature = "__native-fixture-support"))] + pub(super) fn reject_startup_for_test(&self, stage: u8) { + self.startup_failure_stage.store(stage, Ordering::Release); + } + #[cfg(any(test, feature = "__native-fixture-support"))] + pub(super) fn hold_initial_before_transfer_for_test(&self) -> bool { + matches!(self.startup_failure_stage.load(Ordering::Acquire), 7 | 8) + } + #[cfg(any(test, feature = "__native-fixture-support"))] + pub(super) fn startup_checkpoint_for_test(&self, stage: u8) { + if self.startup_failure_stage.load(Ordering::Acquire) == stage { + panic!("controlled startup failure after transfer stage {stage}"); + } + } + #[cfg(any(test, feature = "__native-fixture-support"))] + pub(super) fn reject_worker_spawn_for_test(&self) { + self.reject_worker_spawn.store(true, Ordering::Release); + } + #[cfg(any(test, feature = "__native-fixture-support"))] + pub(super) fn sdk_owner_count_for_test(&self) -> usize { + self.sdk_owners.load(Ordering::Acquire) + } + #[cfg(any(test, feature = "__native-fixture-support"))] + pub(super) fn request_done_for_test(&self) -> bool { + matches!( + self.lock().request_state, + RequestState::Done | RequestState::Panicked + ) + } + fn schedule(self: &Arc) { + let job = { + let mut control = self.lock(); + if control.finish_state != FinishState::Idle { + return; + } + let job = Arc::new(FinishJob { + state: Arc::downgrade(self), + }); + control.job = Some(job.clone()); + control.finish_state = FinishState::Submitted; + job + }; + let state = job.state.upgrade().expect("anchored terminal state"); + // Private per-instance rejection exercises the real submitted-job ownership + // path without changing process-wide thread creation or public authority. + #[cfg(any(test, feature = "__native-fixture-support"))] + let rejected = self.reject_worker_spawn.load(Ordering::Acquire); + #[cfg(not(any(test, feature = "__native-fixture-support")))] + let rejected = false; + let submitted = if rejected { + Err(std::io::Error::other( + "controlled terminal worker spawn failure", + )) + } else { + std::thread::Builder::new() + .name("tinybus-terminal".into()) + .spawn(move || job.run(state)) + }; + if submitted.is_err() { + self.uncertain.store(true, Ordering::Release); + let mut control = self.lock(); + control.finish_state = FinishState::SpawnFailed; + control.completion = Some(TerminalCompletion::Uncertain); + drop(control); + self.changed.notify_all(); + self.host.fault(); + self.commit(TerminalReason::Fault); + self.run_request(); + } + } + fn remove_anchor(&self) { + if let Some(admission) = self.lease.admission.upgrade() { + let anchor = { + let mut slots = admission + .slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + slots[self.lease.index].anchor.take() + }; + drop(anchor); + } + } + pub fn empty_factory_failure(self: &Arc) { + // No callable/controller was transferred. Retained views still hold the permit. + self.commit(TerminalReason::SetupFailed); + self.remove_anchor(); + } + fn run_finish(self: &Arc) { + let _scope = self.owner.as_ref().map(ExecutionOwner::enter); + loop { + let mut control = self.lock(); + // Retirement takes precedence over waiting for active_reinit: + // that caller may itself be awaiting this worker's acknowledgment. + if let Some(operation) = control.reinit_operation.take() { + drop(control); + let (failed, code) = match operation { + ReinitOperation::Retire(future) => { + match catch_unwind(AssertUnwindSafe(|| drop(future))) { + Ok(()) => (false, TB_OK), + Err(payload) => { + self.source_panic(PayloadKind::Cleanup, payload); + (true, TB_CLOSED) + } + } + } + ReinitOperation::Construct(construct) => { + match catch_unwind(AssertUnwindSafe(construct)) { + Ok(code) => (false, code), + Err(payload) => { + self.source_panic(PayloadKind::Reinit, payload); + (false, TB_PANICKED) + } + } + } + }; + let mut control = self.lock(); + control.retirement_failed |= failed; + control.construction_code = code; + control.prior_pending = false; + drop(control); + self.changed.notify_all(); + continue; + } + if control.published + && !control.active_reinit + && !matches!( + control.request_state, + RequestState::Idle | RequestState::Running + ) + { + control.finish_state = FinishState::Running; + break; + } + drop( + self.changed + .wait(control) + .unwrap_or_else(std::sync::PoisonError::into_inner), + ); + } + // Execution settlement is distinct from foreign capture destruction. + // A failed prior retirement retains all later captures, but does not + // spend the controller's still-uninvoked physical finish callback. + let execution_settled = self.settle_execution(); + let retirement_failed = self.lock().retirement_failed; + let disposal_failed = if execution_settled && !retirement_failed { + match catch_unwind(AssertUnwindSafe(|| self.dispose_setup())) { + Ok(()) => false, + Err(payload) => { + self.source_panic(PayloadKind::Cleanup, payload); + true + } + } + } else { + retirement_failed + }; + let finish = self.lock().finish.take(); + let completed = if execution_settled { + if let Some(owner) = &self.owner { + owner.stop(); + } + match catch_unwind(AssertUnwindSafe(|| { + finish.map_or(TerminalCompletion::PhysicallyStopped, |finish| finish()) + })) { + Ok(completion) => completion, + Err(payload) => { + self.source_panic(PayloadKind::Finish, payload); + TerminalCompletion::Uncertain + } + } + } else { + // If actual task settlement could not be proved, retain the original finish owner. + self.lock().finish = finish; + TerminalCompletion::Uncertain + }; + let released = if completed == TerminalCompletion::PhysicallyStopped && !disposal_failed { + match catch_unwind(AssertUnwindSafe(|| self.release_resources())) { + Ok(released) => released, + Err(payload) => { + self.source_panic(PayloadKind::Cleanup, payload); + false + } + } + } else { + false + }; + self.observe_host(); + if !released || (self.has_fault() && self.vtable_published.load(Ordering::Acquire)) { + self.uncertain.store(true, Ordering::Release); + } + let completion = if self.uncertain.load(Ordering::Acquire) { + TerminalCompletion::Uncertain + } else { + completed + }; + { + let mut control = self.lock(); + control.finish_state = FinishState::Complete; + control.completion = Some(completion); + } + self.changed.notify_all(); + if completion == TerminalCompletion::PhysicallyStopped { + self.remove_anchor(); + } + } + fn settle_execution(self: &Arc) -> bool { + let (initial, handle) = { + let mut resources = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ( + resources.initial.take(), + resources + .runtime + .as_ref() + .map(|runtime| runtime.handle().clone()), + ) + }; + if let Some(initial) = initial { + initial.abort(); + let Some(handle) = handle else { + self.resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .initial = Some(initial); + return false; + }; + match handle.block_on(initial) { + Ok(Ok(())) => {} + Ok(Err(_error)) => self.signal(TerminalReason::SetupFailed), + Err(error) if error.is_cancelled() => {} + Err(error) => { + if error.is_panic() { + self.source_panic(PayloadKind::Initial, error.into_panic()); + } else { + self.signal(TerminalReason::SetupFailed); + } + } + } + } + true + } + fn dispose_setup(&self) { + // The worker already observed active_reinit=false. Every last capture + // is taken mechanically and destroyed after all bookkeeping unlocks. + let (reinitialize, future, reinit_future) = { + let resources = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ( + resources.reinitialize.clone(), + resources.initial_future.clone(), + resources.reinit_future.clone(), + ) + }; + if let Some(owner) = &self.owner { + owner.stop(); + } + let future = future + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + drop(future); + let callback = reinitialize + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + drop(callback); + let request = self.lock().request.take(); + drop(request); + let future = reinit_future + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + drop(future); + } + fn release_resources(&self) -> bool { + // Take one owner at a time. If its foreign Drop panics, the enclosing + // single cleanup catch stops here and every later owner stays anchored. + let setup = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .setup + .take(); + drop(setup); + let factory = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retained_factory + .take(); + drop(factory); + let config = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .config + .take(); + drop(config); + let reinit_factory = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .reinitializer_factory + .take(); + drop(reinit_factory); + let connection = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .connection + .clone(); + let connection = connection + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + drop(connection); + let inbound = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .inbound + .take(); + drop(inbound); + let transport = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .transport + .take(); + drop(transport); + let runtime = self + .resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .runtime + .take(); + if let Some(runtime) = runtime { + runtime.shutdown_timeout(Duration::from_secs(4)); + } + // Timeout is not proof: every surviving SDK transport/Connection/method keeps State + // and its resource count. Unknown surviving ownership stays anchored. + self.sdk_owners.load(Ordering::Acquire) == 0 + } + pub fn has_fault(&self) -> bool { + self.reason.load(Ordering::Acquire) >= 3 + } + pub fn mark_uncertain(&self) { + self.uncertain.store(true, Ordering::Release); + } + pub fn wait(&self, deadline: Duration) -> TerminalCompletion { + let control = self.lock(); + let (control, _) = self + .changed + .wait_timeout_while(control, deadline, |control| control.completion.is_none()) + .unwrap_or_else(std::sync::PoisonError::into_inner); + control.completion.unwrap_or(TerminalCompletion::Uncertain) + } +} +#[cfg(test)] +#[path = "terminal_tests.rs"] +mod tests; diff --git a/crates/tinybus-module/src/terminal_tests.rs b/crates/tinybus-module/src/terminal_tests.rs new file mode 100644 index 00000000..cd829cbf --- /dev/null +++ b/crates/tinybus-module/src/terminal_tests.rs @@ -0,0 +1,350 @@ +use super::*; +use std::sync::atomic::AtomicU32; +use tinybus::module::abi::{TB_NATIVE_TERMINAL_STOP_INTENT, TbSlice}; +static TEST_LOCK: StdMutex<()> = StdMutex::new(()); +static FLAGS: AtomicU32 = AtomicU32::new(TB_NATIVE_TERMINAL_LIVE); +unsafe extern "C" fn query(_: *mut c_void) -> u32 { + FLAGS.load(Ordering::Acquire) +} +unsafe extern "C" fn send(_: *mut c_void, _: *const u8, _: usize) -> i32 { + TB_OK +} +unsafe extern "C" fn notify(_: *mut c_void) {} +unsafe extern "C" fn fault(_: *mut c_void, _: *const u8, _: usize) { + FLAGS.store(4, Ordering::Release); +} +unsafe extern "C" fn log(_: *mut c_void, _: u32, _: *const u8, _: usize) {} +fn host() -> HostCalls { + HostCalls(TbHostVtable { + size: size_of::() as u32, + _reserved: 0, + host_ctx: std::ptr::null_mut(), + send, + wake: notify, + log, + ready: notify, + fault, + config: TbSlice { + ptr: std::ptr::null(), + len: 0, + }, + broker_routing: None, + native_terminal_state: Some(query), + }) +} +#[test] +fn observed_native_terminal_never_reopens_a_retained_view() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let view = state.view(); + assert!(view.is_live()); + FLAGS.store(TB_NATIVE_TERMINAL_STOP_INTENT, Ordering::Release); + assert!(!view.is_live()); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + assert!( + !view.is_live(), + "an observed terminal tombstone must stay closed" + ); + state.publish(); + assert_eq!( + state.wait(Duration::from_secs(3)), + TerminalCompletion::PhysicallyStopped + ); +} + +#[test] +fn retired_anchor_cannot_refund_permit_while_old_view_survives() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + let admission = Admission::new(); + let mut views = Vec::new(); + for _ in 0..INSTANCE_LIMIT { + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + views.push(state.view()); + state.empty_factory_failure(); + } + assert!(TerminalState::admit_in(&admission, host(), None).is_none()); + views.pop(); + let replacement = TerminalState::admit_in(&admission, host(), None).unwrap(); + replacement.empty_factory_failure(); + drop(replacement); + drop(views); + assert!( + admission + .slots + .lock() + .unwrap() + .iter() + .all(|slot| !slot.reserved) + ); +} + +#[test] +fn request_primary_payload_is_retained_and_finish_still_attempted_once() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + use std::sync::atomic::AtomicUsize; + struct PayloadDrop(Arc); + impl Drop for PayloadDrop { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let dropped = Arc::new(AtomicUsize::new(0)); + let requests = Arc::new(AtomicUsize::new(0)); + let finishes = Arc::new(AtomicUsize::new(0)); + let (request_count, finish_count, payload) = + (requests.clone(), finishes.clone(), dropped.clone()); + state.register(RuntimeHooks::new( + move |_| { + request_count.fetch_add(1, Ordering::SeqCst); + std::panic::panic_any(PayloadDrop(payload.clone())); + }, + move || { + finish_count.fetch_add(1, Ordering::SeqCst); + TerminalCompletion::PhysicallyStopped + }, + )); + state.publish(); + state.signal(TerminalReason::Stop); + assert_eq!( + state.wait(Duration::from_secs(3)), + TerminalCompletion::Uncertain + ); + state.signal(TerminalReason::Fault); + assert_eq!(requests.load(Ordering::SeqCst), 1); + assert_eq!(finishes.load(Ordering::SeqCst), 1); + assert_eq!(dropped.load(Ordering::SeqCst), 0); + assert!(state.lock().request_state == RequestState::Panicked); + assert!( + admission.slots.lock().unwrap()[state.lease.index] + .anchor + .is_some() + ); +} + +#[test] +fn physical_finish_waits_for_bounded_request_completion_and_startup_publication() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let release = Arc::new(std::sync::Barrier::new(2)); + let request_release = release.clone(); + let finished = Arc::new(AtomicBool::new(false)); + let finish_seen = finished.clone(); + state.register(RuntimeHooks::new( + move |_| { + entered_tx.send(()).unwrap(); + request_release.wait(); + }, + move || { + finish_seen.store(true, Ordering::Release); + TerminalCompletion::PhysicallyStopped + }, + )); + let caller_state = state.clone(); + let caller = std::thread::spawn(move || caller_state.signal(TerminalReason::Stop)); + entered_rx.recv().unwrap(); + state.publish(); + assert!(!finished.load(Ordering::Acquire)); + release.wait(); + caller.join().unwrap(); + assert_eq!( + state.wait(Duration::from_secs(3)), + TerminalCompletion::PhysicallyStopped + ); + assert!(finished.load(Ordering::Acquire)); +} + +#[test] +fn surviving_sdk_owned_transport_keeps_cleanup_uncertain_and_permit_anchored() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let sdk_owner = state.sdk_owner(); + state.register(RuntimeHooks::new( + |_| {}, + || TerminalCompletion::PhysicallyStopped, + )); + state.publish(); + state.signal(TerminalReason::Stop); + assert_eq!( + state.wait(Duration::from_secs(3)), + TerminalCompletion::Uncertain + ); + drop(sdk_owner); + assert!( + admission.slots.lock().unwrap()[state.lease.index] + .anchor + .is_some() + ); +} + +#[test] +fn stale_context_never_selects_replacement_and_unknown_contexts_are_closed() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let old = state.context().addr(); + state.empty_factory_failure(); + drop(state); + let replacement = TerminalState::admit_in(&admission, host(), None).unwrap(); + assert_eq!(replacement.lease.index, old & 255); + assert_ne!(replacement.context().addr(), old); + assert!(TerminalState::lookup_in(&admission, old).is_none()); + for invalid in [0, 1, 255, usize::MAX] { + assert!(TerminalState::lookup_in(&admission, invalid).is_none()); + } + assert!(Arc::ptr_eq( + &TerminalState::lookup_in(&admission, replacement.token).unwrap(), + &replacement + )); + replacement.empty_factory_failure(); +} +#[test] +fn exhausted_slot_generation_never_wraps_or_selects_old_context() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + let admission = Admission::new(); + { + let mut slots = admission.slots.lock().unwrap(); + slots[0].generation = usize::MAX >> 8; + for slot in &mut slots[1..] { + slot.exhausted = true; + } + } + assert!(TerminalState::admit_in(&admission, host(), None).is_none()); + assert!(admission.slots.lock().unwrap()[0].exhausted); + assert!(TerminalState::lookup_in(&admission, (usize::MAX >> 8) << 8).is_none()); +} +#[test] +fn callback_lookup_holds_exact_state_across_retirement_until_actual_last_drop() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let admitted = TerminalState::lookup_in(&admission, state.token).unwrap(); + let index = state.lease.index; + state.empty_factory_failure(); + drop(state); + assert!(admission.slots.lock().unwrap()[index].reserved); + assert_eq!(admitted.lease.index, index); + drop(admitted); + assert!(!admission.slots.lock().unwrap()[index].reserved); +} + +#[test] +fn first_cleanup_drop_panic_retains_all_unprocessed_resource_owners() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + struct FirstDrop; + impl Drop for FirstDrop { + fn drop(&mut self) { + panic!("controlled first cleanup Drop"); + } + } + struct RemainingDrop(Arc); + impl Drop for RemainingDrop { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let admission = Admission::new(); + let state = TerminalState::admit_in(&admission, host(), None).unwrap(); + let remaining = Arc::new(AtomicUsize::new(0)); + { + let mut resources = state.resources.lock().unwrap(); + let first = FirstDrop; + resources.setup = Some(Box::new(move |_| { + drop(first); + Box::pin(async { Ok(()) }) + })); + resources.retained_factory = Some(Box::new(RemainingDrop(remaining.clone()))); + } + state.register(RuntimeHooks::new( + |_| {}, + || TerminalCompletion::PhysicallyStopped, + )); + state.publish(); + state.signal(TerminalReason::Stop); + let completion = state.wait(Duration::from_secs(3)); + let dropped = remaining.load(Ordering::SeqCst); + assert_eq!(completion, TerminalCompletion::Uncertain); + assert_eq!( + dropped, 0, + "first cleanup panic must leave later foreign owners attached" + ); + assert!(state.resources.lock().unwrap().retained_factory.is_some()); +} + +#[cfg(feature = "__native-fixture-support")] +#[test] +fn retained_view_reports_exact_execution_owner_and_surviving_sdk_resources() { + let _serial = TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + FLAGS.store(TB_NATIVE_TERMINAL_LIVE, Ordering::Release); + let admission = Admission::new(); + let owner = ExecutionOwner::new(host()); + let state = TerminalState::admit_in(&admission, host(), Some(owner.clone())).unwrap(); + let view = state.view(); + assert!(!view.__fixture_owner_is_current()); + assert!(!view.__fixture_owner_is_stopped()); + { + let _scope = owner.enter(); + assert!(view.__fixture_owner_is_current()); + let other = ExecutionOwner::new(host()); + { + let _other_scope = other.enter(); + assert!(!view.__fixture_owner_is_current()); + } + assert!(view.__fixture_owner_is_current()); + } + assert!(!view.__fixture_owner_is_current()); + assert_eq!(view.__fixture_resources(), 0); + let sdk_owner = state.sdk_owner(); + assert_eq!(view.__fixture_sdk_owners(), 1); + assert!(!view.__fixture_request_done()); + drop(sdk_owner); + assert_eq!(view.__fixture_sdk_owners(), 0); + state.register(RuntimeHooks::new( + |_| {}, + || TerminalCompletion::PhysicallyStopped, + )); + state.publish(); + view.request_fault(); + assert_eq!( + state.wait(Duration::from_secs(3)), + TerminalCompletion::PhysicallyStopped + ); + assert!(view.__fixture_request_done()); + assert!(view.__fixture_owner_is_stopped()); + assert!(!view.is_live()); +} diff --git a/crates/tinybus/Cargo.toml b/crates/tinybus/Cargo.toml index b98b3681..3a79ee10 100644 --- a/crates/tinybus/Cargo.toml +++ b/crates/tinybus/Cargo.toml @@ -48,6 +48,8 @@ toml = { version = "1", optional = true } libc = { version = "0.2", optional = true } [features] +# Explicit private native lifecycle fixture cohort; never part of defaults. +__native-fixture-support = ["tinybus-module/__native-fixture-support"] # `uds` and `macros` are on by default because a bus you cannot connect to over # a socket, and an interface you have to hand-dispatch, are not what anyone is # installing. The CLI is opt-in: embedding the library should not pull in @@ -140,3 +142,8 @@ required-features = ["modules"] name = "module_controlled_init" crate-type = ["cdylib"] required-features = ["modules"] + +[[example]] +name = "module_terminal" +crate-type = ["cdylib"] +required-features = ["modules"] diff --git a/crates/tinybus/examples/module_terminal.rs b/crates/tinybus/examples/module_terminal.rs new file mode 100644 index 00000000..50044c2c --- /dev/null +++ b/crates/tinybus/examples/module_terminal.rs @@ -0,0 +1,675 @@ +//! Actual opted-in runtime fixture. Controls are trusted test-only C callbacks. +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::{ + ffi::c_void, + future::Future, + pin::Pin, + sync::{Arc, Condvar, Mutex}, + task::{Context, Poll}, +}; +use tinybus::{Connection, Result}; +static DROPPED: AtomicUsize = AtomicUsize::new(0); +static EARLY: Mutex> = Mutex::new(None); +static NEXT_FACTORY_MODE: AtomicUsize = AtomicUsize::new(0); +static FACTORIES: AtomicUsize = AtomicUsize::new(0); +static FACTORY_DISPATCHES: Mutex> = Mutex::new(Vec::new()); +static EXTERNAL_CONTROLLER: Mutex>> = Mutex::new(None); +static RETAINED_VIEWS: Mutex> = Mutex::new(Vec::new()); +#[derive(Clone, Copy)] +struct NativeQuery { + ctx: *mut c_void, + query: unsafe extern "C" fn(*mut c_void) -> u32, +} +unsafe impl Send for NativeQuery {} +static NATIVE_QUERY: Mutex> = Mutex::new(None); +use tinybus::module::abi::*; +use tinybus_module::{NativeInstanceLiveness, RuntimeHooks, TerminalCompletion}; +#[derive(serde::Deserialize)] +struct Config { + mode: u8, + test_controls: Vec, +} +impl Drop for Config { + fn drop(&mut self) { + if self.test_controls.len() == size_of::() { + let controls = unsafe { + self.test_controls + .as_ptr() + .cast::() + .read_unaligned() + }; + controls.emit( + 19, + u32::from(std::thread::current().name() == Some("tinybus-terminal")), + ); + } + } +} +#[repr(C)] +#[derive(Clone, Copy)] +struct Controls { + ctx: *mut c_void, + event: unsafe extern "C" fn(*mut c_void, u32, u32), +} +unsafe impl Send for Controls {} +unsafe impl Sync for Controls {} +impl Controls { + fn emit(self, event: u32, value: u32) { + unsafe { (self.event)(self.ctx, event, value) } + } +} +struct PrimaryPayload(Option); +impl Drop for PrimaryPayload { + fn drop(&mut self) { + if let Some(controls) = self.0 { + controls.emit(17, 1); + } + } +} +struct Controller { + controls: Mutex>, + view: NativeInstanceLiveness, + mode: AtomicUsize, + wake: (Mutex, Condvar), + worker: Mutex>>, +} +impl Controller { + fn observe(&self, event: u32) { + #[cfg(feature = "__native-fixture-support")] + self.emit( + event, + u32::from(self.view.__fixture_owner_is_current()) + | (u32::from(self.view.__fixture_owner_is_stopped()) << 1) + | (u32::from(self.view.__fixture_request_done()) << 2) + | (u32::from(self.view.__fixture_sdk_owners() != 0) << 3), + ); + #[cfg(not(feature = "__native-fixture-support"))] + let _ = event; + } + fn emit(&self, event: u32, value: u32) { + let controls = *self + .controls + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(controls) = controls { + controls.emit(event, value); + } + } +} +impl Drop for Controller { + fn drop(&mut self) { + self.observe(15); + self.emit( + 6, + u32::from(std::thread::current().name() == Some("tinybus-terminal")), + ); + DROPPED.fetch_add(1, Ordering::Release); + } +} +struct FactoryCapture(Arc); +impl FactoryCapture { + fn controller(&self) -> &Arc { + &self.0 + } +} +impl Drop for FactoryCapture { + fn drop(&mut self) { + self.0.emit( + 20, + u32::from(std::thread::current().name() == Some("tinybus-terminal")), + ); + } +} +struct SetupFuture { + controller: Arc, + mode: u8, + first: bool, +} +impl Future for SetupFuture { + type Output = Result<()>; + fn poll(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { + if self.first { + self.first = false; + self.controller.observe(11); + self.controller.emit(2, u32::from(self.mode)); + } + if self.mode == 41 || self.mode == 43 || self.mode == 45 { + self.controller.emit(24, u32::from(self.mode)); + } + match self.mode { + 43 => Poll::Ready(Err(tinybus::Error::failed( + "ready native reinit source error", + ))), + 45 => std::panic::panic_any(PrimaryPayload(*self.controller.controls.lock().unwrap())), + 0 | 48 | 49 => Poll::Ready(Ok(())), + 2 => panic!("controlled native setup poll"), + 5 => Poll::Ready(Err(tinybus::Error::failed("controlled native setup error"))), + _ => Poll::Pending, + } + } +} +impl Drop for SetupFuture { + fn drop(&mut self) { + if self.mode == 48 || self.mode == 49 { + self.controller.observe(39); + self.controller + .emit(38, u32::from(tokio::runtime::Handle::try_current().is_ok())); + } + if self.mode == 42 || self.mode == 47 || self.mode == 48 || self.mode == 49 { + self.controller.emit( + 25, + u32::from(std::thread::current().name() == Some("tinybus-terminal")), + ); + } + if self.mode == 47 || self.mode == 49 { + std::panic::panic_any(PrimaryPayload(*self.controller.controls.lock().unwrap())); + } + self.controller.observe(14); + self.controller.emit( + 4, + u32::from(std::thread::current().name() == Some("tinybus-terminal")), + ); + } +} +type FixtureFuture = Pin> + Send>>; +type FixtureSetup = Box FixtureFuture + Send + Sync>; +fn factory(view: NativeInstanceLiveness) -> Result<(FixtureSetup, RuntimeHooks)> { + FACTORIES.fetch_add(1, Ordering::AcqRel); + let factory_mode = NEXT_FACTORY_MODE.swap(0, Ordering::AcqRel); + if factory_mode == 46 { + FACTORY_DISPATCHES + .lock() + .unwrap() + .push(tracing::dispatcher::get_default(Clone::clone)); + } + if factory_mode == 3 || (20..=29).contains(&factory_mode) || factory_mode == 46 { + RETAINED_VIEWS.lock().unwrap().push(view.clone()); + } + let early = EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + if let Some(controls) = early { + controls.emit(9, factory_mode as u32); + } + if factory_mode == 1 { + return Err(tinybus::Error::failed("controlled factory error")); + } + if factory_mode == 2 { + std::panic::panic_any(PrimaryPayload(early)); + } + let held_finish = early.is_some() && factory_mode == 0; + let controller = Arc::new(Controller { + controls: Mutex::new(early), + view, + mode: AtomicUsize::new(factory_mode), + wake: (Mutex::new(false), Condvar::new()), + worker: Mutex::new(None), + }); + controller.observe(10); + controller.observe(27); + #[cfg(feature = "__native-fixture-support")] + match factory_mode { + 20 => controller.view.__fixture_reject_worker_spawn(), + 21..=29 => controller + .view + .__fixture_reject_startup((factory_mode - 20) as u8), + _ => {} + } + if factory_mode == 20 { + *EXTERNAL_CONTROLLER.lock().unwrap() = Some(Arc::downgrade(&controller)); + } + if held_finish { + controller.view.request_fault(); + } + let (request, finish) = (controller.clone(), controller.clone()); + let capture = FactoryCapture(controller); + let setup = move |connection: Connection, config: Config| { + let controller = capture.controller(); + controller.observe(16); + controller.emit(26, u32::from(tokio::runtime::Handle::try_current().is_ok())); + controller.observe(28); + if config.test_controls.len() != size_of::() { + return Box::pin(async { Err(tinybus::Error::failed("bad test controls")) }) + as Pin> + Send>>; + } + // SAFETY: this explicit fixture accepts trusted in-process test pointers. + let controls = unsafe { + config + .test_controls + .as_ptr() + .cast::() + .read_unaligned() + }; + controller + .mode + .store(usize::from(config.mode), Ordering::Release); + *controller + .controls + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(controls); + let mut worker = controller + .worker + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if worker.is_none() { + let work = controller.clone(); + *worker = Some( + std::thread::Builder::new() + .name("native-terminal-controller".into()) + .spawn(move || { + let mut cancelled = work + .wake + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + while !*cancelled { + cancelled = work + .wake + .1 + .wait(cancelled) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + drop(cancelled); + let denied = !work.view.is_live(); + work.emit(7, u32::from(denied)); + denied + }) + .expect("controlled worker spawn"), + ); + } + drop(worker); + if config.mode == 40 || config.mode == 44 { + controller.emit(23, u32::from(config.mode)); + } + if config.mode == 44 { + std::panic::panic_any(PrimaryPayload(*controller.controls.lock().unwrap())); + } + if config.mode == 4 { + panic!("controlled native setup construction"); + } + if (50..=52).contains(&config.mode) { + let object = TerminalObject { + controller: controller.clone(), + mode: config.mode, + }; + let work = controller.clone(); + return Box::pin(async move { + connection.serve_at("/terminal".parse()?, object).await?; + connection + .request_name("ai.tinyhumans.TerminalFixture") + .await?; + work.emit(31, 1); + Ok(()) + }) as FixtureFuture; + } + Box::pin(SetupFuture { + controller: controller.clone(), + mode: config.mode, + first: true, + }) as Pin> + Send>> + }; + let hooks = RuntimeHooks::new( + move |_| { + request.observe(12); + #[cfg(feature = "__native-fixture-support")] + request.emit(22, request.view.__fixture_resources()); + *request + .wake + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = true; + request.wake.1.notify_all(); + request.emit(3, 1); + if request.mode.load(Ordering::Acquire) == 10 { + std::panic::panic_any(PrimaryPayload(*request.controls.lock().unwrap())); + } + }, + move || { + finish.observe(13); + if held_finish { + finish.emit(8, 0); + } + let worker = finish + .worker + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let outcome = worker.map(|worker| worker.join()); + let joined = outcome.is_none() || matches!(outcome, Some(Ok(true))); + finish.emit(5, u32::from(joined)); + match finish.mode.load(Ordering::Acquire) { + 11 => std::panic::panic_any(PrimaryPayload(*finish.controls.lock().unwrap())), + 12 => return TerminalCompletion::Uncertain, + _ => {} + } + if joined { + TerminalCompletion::PhysicallyStopped + } else { + TerminalCompletion::Uncertain + } + }, + ); + Ok((Box::new(setup), hooks)) +} +#[unsafe(no_mangle)] +pub static TINYBUS_MODULE_ABI_V1: TbAbiDescriptor = + TbAbiDescriptor::current("module-terminal", env!("CARGO_PKG_VERSION")); +#[unsafe(no_mangle)] +pub extern "C" fn tinybus_module_manifest_v1() -> TbSlice { + tinybus_module::manifest_slice(tinybus_module::ManifestDeclaration { + name: "module-terminal", + version: env!("CARGO_PKG_VERSION"), + provides: &["ai.tinyhumans.TerminalFixture"], + methods: &["Panic"], + signals: &[], + requires: &[], + optional: &[], + lazy: false, + worker_threads: 1, + }) +} +#[derive(Clone, Copy)] +struct Table(TbModuleVtable); +unsafe impl Send for Table {} +static LATEST: Mutex> = Mutex::new(None); +/// # Safety +/// Standard trusted module init contract plus live test controls in config. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn tinybus_module_init_v1( + host: *const TbHostVtable, + out: *mut TbModuleVtable, +) -> i32 { + if let Some(host) = unsafe { TbHostVtable::read_compatible(host) } { + // The fixture needs its trusted callback before factory evaluation. + // Product authority still comes exclusively from the untouched host query. + if !host.config.ptr.is_null() { + let bytes = unsafe { std::slice::from_raw_parts(host.config.ptr, host.config.len) }; + if let Ok(config) = serde_json::from_slice::(bytes) { + if let Some(encoded) = config + .get("test_controls") + .and_then(|value| serde_json::from_value::>(value.clone()).ok()) + .filter(|bytes| bytes.len() == size_of::()) + { + let controls = unsafe { encoded.as_ptr().cast::().read_unaligned() }; + let mut early = EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if early.is_none() { + *early = Some(controls); + NEXT_FACTORY_MODE.store(99, Ordering::Release); + } + } + } + } + if let Some(query) = host.native_terminal_state { + *NATIVE_QUERY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(NativeQuery { + ctx: host.host_ctx, + query, + }); + } + } + if NEXT_FACTORY_MODE.load(Ordering::Acquire) == 30 { + let controls = *EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(controls) = controls { + controls.emit(18, 0); + } + } + let factories_before = FACTORIES.load(Ordering::Acquire); + let code = unsafe { + tinybus_module::start_terminal_reconfigurable_module_with_factory::( + host, out, 1, true, factory, + ) + }; + if FACTORIES.load(Ordering::Acquire) == factories_before { + EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + NEXT_FACTORY_MODE.store(0, Ordering::Release); + } + if code == TB_OK { + *LATEST + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Table(unsafe { *out })); + } + code +} +/// Test-only stale-context probe, using the same actual SDK callback. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_context() -> usize { + LATEST + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .map_or(0, |table| table.0.module_ctx.addr()) +} +/// Test-only copied callback probe; the token is never dereferenced. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_old_context(token: usize) -> i32 { + let table = *LATEST + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + table.map_or(TB_CLOSED, |table| unsafe { + (table.0.shutdown)(std::ptr::with_exposed_provenance_mut(token), 1000) + }) +} + +/// Test-only acquire fence after the final controller event callback returned. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_drops() -> usize { + DROPPED.load(Ordering::Acquire) +} + +/// # Safety +/// The test context and callback must remain live until actual controller Drop. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn terminal_test_early( + ctx: *mut c_void, + event: unsafe extern "C" fn(*mut c_void, u32, u32), +) { + *EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Controls { ctx, event }); +} +/// Sample only the actual copied host query; no local or fabricated success bit. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_native_flags() -> u32 { + let query = *NATIVE_QUERY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + query.map_or(0, |query| unsafe { (query.query)(query.ctx) }) +} + +/// # Safety +/// Controls must remain live through all emitted callbacks. Modes only configure +/// this example fixture; they do not grant native permission or construct views. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn terminal_test_factory_mode( + mode: usize, + ctx: *mut c_void, + event: unsafe extern "C" fn(*mut c_void, u32, u32), +) { + *EARLY + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Controls { ctx, event }); + NEXT_FACTORY_MODE.store(mode, Ordering::Release); +} +/// Counts actual factory invocations, including those which return no hooks. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_factories() -> usize { + FACTORIES.load(Ordering::Acquire) +} +/// Samples escaped actual factory-provided views against their copied host query. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_retained_live() -> usize { + let views = RETAINED_VIEWS.lock().unwrap().clone(); + views.iter().filter(|view| view.is_live()).count() +} +/// Release escaped actual views outside their fixture storage lock. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_release_views() -> usize { + let views = std::mem::take(&mut *RETAINED_VIEWS.lock().unwrap()); + let count = views.len(); + drop(views); + count +} +/// Release one genuinely retained view, allowing its actual State last-Drop. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_release_one_view() -> bool { + let view = { + let mut views = RETAINED_VIEWS.lock().unwrap(); + if views.is_empty() { + None + } else { + Some(views.remove(0)) + } + }; + let released = view.is_some(); + drop(view); + released +} + +/// Observes resources of the last genuinely escaped factory view. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_resources() -> u32 { + #[cfg(feature = "__native-fixture-support")] + { + let view = RETAINED_VIEWS.lock().unwrap().last().cloned(); + view.map_or(0, |view| view.__fixture_resources()) + } + #[cfg(not(feature = "__native-fixture-support"))] + { + 0 + } +} +/// Observes actual received transport disposal, independently of finish. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_transport_drop() -> bool { + #[cfg(feature = "__native-fixture-support")] + { + let view = RETAINED_VIEWS.lock().unwrap().last().cloned(); + view.is_some_and(|view| view.__fixture_transport_dropped_on_physical_worker()) + } + #[cfg(not(feature = "__native-fixture-support"))] + { + false + } +} +/// Explicit fixture cleanup after SDK spawn rejection; never claims SDK finish. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_external_join() -> bool { + let controller = EXTERNAL_CONTROLLER + .lock() + .unwrap() + .take() + .and_then(|weak| weak.upgrade()); + let Some(controller) = controller else { + return false; + }; + let worker = controller.worker.lock().unwrap().take(); + let joined = worker.is_some_and(|worker| worker.join().unwrap_or(false)); + controller.emit(21, u32::from(joined)); + joined +} + +/// Invoke the exact exported SDK reinitializer with a previously observed token. +/// # Safety +/// Input must remain readable throughout this synchronous callback. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn terminal_test_reinit(token: usize, ptr: *const u8, len: usize) -> i32 { + let table = *LATEST.lock().unwrap(); + table + .and_then(|table| table.0.reinitialize) + .map_or(TB_CLOSED, |reinit| unsafe { + reinit(std::ptr::with_exposed_provenance_mut(token), ptr, len) + }) +} +/// Current SDK owner observation on an external caller, with no scope installed. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_unowned() -> bool { + #[cfg(feature = "__native-fixture-support")] + { + let views = RETAINED_VIEWS.lock().unwrap().clone(); + views.iter().all(|view| !view.__fixture_owner_is_current()) + } + #[cfg(not(feature = "__native-fixture-support"))] + { + true + } +} + +struct MethodPayload(Arc); +impl Drop for MethodPayload { + fn drop(&mut self) { + let query = *NATIVE_QUERY.lock().unwrap(); + let native = query.map_or(0, |query| unsafe { (query.query)(query.ctx) }); + #[cfg(feature = "__native-fixture-support")] + let evidence = u32::from(native & TB_NATIVE_TERMINAL_FAULT != 0) + | (u32::from(self.0.view.__fixture_request_done()) << 1) + | (u32::from(self.0.view.__fixture_sdk_owners() != 0) << 2); + #[cfg(not(feature = "__native-fixture-support"))] + let evidence = native; + // First payload-disposal action reads the actual native query and + // actual request state before entering the trusted blocking control. + self.0.emit(29, evidence); + self.0.emit(30, 0); + self.0.emit(33, 1); + } +} +struct TerminalObject { + controller: Arc, + mode: u8, +} +#[tinybus::interface(name = "ai.tinyhumans.TerminalFixture")] +impl TerminalObject { + async fn panic(&self) -> Result<()> { + std::panic::panic_any(MethodPayload(self.controller.clone())); + } +} +impl Drop for TerminalObject { + fn drop(&mut self) { + if self.mode == 51 || self.mode == 52 { + self.controller.observe(35); + self.controller.emit(32, u32::from(self.mode)); + self.controller.emit(33, u32::from(self.mode)); + if self.mode == 52 { + panic!("controlled native Connection object destructor"); + } + } + } +} +/// Counts actual surviving SDK Connection/task owners, not messages/futures. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_sdk_owners() -> usize { + #[cfg(feature = "__native-fixture-support")] + { + let view = RETAINED_VIEWS.lock().unwrap().last().cloned(); + view.map_or(0, |view| view.__fixture_sdk_owners()) + } + #[cfg(not(feature = "__native-fixture-support"))] + { + 0 + } +} + +/// Probe a dispatcher captured by the genuine factory's actual caller scope. +/// It supplies no execution owner or native admission authority. +#[unsafe(no_mangle)] +pub extern "C" fn terminal_test_dispatch(index: usize) -> bool { + let dispatch = FACTORY_DISPATCHES.lock().unwrap().get(index).cloned(); + dispatch.is_some_and(|dispatch| { + tracing::dispatcher::with_default(&dispatch, || { + let span = tracing::info_span!("terminal_factory_dispatch"); + let enabled = span + .metadata() + .is_some_and(|metadata| dispatch.enabled(metadata)); + tracing::info!("retained native factory dispatcher"); + enabled + }) + }) +} diff --git a/crates/tinybus/src/connection.rs b/crates/tinybus/src/connection.rs index 6d4727bb..31fddc18 100644 --- a/crates/tinybus/src/connection.rs +++ b/crates/tinybus/src/connection.rs @@ -196,6 +196,9 @@ struct Inner { /// tree because a chunk has to be checked against the header's stamped /// `sender`; interfaces receive only the verified subset in CallContext. streams: StreamRegistry, + // Last: opted-in observers retain whole-Connection teardown ownership, + // including ObjectTree, streams and cloned method observers. + catch_observer: std::sync::RwLock>>, } /// Closes the transport when the last [`Connection`] handle goes away. @@ -389,6 +392,7 @@ impl Connection { signal_deliveries, local_publisher: crate::LocalPublisherIdentity::new(), panic_handler: std::sync::RwLock::new(None), + catch_observer: std::sync::RwLock::new(None), streams: StreamRegistry::new(), }); tokio::spawn(writer_loop(inner.transport.clone(), outbound)); @@ -1021,6 +1025,19 @@ impl Connection { .expect("panic handler lock") = Some(handler); } + /// Install SDK-local catch-time notification before a method panic payload drops. + /// Requires `__set_panic_handler`: without a handler no catch fence is constructed. + /// The observer must be bounded, must not unwind, and must never join workers. + /// This plumbing does not grant native authority or replace the panic policy. + #[doc(hidden)] + pub fn __set_catch_observer(&self, observer: Arc) { + *self + .inner + .catch_observer + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(observer); + } + /// What this connection will accept from peers sending it bulk streams. pub fn stream_limits(&self) -> StreamLimits { self.inner.streams.limits() @@ -1444,12 +1461,22 @@ async fn handle_call(inner: Arc, message: Message, brokered: bool) { .read() .expect("panic handler lock") .clone(); + let catch_observer = inner + .catch_observer + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); let result = if header.stream_reply && header.confidential { Err(Error::protocol( "a confidential call cannot request a streamed reply", )) } else if let Some(panic_handler) = panic_handler { - match CatchUnwind::new(dispatch(&inner, &header, message.body, brokered)).await { + match CatchUnwind::new( + dispatch(&inner, &header, message.body, brokered), + catch_observer, + ) + .await + { Ok(result) => result, Err(()) => Err(panic_handler()), } @@ -1537,12 +1564,14 @@ async fn send_reply(inner: &Inner, mut reply: Message) -> bool { struct CatchUnwind { future: Pin>, + observer: Option>, } impl CatchUnwind { - fn new(future: F) -> Self { + fn new(future: F, observer: Option>) -> Self { Self { future: Box::pin(future), + observer, } } } @@ -1551,11 +1580,20 @@ impl Future for CatchUnwind { type Output = std::result::Result; fn poll(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll { - let future = self.get_mut().future.as_mut(); + let this = self.get_mut(); + let future = this.future.as_mut(); match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| future.poll(context))) { Ok(Poll::Ready(value)) => Poll::Ready(Ok(value)), Ok(Poll::Pending) => Poll::Pending, - Err(_) => Poll::Ready(Err(())), + Err(payload) => { + // No registry or handler lock is held here. Opted-in modules + // close actual native authority before foreign payload Drop. + if let Some(observer) = &this.observer { + observer(); + } + drop(payload); + Poll::Ready(Err(())) + } } } } diff --git a/crates/tinybus/src/connection_tests.rs b/crates/tinybus/src/connection_tests.rs index 54ca5ed6..e8cabdc3 100644 --- a/crates/tinybus/src/connection_tests.rs +++ b/crates/tinybus/src/connection_tests.rs @@ -891,3 +891,60 @@ async fn weak_references_cannot_resurrect_dispatch_state_after_the_last_owner_dr assert!(format!("{weak:?}").contains("has_strong_owner: false")); Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn opted_in_method_fault_fences_before_primary_payload_drop() { + struct Payload { + fenced: Arc, + observed: Arc, + } + impl Drop for Payload { + fn drop(&mut self) { + self.observed + .store(self.fenced.load(Ordering::Acquire), Ordering::Release); + } + } + struct PanicService(Arc, Arc); + #[async_trait] + impl Interface for PanicService { + fn name(&self) -> InterfaceName { + "ai.tinyhumans.Test".parse().unwrap() + } + fn members(&self) -> Vec { + vec!["Panic".parse().unwrap()] + } + async fn call(&self, _: &MemberName, _: Value) -> Result { + std::panic::panic_any(Payload { + fenced: self.0.clone(), + observed: self.1.clone(), + }); + } + } + let fenced = Arc::new(AtomicBool::new(false)); + let observed = Arc::new(AtomicBool::new(false)); + let (client, service) = pair().await; + service + .serve_at(path(), PanicService(fenced.clone(), observed.clone())) + .await + .unwrap(); + let fence = fenced.clone(); + service.__set_catch_observer(Arc::new(move || { + fence.store(true, Ordering::Release); + })); + service.__set_panic_handler(Arc::new(|| Error::MethodFailed { + name: "ai.tinyhumans.tinybus.Error.ModulePanicked".into(), + message: "a module method panicked".into(), + })); + let error = client + .call_raw(call("Panic", Value::Null), DEFAULT_TIMEOUT) + .await + .unwrap_err(); + assert_eq!( + error.wire_name(), + "ai.tinyhumans.tinybus.Error.ModulePanicked" + ); + assert!( + observed.load(Ordering::Acquire), + "primary payload dropped before opted-in fault observer" + ); +} diff --git a/crates/tinybus/src/module/client_tests.rs b/crates/tinybus/src/module/client_tests.rs index c25b8946..dce7fc5b 100644 --- a/crates/tinybus/src/module/client_tests.rs +++ b/crates/tinybus/src/module/client_tests.rs @@ -171,3 +171,93 @@ async fn a_fixture_proxy_is_classified_and_reported_when_the_call_fails() { assert!(result.is_err()); assert_eq!(sink.0.lock().unwrap().len(), 1); } + +struct JsonFixture; +#[async_trait::async_trait] +impl crate::Interface for JsonFixture { + fn name(&self) -> crate::InterfaceName { + crate::InterfaceName::new(RECORD.bus_name).unwrap() + } + fn members(&self) -> Vec { + vec![crate::MemberName::new("Json").unwrap()] + } + async fn call( + &self, + _: &crate::MemberName, + _: serde_json::Value, + ) -> crate::Result { + Ok(serde_json::json!("{\"answer\":42}")) + } +} + +#[tokio::test] +async fn fixture_json_and_object_calls_preserve_deadlines_and_confidential_refusals() { + let (sink, resolver) = resolver(true); + let bus = crate::transport::memory::MemoryBus::new(); + crate::broker::Broker::new().spawn(bus.clone()); + let service = crate::Connection::connect(bus.connect().await.unwrap()) + .await + .unwrap(); + service.request_name(RECORD.bus_name).await.unwrap(); + for path in [RECORD.object_path, "/child"] { + service + .serve_at(crate::ObjectPath::new(path).unwrap(), JsonFixture) + .await + .unwrap(); + } + let caller = crate::Connection::connect(bus.connect().await.unwrap()) + .await + .unwrap(); + let proxy = caller + .proxy(RECORD.bus_name, RECORD.object_path, RECORD.bus_name) + .unwrap(); + let client = ModuleClient::new(resolver) + .with_fixture(proxy) + .with_timeout(Duration::from_secs(1)); + let reply: serde_json::Value = client.call_json(RECORD.id, "Json", ()).await.unwrap(); + assert_eq!(reply, serde_json::json!({"answer":42})); + let object = client.object_client(RECORD.id, "/child").await.unwrap(); + let reply: String = object.call("Json", ()).await.unwrap(); + assert_eq!(reply, "{\"answer\":42}"); + assert!( + client + .object_client(RECORD.id, "invalid path") + .await + .is_err() + ); + let reply: Result = object.call_confidential("Json", ()).await; + assert_eq!(reply, Err(ModuleCallError::TransportFailed)); + let reply: Result = client.call_confidential(RECORD.id, "Json", ()).await; + assert_eq!(reply, Err(ModuleCallError::TransportFailed)); + let reply: Result = + client.call_confidential_json(RECORD.id, "Json", ()).await; + assert_eq!(reply, Err(ModuleCallError::TransportFailed)); + assert!( + sink.0 + .lock() + .unwrap() + .iter() + .all(|(_, reason)| *reason == Reason::TransportFailed) + ); +} + +#[tokio::test] +async fn unavailable_loads_and_explicit_reports_use_only_registered_identity() { + let (sink, resolver) = resolver(true); + let client = ModuleClient::new(resolver); + let reply: Result = client.call(RECORD.id, "Json", ()).await; + assert_eq!(reply, Err(ModuleCallError::Unavailable)); + assert!(client.object_client(RECORD.id, "/child").await.is_err()); + client.report_incompatible_contract(RECORD.id); + client.report_unavailable(RECORD.id); + client.report_malformed_reply("unknown"); + client.report_incompatible_contract("unknown"); + client.report_unavailable("unknown"); + assert!( + sink.0 + .lock() + .unwrap() + .iter() + .all(|(module, _)| module == RECORD.id) + ); +} diff --git a/crates/tinybus/src/module/host.rs b/crates/tinybus/src/module/host.rs index 23887549..2569d402 100644 --- a/crates/tinybus/src/module/host.rs +++ b/crates/tinybus/src/module/host.rs @@ -1501,6 +1501,9 @@ impl ModuleControl for ModuleHostInner { .lock() .expect("router lock") .revoke_callbacks_for_peer(&module.unique_name); + // Revoke native admission immediately; provider drain owns the + // later shutdown handoff, so this must not submit a physical job. + module.transport.commit_terminal(false); module.transition_from.get_or_insert(old); module.lifecycle_busy = true; (module.transport.clone(), module.callback_state.clone()) diff --git a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs index fc6bf84e..ac770696 100644 --- a/crates/tinybus/src/module/host_callback_lifecycle_tests.rs +++ b/crates/tinybus/src/module/host_callback_lifecycle_tests.rs @@ -5,6 +5,7 @@ struct CommittingCallback { entered: tokio::sync::mpsc::Sender<()>, release: Arc, committed: Arc, + context: Arc>>, } #[async_trait::async_trait] impl crate::Interface for CommittingCallback { @@ -25,6 +26,7 @@ impl crate::Interface for CommittingCallback { ) -> Result { assert!(context.is_confidential()); assert!(context.authenticated_sender().is_some()); + *self.context.lock().unwrap() = Some(context.clone()); self.entered.send(()).await.unwrap(); let _release = self.release.acquire().await.unwrap(); // A native port may already have admitted its transaction. Revoking @@ -34,9 +36,63 @@ impl crate::Interface for CommittingCallback { } } +#[derive(Clone, Copy)] +enum DrainStop { + Complete, + HeldProbeObserver, + Timeout, + Cancel, + DropHost, +} + +struct CallbackRelease(Arc); +impl Drop for CallbackRelease { + fn drop(&mut self) { + self.0.add_permits(1); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires TINYBUS_TEST_CONTEXT_MODULE to point at the built cdylib"] async fn stopping_a_verified_module_drains_its_admitted_host_callback_before_native_shutdown() { + stop_after_admitted_callback(DrainStop::Complete).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTEXT_MODULE to point at the built cdylib"] +async fn host_stop_drain_timeout_retains_stop_intent_and_denies_new_admission() { + stop_after_admitted_callback(DrainStop::Timeout).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTEXT_MODULE to point at the built cdylib"] +async fn host_stop_drain_cancellation_retains_stop_intent_and_denies_new_admission() { + stop_after_admitted_callback(DrainStop::Cancel).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTEXT_MODULE to point at the built cdylib"] +async fn host_drop_after_cancelled_stop_waits_for_actual_provider_drain() { + stop_after_admitted_callback(DrainStop::DropHost).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires TINYBUS_TEST_CONTEXT_MODULE to point at the built cdylib"] +async fn native_stop_completion_does_not_wait_for_the_denial_probe_observer() { + stop_after_admitted_callback(DrainStop::HeldProbeObserver).await; +} + +async fn native_cleanup_finished(transport: &crate::module::transport::ModuleTransport) { + while transport.native_terminal_state_for_test() + & crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED + == 0 + || transport.lifecycle_is_owned_for_test() + { + tokio::task::yield_now().await; + } +} + +async fn stop_after_admitted_callback(mode: DrainStop) { let artifact = PathBuf::from(std::env::var_os("TINYBUS_TEST_CONTEXT_MODULE").unwrap()); let hash = crate::module::hash::file_hex(std::fs::File::open(&artifact).unwrap()).unwrap(); let (_dir, artifact) = staged_module(&artifact, &hash); @@ -52,6 +108,7 @@ async fn stopping_a_verified_module_drains_its_admitted_host_callback_before_nat let (entered, mut receiver) = tokio::sync::mpsc::channel(1); let release = Arc::new(tokio::sync::Semaphore::new(0)); let committed = Arc::new(AtomicUsize::new(0)); + let context = Arc::new(StdMutex::new(None)); binding .serve_at( "/callback".parse().unwrap(), @@ -59,6 +116,7 @@ async fn stopping_a_verified_module_drains_its_admitted_host_callback_before_nat entered, release: release.clone(), committed: committed.clone(), + context: context.clone(), }, ) .await @@ -75,8 +133,10 @@ async fn stopping_a_verified_module_drains_its_admitted_host_callback_before_nat ) .unwrap() .with_timeout(Duration::from_secs(2)); + let release_on_failure = CallbackRelease(release.clone()); + let first_proxy = proxy.clone(); let request = tokio::spawn(async move { - proxy + first_proxy .call_confidential::("ConfidentialCallback", ()) .await }); @@ -84,35 +144,197 @@ async fn stopping_a_verified_module_drains_its_admitted_host_callback_before_nat .await .unwrap() .unwrap(); - assert_eq!(binding.guard().running(), 1); let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); - let stopping = host.inner.stop(&loaded.name, Duration::from_secs(2)); - tokio::pin!(stopping); - let completed_early = std::future::poll_fn(|context| { - std::task::Poll::Ready(std::future::Future::poll(stopping.as_mut(), context).is_ready()) + let deadline = if matches!(mode, DrainStop::Timeout) { + Duration::ZERO + } else { + Duration::from_secs(2) + }; + let stop_started = std::time::Instant::now(); + let mut stopping = Some(Box::pin(host.inner.stop(&loaded.name, deadline))); + let mut early = std::future::poll_fn(|cx| { + std::task::Poll::Ready( + match std::future::Future::poll(stopping.as_mut().unwrap().as_mut(), cx) { + std::task::Poll::Ready(result) => Some(result), + std::task::Poll::Pending => None, + }, + ) }) .await; + if matches!(mode, DrainStop::Timeout) && early.is_none() { + early = Some(stopping.take().unwrap().await); + } + if !matches!(mode, DrainStop::Complete | DrainStop::HeldProbeObserver) { + drop(stopping.take()); + } + let terminal_while_held = transport.native_terminal_state_for_test(); let shutdown_started_early = transport.stop_task_spawned(); + let physically_stopped_early = + terminal_while_held & crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED != 0; let still_running = binding.guard().running(); - // Always release the actual worker before asserting the intended RED. - release.add_permits(1); - tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()) - .await - .unwrap(); - if !completed_early { - stopping.await.unwrap(); + let authority_denied = !binding + .guard() + .authorizes(context.lock().unwrap().as_ref().unwrap()); + let owner_retained = host + .inner + .loaded + .lock() + .unwrap() + .iter() + .any(|module| Arc::ptr_eq(&module.transport, &transport)); + let enable_denied = host.inner.enable(&loaded.name, true).is_err(); + // This observation is independently owned: its scheduling must not consume + // the stop's semantic deadline while the fixture deliberately holds work. + let probe_entered = Arc::new(tokio::sync::Semaphore::new(0)); + let probe_release = Arc::new(tokio::sync::Semaphore::new(0)); + let probe_release_on_failure = CallbackRelease(probe_release.clone()); + let held_probe = matches!(mode, DrainStop::HeldProbeObserver); + let entered = probe_entered.clone(); + let release_probe = probe_release.clone(); + let denial = tokio::spawn(async move { + let denied = proxy + .call_confidential::("ConfidentialCallback", ()) + .await + .is_err(); + if held_probe { + entered.add_permits(1); + let _permit = release_probe.acquire().await; + } + denied + }); + let observer = if held_probe { + let transport = transport.clone(); + Some(tokio::spawn(async move { + // Always release the observer, including a failed watchdog. + let _release = CallbackRelease(probe_release); + let entered = + tokio::time::timeout(Duration::from_secs(5), probe_entered.acquire()).await; + let completed = + tokio::time::timeout(Duration::from_secs(1), native_cleanup_finished(&transport)) + .await; + eprintln!( + "denial observer: entered={} actual_cleanup_while_held={}", + entered.is_ok(), + completed.is_ok() + ); + (entered.is_ok(), completed.is_ok()) + })) + } else { + None + }; + if matches!(mode, DrainStop::DropHost) { + let account = host.inner.loaded.lock().unwrap()[0] + .callback_state + .clone() + .unwrap(); + let host_owner = Arc::downgrade(&host.inner); + drop(stopping); + drop(host); + let entered = tokio::time::timeout( + Duration::from_secs(2), + account.physical_wait_started.notified(), + ) + .await; + let terminal_after_drop = transport.native_terminal_state_for_test(); + let still_counted = binding.guard().running(); + let lifecycle_free = !transport.lifecycle_is_owned_for_test(); + drop(release_on_failure); + tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()) + .await + .unwrap(); + let completed = tokio::time::timeout(Duration::from_secs(2), async { + while transport.native_terminal_state_for_test() + & crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED + == 0 + { + tokio::task::yield_now().await; + } + }) + .await; + let _outcome = request.await; + let new_call_denied = denial.await; + drop(probe_release_on_failure); + assert!(new_call_denied.is_ok_and(|denied| denied)); + assert!(completed.is_ok()); + assert_eq!( + terminal_after_drop, + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT, + "opaque shutdown returned while provider work was still admitted" + ); + assert!(entered.is_ok()); + assert!(host_owner.upgrade().is_none()); + assert_eq!(still_counted, 1); + assert!(lifecycle_free && authority_denied); + assert_eq!(committed.load(Ordering::Acquire), 1); + assert!(!transport.is_faulted()); + return; } - let _outcome = request.await.unwrap(); + // Release and consume real callback/physical cleanup before all assertions. + drop(release_on_failure); + let stopped = if let Some(stopping) = stopping { + if let Some(result) = early.take() { + result + } else { + stopping.await + } + } else { + host.inner.stop(&loaded.name, Duration::from_secs(2)).await + }; + let stop_elapsed = stop_started.elapsed(); + let stop_code = stopped + .as_ref() + .err() + .map(|error| error.wire_name().to_owned()); + let drained = tokio::time::timeout(Duration::from_secs(2), binding.guard().drained()).await; + // An error is not evidence of reclamation. Complete the retained ordinary + // terminal job and independently observe native shutdown and lock release. + transport.request_terminal(false); + let cleanup = + tokio::time::timeout(Duration::from_secs(2), native_cleanup_finished(&transport)).await; + let request_outcome = request.await; + // Observe the unrelated RPC only after the original stop and real cleanup. + // Its denial is measured at the task's actual execution phase. + let new_call_denied = denial.await; + drop(probe_release_on_failure); + let observer_outcome = match observer { + Some(observer) => Some(observer.await), + None => None, + }; + eprintln!( + "stop fixture: code={stop_code:?} elapsed={stop_elapsed:?} cleanup={}", + cleanup.is_ok() + ); + assert!(cleanup.is_ok(), "actual native cleanup did not finish"); + assert!(request_outcome.is_ok()); + assert!(drained.is_ok()); + if let Some(outcome) = observer_outcome { + assert!( + outcome.is_ok_and(|(entered, completed)| entered && completed), + "actual native cleanup waited for the unrelated denial observer" + ); + } + assert!( + stopped.is_ok(), + "stop code={stop_code:?}, elapsed={stop_elapsed:?}" + ); assert_eq!(committed.load(Ordering::Acquire), 1); assert_eq!(still_running, 1); assert!( - !completed_early, - "stop completed before admitted callback commit" + authority_denied + && owner_retained + && enable_denied + && new_call_denied.is_ok_and(|denied| denied) ); - assert!( - !shutdown_started_early, - "native shutdown started before callback commit" + assert!(!shutdown_started_early && !physically_stopped_early); + assert_eq!( + terminal_while_held, + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT, + "native authority must close before admitted provider work drains" ); + assert!(!transport.is_faulted()); + if matches!(mode, DrainStop::Timeout) { + assert!(early.is_some_and(|result| result.is_err())); + } } #[tokio::test] diff --git a/crates/tinybus/src/module/host_callbacks.rs b/crates/tinybus/src/module/host_callbacks.rs index 5e184e2b..56d636c1 100644 --- a/crates/tinybus/src/module/host_callbacks.rs +++ b/crates/tinybus/src/module/host_callbacks.rs @@ -47,9 +47,30 @@ pub(crate) struct CallbackState { module_transport: Mutex>>, lifetime: Mutex, drained: Notify, + physical_drained: std::sync::Condvar, + #[cfg(test)] + pub(crate) physical_wait_started: Notify, } impl CallbackState { + /// Physical cleanup waits offruntime, without a watchdog or native exclusion. + pub(crate) fn wait_drained(&self) { + let mut state = self + .lifetime + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.active = false; + state.revoked = true; + #[cfg(test)] + self.physical_wait_started.notify_one(); + while state.running != 0 { + state = self + .physical_drained + .wait(state) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + pub(crate) async fn drain(self: &Arc, deadline: Duration) -> Result<()> { tokio::time::timeout(deadline, HostCallbackGuard(self.clone()).drained()) .await @@ -210,6 +231,8 @@ pub struct HostCallbackBinding { fixture_controller: Mutex>>, #[cfg(test)] before_load_decision: Mutex, tokio::sync::oneshot::Receiver<()>)>>, + #[cfg(test)] + after_load_termination: Mutex>>, } /// Outgoing operations on the binding's private connection. @@ -435,10 +458,15 @@ struct GuardedInterface { struct Running(Arc); impl Drop for Running { fn drop(&mut self) { - let mut state = self.0.lifetime.lock().expect("callback lifetime lock"); + let mut state = self + .0 + .lifetime + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); state.running -= 1; if state.running == 0 { self.0.drained.notify_waiters(); + self.0.physical_drained.notify_all(); } } } @@ -476,9 +504,14 @@ impl Interface for GuardedInterface { state.running += 1; Running(self.state.clone()) }; - self.interface - .call_with_context(member, args, context) - .await + // The later-created user future is destroyed before Running on both + // completion and cancellation; no response queue owns this accounting. + let result = { + let user = self.interface.call_with_context(member, args, context); + user.await + }; + drop(_running); + result } } @@ -570,6 +603,8 @@ impl ModuleHost { fixture_controller: Mutex::new(None), #[cfg(test)] before_load_decision: Mutex::new(None), + #[cfg(test)] + after_load_termination: Mutex::new(None), state: Arc::new(CallbackState { destination, endpoints, @@ -585,6 +620,9 @@ impl ModuleHost { registering: Vec::new(), }), drained: Notify::new(), + physical_drained: std::sync::Condvar::new(), + #[cfg(test)] + physical_wait_started: Notify::new(), }), }) } @@ -667,6 +705,7 @@ impl ModuleHost { let fingerprint = callback_config_fingerprint(&bytes); drop(config); let (transport, host) = ModuleTransport::new(admitted.name.clone(), bytes); + transport.bind_host_callbacks(binding.state.clone())?; #[cfg(test)] { // Transfer the fixture config borrow before C can enter, so @@ -775,6 +814,10 @@ impl ModuleHost { let fault = transport.commit_admission_termination(cause); binding.revoke(); transport.request_terminal(fault); + #[cfg(test)] + if let Some(entered) = binding.after_load_termination.lock().unwrap().take() { + entered.notify_one(); + } let _ = transport.close().await; self.inner .loaded diff --git a/crates/tinybus/src/module/host_callbacks_tests.rs b/crates/tinybus/src/module/host_callbacks_tests.rs index 1ebf63bd..e753b508 100644 --- a/crates/tinybus/src/module/host_callbacks_tests.rs +++ b/crates/tinybus/src/module/host_callbacks_tests.rs @@ -33,6 +33,8 @@ fn state(source: BusName, destination: BusName) -> Arc { registering: vec![], }), drained: Notify::new(), + physical_drained: std::sync::Condvar::new(), + physical_wait_started: Notify::new(), }) } @@ -633,6 +635,186 @@ fn configuration_fingerprints_preserve_json_equality_without_normalizing_strings ); } +// Private fixture-only release ownership. Drop never joins or asserts. +struct ControlledRelease { + barrier: Arc, + finish: Option>, + resume: Option>, + transport: Option>, +} +impl ControlledRelease { + fn new(barrier: Arc) -> Self { + Self { + barrier, + finish: None, + resume: None, + transport: None, + } + } +} +impl ControlledRelease { + fn release(&mut self) { + self.barrier + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .1 = true; + self.barrier.changed.notify_all(); + if let Some(resume) = self.resume.take() { + let _ = resume.send(()); + } + if let Some(finish) = self.finish.take() { + let _ = finish.send(()); + } + } +} +impl Drop for ControlledRelease { + fn drop(&mut self) { + self.release(); + // Commit and schedule synchronously, before any Tokio cleanup observer + // could be canceled. request_terminal anchors PhysicalJob in the actual + // context and submits its OS thread; neither depends on this runtime. + // Only fixture-local locks can be poisoned by test assertions; release + // above recovers those. No SDK internal lock is held by this guard. + if let Some(transport) = self.transport.take() { + transport.request_terminal(false); + } + } +} + +#[test] +fn controlled_release_wakes_entry_and_releases_channels_on_early_unwind() { + use super::super::super::transport::tests::NativeBarrier; + let barrier = Arc::new(NativeBarrier { + state: Mutex::new((None, false, false)), + changed: std::sync::Condvar::new(), + entered: Notify::new(), + joined: Notify::new(), + finish_gate: Mutex::new(None), + }); + let (finish, receive) = std::sync::mpsc::channel(); + let (resume, decision) = tokio::sync::oneshot::channel(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut cleanup = ControlledRelease::new(barrier.clone()); + cleanup.finish = Some(finish); + cleanup.resume = Some(resume); + panic!("fixture assertion before controlled release"); + })); + let finish_result = receive.try_recv(); + let mut decision = decision; + let decision_result = decision.try_recv(); + assert!(result.is_err()); + assert!( + barrier.state.lock().unwrap().1, + "early unwind must wake native entry" + ); + assert!( + finish_result.is_ok(), + "finish must be released, not disconnected" + ); + assert!( + decision_result.is_ok(), + "decision must be released, not disconnected" + ); +} + +#[tokio::test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +async fn actual_native_early_unwind_releases_entry_and_reclaims_after_callback_return() { + use super::super::super::transport::tests::native_fixture; + let (transport, host, init, barrier) = native_fixture(); + let weak = Arc::downgrade(&barrier); + let mut cleanup = ControlledRelease::new(barrier.clone()); + cleanup.transport = Some(transport.clone()); + let (finish, receive) = std::sync::mpsc::channel(); + cleanup.finish = Some(finish); + *barrier.finish_gate.lock().unwrap() = Some(receive); + transport.defer_initialize(init, host); + let initializing = tokio::spawn({ + let transport = transport.clone(); + async move { transport.ensure_initialized().await } + }); + let entered = tokio::time::timeout(Duration::from_secs(2), barrier.entered.notified()).await; + // Deliberate ordinary Rust unwind exercises the guard while actual C entry + // is blocked on its condition variable. No panic crosses a C boundary. + let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || { + let _cleanup = cleanup; + panic!("early controlled fixture failure"); + })); + let initialized = initializing.await; + let stopped = transport.clone().stop(Duration::from_secs(2)).await; + let callback_entered_after_worker_join = barrier.state.lock().unwrap().2; + drop(barrier); + drop(transport); + let reclaimed = tokio::time::timeout(Duration::from_secs(2), async { + while weak.strong_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await; + // Stop completion, rather than callback entry or abort, proves actual return. + assert!(entered.is_ok()); + assert!(unwind.is_err()); + assert!(initialized.is_ok()); + assert!(matches!( + stopped, + Ok(_) + | Err(super::super::super::transport::StopError::NotStarted( + Error::ConnectionClosed + )) + )); + assert!(callback_entered_after_worker_join); + assert!(reclaimed.is_ok()); +} + +#[test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +fn actual_native_release_owner_survives_canceled_waiter_and_runtime_drop() { + use super::super::super::transport::tests::native_fixture; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + let (transport, host, init, barrier) = native_fixture(); + barrier.state.lock().unwrap().1 = true; + transport.defer_initialize(init, host); + let initialized = runtime.block_on(transport.ensure_initialized()); + let weak = Arc::downgrade(&barrier); + let (finish, receive) = std::sync::mpsc::channel(); + *barrier.finish_gate.lock().unwrap() = Some(receive); + let mut cleanup = ControlledRelease::new(barrier.clone()); + cleanup.transport = Some(transport.clone()); + // This test's external finish control stays held to inspect physical owner + // retention. It is released before any assertions, including failure paths. + drop(cleanup); + let waiter = runtime.spawn({ + let transport = transport.clone(); + async move { transport.stop(Duration::from_secs(2)).await } + }); + waiter.abort(); // intentionally cancel without relying on a first poll + let canceled = runtime.block_on(waiter); + let joined_worker = runtime.block_on(async { + tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()).await + }); + drop(runtime); + drop(barrier); + drop(transport); + let retained_during_callback = weak.strong_count() != 0; + let released = finish.send(()); + // The physical thread is independent of the dropped Tokio runtime. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while weak.strong_count() != 0 && std::time::Instant::now() < deadline { + std::thread::yield_now(); + } + let reclaimed_after_return = weak.strong_count() == 0; + assert!(initialized.is_ok()); + assert!(canceled.unwrap_err().is_cancelled()); + assert!(joined_worker.is_ok()); + assert!(retained_during_callback); + assert!(released.is_ok()); + assert!(reclaimed_after_return); +} + // Manually polling the public future lets ordinary shutdown commit while the // loader is suspended; no scheduler race decides which cancellation is tested. async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: bool) { @@ -661,46 +843,49 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: ) .await .unwrap(); + let mut cleanup = ControlledRelease::new(barrier.clone()); let mut load = Some(Box::pin( host.load_file_with_host_callbacks(&artifact, config, &binding), )); - std::future::poll_fn(|cx| { - assert!( - load.as_mut().unwrap().as_mut().poll(cx).is_pending(), - "loader awaits real C init" - ); - std::task::Poll::Ready(()) - }) - .await; - tokio::time::timeout(Duration::from_secs(2), barrier.entered.notified()) - .await - .unwrap(); + let first = + std::future::poll_fn(|cx| std::task::Poll::Ready(load.as_mut().unwrap().as_mut().poll(cx))) + .await; + if let std::task::Poll::Ready(completion) = first { + cleanup.release(); + drop(completion); + drop(load.take()); // already completed, no pending owner is discarded + panic!("native fixture completed before C entry"); + } + let entered = tokio::time::timeout(Duration::from_secs(2), barrier.entered.notified()).await; + if entered.is_err() { + cleanup.release(); + let completion = load.take().unwrap().await; + drop(completion); + panic!("native fixture entry watchdog expired; loader completion consumed"); + } let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + cleanup.transport = Some(transport.clone()); let view = barrier.state.lock().unwrap().0.unwrap(); barrier.state.lock().unwrap().1 = true; barrier.changed.notify_all(); - transport.ensure_initialized().await.unwrap(); + let initialized = transport.ensure_initialized().await; // Advance the public future to its real withheld-Ready observation. - std::future::poll_fn(|cx| { - assert!( - load.as_mut().unwrap().as_mut().poll(cx).is_pending(), - "fixture withholds Ready" - ); - std::task::Poll::Ready(()) - }) - .await; + let withheld = + std::future::poll_fn(|cx| std::task::Poll::Ready(load.as_mut().unwrap().as_mut().poll(cx))) + .await; + let was_withheld = withheld.is_pending(); let (finish, receive) = std::sync::mpsc::channel(); *barrier.finish_gate.lock().unwrap() = Some(receive); + cleanup.finish = Some(finish); let stopping = transport.clone(); let stop = tokio::spawn(async move { stopping.stop(Duration::from_secs(2)).await }); tokio::time::timeout(Duration::from_secs(2), barrier.joined.notified()) .await .unwrap(); - assert_eq!( - unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, - 2 - ); - let load_result = if reject_gate { + let stopped_flags = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; + let load_result = if let std::task::Poll::Ready(result) = withheld { + Some(result) + } else if reject_gate { transport.release_initialization(false); drop(load.take()); None @@ -717,14 +902,24 @@ async fn native_callback_load_after_ordinary_stop(drop_load: bool, reject_gate: }; let held_flags = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; let held_fault = transport.is_faulted(); - finish.send(()).unwrap(); // Always release opaque C before RED assertions. - assert!(stop.await.unwrap().is_ok()); - if let Some(result) = load_result { - assert!(result.is_err()); + cleanup.release(); + let stopped = stop.await; + let loaded = if let Some(result) = load_result { drop(load.take()); + Some(result) } else if let Some(load) = load { - assert!(load.await.is_err()); + Some(load.await) + } else { + None + }; + cleanup.transport = None; + assert!(initialized.is_ok()); + assert!(was_withheld, "fixture withholds Ready"); + assert!(stopped.unwrap().is_ok()); + if let Some(result) = loaded { + assert!(result.is_err()); } + assert_eq!(stopped_flags, 2); assert_eq!(held_flags, 2, "ordinary cancellation cannot invent Fault"); assert!(!held_fault); assert_eq!( @@ -782,8 +977,12 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { let binding = Arc::new(served_native_binding(&host).await); *binding.fixture_controller.lock().unwrap() = Some(barrier.clone()); let decision = Arc::new(Notify::new()); + let termination = Arc::new(Notify::new()); + *binding.after_load_termination.lock().unwrap() = Some(termination.clone()); let (resume, release) = tokio::sync::oneshot::channel(); *binding.before_load_decision.lock().unwrap() = Some((decision.clone(), release)); + let mut cleanup = ControlledRelease::new(barrier.clone()); + cleanup.resume = Some(resume); let loading_host = host.clone(); let loading_binding = binding.clone(); let load = tokio::spawn(async move { @@ -797,6 +996,7 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { tokio::task::yield_now().await; } let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + cleanup.transport = Some(transport.clone()); let view = barrier.state.lock().unwrap().0.unwrap(); barrier.state.lock().unwrap().1 = true; barrier.changed.notify_all(); @@ -808,17 +1008,17 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { // init_started precedes actual publication; the observer result is the proof. let initialized = transport.ensure_initialized(); tokio::pin!(initialized); - std::future::poll_fn( - |cx| match std::future::Future::poll(initialized.as_mut(), cx) { - std::task::Poll::Ready(result) => std::task::Poll::Ready(result), - std::task::Poll::Pending => { - cx.waker().wake_by_ref(); - std::task::Poll::Pending - } - }, - ) - .await - .unwrap(); + let initialization = + std::future::poll_fn( + |cx| match std::future::Future::poll(initialized.as_mut(), cx) { + std::task::Poll::Ready(result) => std::task::Poll::Ready(result), + std::task::Poll::Pending => { + cx.waker().wake_by_ref(); + std::task::Poll::Pending + } + }, + ) + .await; if !ready { // Real wait_ready deadline, not a fake readiness flag. Let the public // future install its timer before advancing past the actual bound. @@ -826,41 +1026,50 @@ async fn native_callback_decision_race(ready: bool, native_fault: bool) { tokio::time::advance(Duration::from_secs(6)).await; } decision.notified().await; - assert_eq!(transport.is_ready(), ready); + let observed_ready = transport.is_ready(); let (finish, receive) = std::sync::mpsc::channel(); *barrier.finish_gate.lock().unwrap() = Some(receive); + cleanup.finish = Some(finish); let stopping = transport.clone(); let stop = tokio::spawn(async move { stopping.stop(Duration::from_secs(2)).await }); while !barrier.state.lock().unwrap().2 { tokio::task::yield_now().await; } - assert_eq!( - unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, - 2 - ); + let stopped_flags = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; + let mut fault_flags = None; if native_fault { unsafe { (view.fault)(view.host_ctx, std::ptr::null(), 0); } - assert_eq!( - unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, - 6 - ); - } - resume.send(()).unwrap(); - // Rollback revokes before awaiting physical close or returning failure. - while !binding.state.lifetime.lock().unwrap().revoked && !load.is_finished() { - tokio::task::yield_now().await; + fault_flags = Some(unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }); } + let resumed = cleanup.resume.take().unwrap().send(()); + // The physical worker can revoke the account independently. Observe the + // loader's actual failure commit instead of treating revocation as proof. + let terminated = if !ready { + Some(tokio::time::timeout(Duration::from_secs(2), termination.notified()).await) + } else { + None + }; let held = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; - finish.send(()).unwrap(); + cleanup.release(); // Keep virtual time runnable through the actual opaque return. while !stop.is_finished() { tokio::task::yield_now().await; } - let stopped = stop.await.unwrap(); - assert!(stopped.is_ok(), "actual stop: {stopped:?}"); - assert!(load.await.unwrap().is_err()); + let stopped = stop.await; + let loaded = load.await; + cleanup.transport = None; + assert!(initialization.is_ok()); + assert!(resumed.is_ok()); + assert!(terminated.is_none_or(|result| result.is_ok())); + assert!(stopped.unwrap().is_ok()); + assert!(loaded.unwrap().is_err()); + assert_eq!(observed_ready, ready); + assert_eq!(stopped_flags, 2); + if let Some(flags) = fault_flags { + assert_eq!(flags, 6); + } assert_eq!(held, if ready && !native_fault { 2 } else { 6 }); assert_eq!( unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }, @@ -885,6 +1094,140 @@ async fn actual_native_fault_at_ready_commit_dominates_ordinary_stop() { native_callback_decision_race(true, true).await; } +#[derive(Debug, PartialEq, Eq)] +enum AttachOutcome { + Attached, + LoadSucceeded, + LoadRefused, + LoadConnectionClosed, + LoadFailed, + LoadPanicked, + LoadCanceled, + WatchdogExpired, +} + +async fn await_native_attach( + attached: &Notify, + load: &mut tokio::task::JoinHandle>, + completed: &mut Option, tokio::task::JoinError>>, +) -> AttachOutcome { + // A fixture precondition watchdog, not a product security/readiness TTL. + // Abort requests cooperative cancellation; it cannot interrupt synchronous + // admission or prove physical native quiescence. The caller owns cleanup. + tokio::select! { + biased; + _ = attached.notified() => AttachOutcome::Attached, + result = &mut *load => { + let category = match &result { + Ok(Ok(_)) => AttachOutcome::LoadSucceeded, + Ok(Err(Error::ModuleRefused { .. })) => AttachOutcome::LoadRefused, + Ok(Err(Error::ConnectionClosed)) => AttachOutcome::LoadConnectionClosed, + Ok(Err(_)) => AttachOutcome::LoadFailed, + Err(error) if error.is_cancelled() => AttachOutcome::LoadCanceled, + Err(_) => AttachOutcome::LoadPanicked, + }; + *completed = Some(result); + category + }, + _ = tokio::time::sleep(Duration::from_secs(60)) => { + load.abort(); + AttachOutcome::WatchdogExpired + } + } +} + +#[tokio::test(start_paused = true)] +async fn setup_attachment_wait_allows_preload_to_outlive_product_deadlines() { + let attached = Arc::new(Notify::new()); + let observed = attached.clone(); + let (release, preload) = tokio::sync::oneshot::channel::<()>(); + let mut load = tokio::spawn(async move { + let _ = preload.await; + std::future::pending::>().await + }); + let wait = tokio::spawn(async move { + let mut completed = None; + let outcome = await_native_attach(&observed, &mut load, &mut completed).await; + load.abort(); + let result = match completed { + Some(result) => result, + None => load.await, + }; + (outcome, result) + }); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(3)).await; + tokio::task::yield_now().await; + let waiting = !wait.is_finished(); + let released = release.send(()); + attached.notify_one(); + let joined = wait.await; + assert!(waiting, "valid preload was mistaken for setup failure"); + assert!(released.is_ok()); + let (outcome, result) = joined.unwrap(); + assert_eq!(outcome, AttachOutcome::Attached); + assert!(result.unwrap_err().is_cancelled()); +} + +#[tokio::test] +async fn setup_attachment_wait_reports_an_early_load_failure() { + let attached = Notify::new(); + let mut load = tokio::spawn(async { Err::<(), _>(Error::ConnectionClosed) }); + let mut completed = None; + let outcome = await_native_attach(&attached, &mut load, &mut completed).await; + let result = completed.unwrap(); + assert_eq!(outcome, AttachOutcome::LoadConnectionClosed); + assert!(matches!(result, Ok(Err(Error::ConnectionClosed)))); +} + +#[tokio::test] +async fn setup_attachment_wait_preserves_actual_admission_refusal_before_c_entry() { + let host = Arc::new(ModuleHost::new(Broker::new())); + let binding = Arc::new(served_native_binding(&host).await); + let attached = Notify::new(); + let directory = tempfile::tempdir().unwrap(); + let missing = directory.path().join("absent-module.so"); + let mut load = tokio::spawn({ + let host = host.clone(); + let binding = binding.clone(); + async move { + host.load_file_with_host_callbacks(missing, serde_json::json!({}), &binding) + .await + } + }); + let mut completed = None; + let outcome = await_native_attach(&attached, &mut load, &mut completed).await; + let completion = completed.unwrap(); + assert_eq!(outcome, AttachOutcome::LoadRefused); + assert!(matches!(completion, Ok(Err(Error::ModuleRefused { .. })))); + assert!( + host.list().is_empty(), + "refusal cannot attach or publish Ready" + ); +} + +#[tokio::test(start_paused = true)] +async fn setup_attachment_watchdog_cancels_a_stuck_preload() { + let attached = Notify::new(); + let mut load = tokio::spawn(std::future::pending::>()); + let wait = tokio::spawn(async move { + let mut completed = None; + let outcome = await_native_attach(&attached, &mut load, &mut completed).await; + // Actual task completion is consumed. This synthetic task is cooperative; + // this is not evidence of synchronous/native cancellation or C return. + let result = match completed { + Some(result) => result, + None => load.await, + }; + (outcome, result) + }); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(61)).await; + let (outcome, result) = wait.await.unwrap(); + assert_eq!(outcome, AttachOutcome::WatchdogExpired); + assert!(result.unwrap_err().is_cancelled()); +} + async fn native_callback_setup_failure_race(panic_setup: bool) { use super::super::super::transport::tests::{native_fixture, native_fixture_json}; let (unused, _, _, barrier) = native_fixture(); @@ -900,9 +1243,11 @@ async fn native_callback_setup_failure_race(panic_setup: bool) { let attached = Arc::new(Notify::new()); let seen = attached.clone(); let (resume, release) = std::sync::mpsc::channel(); + let mut cleanup = ControlledRelease::new(barrier.clone()); + cleanup.finish = Some(resume); *binding.after_attach.lock().unwrap() = Some(Box::new(move || { seen.notify_one(); - release.recv().unwrap(); + let _ = release.recv(); assert!( !panic_setup, "actual Rust unwind during host callback setup" @@ -910,28 +1255,50 @@ async fn native_callback_setup_failure_race(panic_setup: bool) { })); let loading_host = host.clone(); let loading_binding = binding.clone(); - let load = tokio::spawn(async move { + let mut load = tokio::spawn(async move { loading_host .load_file_with_host_callbacks(artifact, config, &loading_binding) .await }); - // Admission hashes the real artifact twice before this hook; instrumented - // CI builds can spend several seconds there. This is a fixture setup bound, - // independent of the lifecycle deadline and the race assertions below. - tokio::time::timeout(Duration::from_secs(30), attached.notified()) - .await - .unwrap(); + let mut completed = None; + let attachment = await_native_attach(&attached, &mut load, &mut completed).await; + if attachment != AttachOutcome::Attached { + cleanup.release(); + // Report the original finite cause before cleanup: synchronous admission + // cannot be interrupted by abort. Do not turn it into a readiness failure. + eprintln!("native fixture attachment: {attachment:?}"); + if let Some(completion) = completed { + drop(completion); + } else if tokio::time::timeout(Duration::from_secs(2), &mut load) + .await + .is_err() + { + // Pending ownership is explicit, not a quiescence claim. The task + // consumes eventual completion and keeps the real staging, binding, + // host and controller alive; transport's physical anchor also remains. + tokio::spawn(async move { + let _owners = (_dir, host, binding, barrier); + let _ = load.await; + }); + panic!("native fixture attachment: {attachment:?}; cleanup pending"); + } + panic!("native fixture attachment: {attachment:?}; loader completion consumed"); + } let transport = host.inner.loaded.lock().unwrap()[0].transport.clone(); + cleanup.transport = Some(transport.clone()); // Actual ordinary lifecycle ownership, before the callback initialization // gate opens. This instance proves C never entered, rather than any join. - let _ = transport.clone().stop(Duration::from_secs(2)).await; - assert!(!transport.is_faulted()); + let stopped = transport.clone().stop(Duration::from_secs(2)).await; + let held_fault = transport.is_faulted(); if !panic_setup { // Public revocation makes real setup return its concurrently-used error. binding.revoke(); } - resume.send(()).unwrap(); + cleanup.release(); let result = load.await; + cleanup.transport = None; + assert!(stopped.is_ok()); + assert!(!held_fault); if panic_setup { assert!(result.unwrap_err().is_panic()); } else { @@ -957,3 +1324,179 @@ async fn actual_callback_setup_error_still_faults_after_ordinary_stop() { async fn actual_callback_setup_unwind_still_faults_after_ordinary_stop() { native_callback_setup_failure_race(true).await; } + +#[test] +fn a_transport_rejects_replacement_of_its_exact_callback_account() { + let first = state(":1.1".parse().unwrap(), ":1.2".parse().unwrap()); + let second = state(":1.3".parse().unwrap(), ":1.4".parse().unwrap()); + let original = Arc::downgrade(&first); + let replacement = Arc::downgrade(&second); + let (transport, _) = ModuleTransport::new("one-account".into(), vec![]); + transport.bind_host_callbacks(first.clone()).unwrap(); + let rejected = transport.bind_host_callbacks(second.clone()); + drop(first); + drop(second); + assert!(rejected.is_err()); + assert!(original.upgrade().is_some()); + assert!(replacement.upgrade().is_none()); +} + +struct DestructionObserved { + guard: HostCallbackGuard, + observed: Arc, +} +impl Drop for DestructionObserved { + fn drop(&mut self) { + self.observed.store(self.guard.running(), Ordering::Release); + } +} +struct PendingUntilDestroyed { + complete: bool, + entered: Arc, + guard: HostCallbackGuard, + observed: Arc, +} +#[async_trait] +impl Interface for PendingUntilDestroyed { + fn name(&self) -> InterfaceName { + endpoint().interface + } + fn members(&self) -> Vec { + endpoint().members + } + async fn call(&self, _: &MemberName, _: serde_json::Value) -> Result { + Err(Error::failed("context required")) + } + async fn call_with_context( + &self, + _: &MemberName, + _: serde_json::Value, + _: &CallContext, + ) -> Result { + let _destruction = DestructionObserved { + guard: self.guard.clone(), + observed: self.observed.clone(), + }; + self.entered.notify_one(); + if self.complete { + Ok(serde_json::Value::Null) + } else { + std::future::pending().await + } + } +} + +#[test] +#[ignore = "requires TINYBUS_TEST_CONTROLLED_INIT_MODULE built cdylib"] +fn runtime_drop_destroys_provider_future_before_offruntime_native_cleanup() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let account = state(":1.1".parse().unwrap(), ":1.2".parse().unwrap()); + let observed = Arc::new(AtomicUsize::new(0)); + let entered = Arc::new(Notify::new()); + let (transport, view, barrier, user_entered, wait_entered, replacement_rejected) = runtime + .block_on(async { + let (transport, view, init, barrier) = + crate::module::transport::tests::native_fixture(); + barrier.state.lock().unwrap().1 = true; + transport.bind_host_callbacks(account.clone()).unwrap(); + transport.defer_initialize(init, view); + transport.ensure_initialized().await.unwrap(); + barrier.entered.notified().await; + let guarded = GuardedInterface { + interface: PendingUntilDestroyed { + complete: false, + entered: entered.clone(), + guard: HostCallbackGuard(account.clone()), + observed: observed.clone(), + }, + endpoint: endpoint(), + state: account.clone(), + }; + let context = CallContext::brokered(account.module_peer().as_ref(), true); + tokio::spawn(async move { + guarded + .call_with_context(&endpoint().members[0], serde_json::json!([]), &context) + .await + }); + let user_entered = tokio::time::timeout(Duration::from_secs(2), entered.notified()) + .await + .is_ok(); + let replacement = state(":1.3".parse().unwrap(), ":1.4".parse().unwrap()); + let replacement_rejected = transport.bind_host_callbacks(replacement).is_err(); + transport.request_terminal(false); + let wait_entered = tokio::time::timeout( + Duration::from_secs(2), + account.physical_wait_started.notified(), + ) + .await + .is_ok(); + ( + transport, + view, + barrier, + user_entered, + wait_entered, + replacement_rejected, + ) + }); + let count_before_drop = HostCallbackGuard(account.clone()).running(); + let terminal_before_drop = unsafe { view.native_terminal_state.unwrap()(view.host_ctx) }; + let lifecycle_free = !transport.lifecycle_is_owned_for_test(); + // This drops the actual GuardedInterface/user future, not a caller observer. + drop(runtime); + let deadline = std::time::Instant::now() + Duration::from_secs(2); + let completed = loop { + if unsafe { view.native_terminal_state.unwrap()(view.host_ctx) } + & crate::module::abi::TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED + != 0 + { + break true; + } + if std::time::Instant::now() >= deadline { + break false; + } + std::thread::yield_now(); + }; + assert!(completed, "actual native callback return remains pending"); + assert!(user_entered && wait_entered && replacement_rejected); + assert_eq!(count_before_drop, 1); + assert_eq!( + terminal_before_drop, + crate::module::abi::TB_NATIVE_TERMINAL_STOP_INTENT + ); + assert!(lifecycle_free); + assert_eq!( + observed.load(Ordering::Acquire), + 1, + "user future destruction precedes counter release" + ); + assert_eq!(HostCallbackGuard(account).running(), 0); + assert!(barrier.state.lock().unwrap().2); + assert!(!transport.is_faulted()); +} + +#[tokio::test] +async fn completing_provider_future_is_destroyed_before_its_running_account() { + let account = state(":1.1".parse().unwrap(), ":1.2".parse().unwrap()); + let observed = Arc::new(AtomicUsize::new(0)); + let guarded = GuardedInterface { + interface: PendingUntilDestroyed { + complete: true, + entered: Arc::new(Notify::new()), + guard: HostCallbackGuard(account.clone()), + observed: observed.clone(), + }, + endpoint: endpoint(), + state: account.clone(), + }; + let context = CallContext::brokered(account.module_peer().as_ref(), true); + let result = guarded + .call_with_context(&endpoint().members[0], serde_json::json!([]), &context) + .await; + assert!(result.is_ok()); + assert_eq!(observed.load(Ordering::Acquire), 1); + assert_eq!(HostCallbackGuard(account).running(), 0); +} diff --git a/crates/tinybus/src/module/host_reload_tests.rs b/crates/tinybus/src/module/host_reload_tests.rs index 22e38331..a065e4c2 100644 --- a/crates/tinybus/src/module/host_reload_tests.rs +++ b/crates/tinybus/src/module/host_reload_tests.rs @@ -66,13 +66,27 @@ static LOGS: OnceLock>>> = OnceLock::new(); unsafe extern "C" fn observe_log(ctx: *mut std::ffi::c_void, _: u32, ptr: *const u8, len: usize) { let line = String::from_utf8_lossy(unsafe { std::slice::from_raw_parts(ptr, len) }).into_owned(); + record_log(LOGS.get(), ctx as usize, line); +} +fn record_log(storage: Option<&Mutex>>>, key: usize, line: String) { + if let Some(storage) = storage { + storage + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .entry(key) + .or_default() + .push(line); + } +} +fn recorded_lines(key: usize) -> Vec { LOGS.get() - .unwrap() - .lock() - .unwrap() - .entry(ctx as usize) - .or_default() - .push(line); + .and_then(|logs| { + logs.lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&key) + .cloned() + }) + .unwrap_or_default() } fn observed_native(host: &ModuleHost, path: &Path, prefix: &str) -> usize { let artifact = loader::load(path, false).unwrap(); @@ -85,7 +99,7 @@ fn observed_native(host: &ModuleHost, path: &Path, prefix: &str) -> usize { let key = vtable.host_ctx as usize; LOGS.get_or_init(|| Mutex::new(HashMap::new())) .lock() - .unwrap() + .unwrap_or_else(std::sync::PoisonError::into_inner) .insert(key, Vec::new()); vtable.log = observe_log; transport.defer_initialize(artifact.init, vtable); @@ -131,8 +145,9 @@ async fn native_replacement_logs_and_sanitized_panics_use_only_its_original_host .unwrap() .starts_with("first:") ); + let first_lines = recorded_lines(first_key); assert!( - LOGS.get().unwrap().lock().unwrap()[&first_key] + first_lines .iter() .any(|line| line.contains("clock fixture served")) ); @@ -151,7 +166,7 @@ async fn native_replacement_logs_and_sanitized_panics_use_only_its_original_host .await .unwrap(); drop(first); - let first_count = LOGS.get().unwrap().lock().unwrap()[&first_key].len(); + let first_count = recorded_lines(first_key).len(); let second = ModuleHost::new(broker); let second_key = observed_native(&second, &artifact, "second:"); assert_ne!(first_key, second_key); @@ -163,29 +178,226 @@ async fn native_replacement_logs_and_sanitized_panics_use_only_its_original_host .starts_with("second:") ); assert!(proxy.call::("Panic", ()).await.is_err()); - { - let logs = LOGS.get().unwrap().lock().unwrap(); + let first_lines = recorded_lines(first_key); + let second_lines = recorded_lines(second_key); + second.shutdown(Duration::from_secs(3)).await; + broker_task.abort(); + assert_eq!( + first_lines.len(), + first_count, + "B must not call A's original logging callback" + ); + assert!( + second_lines + .iter() + .any(|line| line.contains("clock fixture served")) + ); + assert!( + second_lines + .iter() + .any(|line| line.contains("module panicked at")) + ); + assert!( + second_lines + .iter() + .all(|line| !line.contains("secret-token")) + ); +} + +#[test] +fn native_log_fixture_recovers_poison_without_unwinding_across_abi() { + let storage = Mutex::new(HashMap::new()); + let context = 0u8; + let key = std::ptr::from_ref(&context) as usize; + let poisoned = std::panic::catch_unwind(|| { + let _guard = storage.lock().unwrap(); + panic!("controlled Rust fixture poison"); + }); + // Exercise the callback's Rust body first, safely containing baseline RED. + let recovered = std::panic::catch_unwind(|| record_log(Some(&storage), 1, "local".into())); + let unavailable = std::panic::catch_unwind(|| record_log(None, 1, "unavailable".into())); + if recovered.is_ok() { + let logs = LOGS.get_or_init(|| Mutex::new(HashMap::new())); + let _ = std::panic::catch_unwind(|| { + let _guard = logs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + panic!("controlled Rust fixture poison"); + }); + let line = b"actual ABI callback after fixture poison"; + unsafe { observe_log(key as *mut std::ffi::c_void, 1, line.as_ptr(), line.len()) }; + } + let recorded = LOGS.get().and_then(|logs| { + logs.lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&key) + .cloned() + }); + assert!(poisoned.is_err()); + assert!( + unavailable.is_ok(), + "unavailable fixture storage must not unwind" + ); + assert!( + recovered.is_ok(), + "callback body must recover poisoned fixture storage" + ); + assert!(recorded.is_some_and(|lines| { + lines + .iter() + .any(|line| line.contains("actual ABI callback")) + })); +} + +#[cfg(unix)] +unsafe fn terminal_symbol(path: &Path, name: &std::ffi::CStr) -> T { + use std::os::unix::ffi::OsStrExt; + #[cfg_attr(target_os = "linux", link(name = "dl"))] + unsafe extern "C" { + fn dlopen(path: *const std::ffi::c_char, flags: i32) -> *mut std::ffi::c_void; + fn dlsym( + handle: *mut std::ffi::c_void, + name: *const std::ffi::c_char, + ) -> *mut std::ffi::c_void; + fn dlclose(handle: *mut std::ffi::c_void) -> i32; + } + let path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + let handle = unsafe { dlopen(path.as_ptr(), 2) }; + assert!(!handle.is_null()); + let raw = unsafe { dlsym(handle, name.as_ptr()) }; + unsafe { + dlclose(handle); + } + assert!(!raw.is_null()); + assert_eq!(std::mem::size_of::(), std::mem::size_of_val(&raw)); + unsafe { std::mem::transmute_copy(&raw) } +} +#[repr(C)] +#[derive(Clone, Copy)] +struct TerminalInitControls { + ctx: *mut std::ffi::c_void, + event: unsafe extern "C" fn(*mut std::ffi::c_void, u32, u32), +} +struct TerminalInitBarrier { + entered: std::sync::mpsc::Sender<()>, + release: (std::sync::Mutex, std::sync::Condvar), +} +unsafe extern "C" fn terminal_init_event(ctx: *mut std::ffi::c_void, event: u32, _: u32) { + if event != 18 { + return; + } + let control = unsafe { &*ctx.cast::() }; + let _ = control.entered.send(()); + let mut release = control.release.0.lock().unwrap(); + while !*release { + release = control.release.1.wait(release).unwrap(); + } +} +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires the privately pinned __native-fixture-support eight-artifact cohort"] +async fn compiled_native_host_stop_and_fault_deny_factory_after_entered_init_barrier() { + use crate::module::abi::*; + let path = PathBuf::from(std::env::var_os("TINYBUS_TEST_TERMINAL_MODULE").unwrap()); + // Real loader admission pins the image before any test symbol resolution. + let artifact = loader::load(&path, false).unwrap(); + let configure: unsafe extern "C" fn( + usize, + *mut std::ffi::c_void, + unsafe extern "C" fn(*mut std::ffi::c_void, u32, u32), + ) = unsafe { terminal_symbol(&path, c"terminal_test_factory_mode") }; + let factories: extern "C" fn() -> usize = + unsafe { terminal_symbol(&path, c"terminal_test_factories") }; + for fault in [false, true] { + let broker = Broker::new(); + let broker_task = broker.clone().spawn(MemoryBus::new()); + let host = ModuleHost::new(broker); + let admitted = host + .validate(&path, &artifact.descriptor, &artifact.manifest) + .unwrap(); + let (sender, receiver) = std::sync::mpsc::channel(); + let control = Box::into_raw(Box::new(TerminalInitBarrier { + entered: sender, + release: (std::sync::Mutex::new(false), std::sync::Condvar::new()), + })); + let controls = TerminalInitControls { + ctx: control.cast(), + event: terminal_init_event, + }; + let encoded = unsafe { + std::slice::from_raw_parts( + (&controls as *const TerminalInitControls).cast::(), + std::mem::size_of::(), + ) + }; + let config = + serde_json::to_vec(&serde_json::json!({"mode": 0, "test_controls": encoded})).unwrap(); + let (transport, calls) = ModuleTransport::new(admitted.name.clone(), config); + unsafe { + configure(30, control.cast(), terminal_init_event); + } + let before = factories(); + transport.defer_initialize(artifact.init, calls); + // Attach the genuine loaded artifact and transport to the real host, + // matching the existing observed_native deferred-init fixture above. + host.attach_transport( + &path, + admitted, + artifact.manifest.clone(), + transport.clone(), + Activation { + lazy_init: true, + lazy_load: false, + descriptor_info: None, + pinned_sha256: None, + }, + ) + .unwrap(); + let initialize = { + let transport = transport.clone(); + tokio::spawn(async move { transport.ensure_initialized().await }) + }; + let entered = receiver.recv_timeout(Duration::from_secs(5)).is_ok(); + if fault { + unsafe { + (calls.fault)(calls.host_ctx, std::ptr::null(), 0); + } + } else { + host.shutdown(Duration::from_millis(20)).await; + } + let flags = unsafe { calls.native_terminal_state.unwrap()(calls.host_ctx) }; + // Release and await the actual opaque init worker before assertions. + unsafe { + *(*control).release.0.lock().unwrap() = true; + (*control).release.1.notify_all(); + } + let initialized = tokio::time::timeout(Duration::from_secs(5), initialize) + .await + .unwrap(); + host.shutdown(Duration::from_secs(5)).await; + drop(host); + let final_flags = unsafe { calls.native_terminal_state.unwrap()(calls.host_ctx) }; + unsafe { + drop(Box::from_raw(control)); + } + broker_task.abort(); + let _ = broker_task.await; + assert!(entered); + assert!(initialized.is_ok_and(|result| result.is_err())); assert_eq!( - logs[&first_key].len(), - first_count, - "B must not call A's original logging callback" - ); - assert!( - logs[&second_key] - .iter() - .any(|line| line.contains("clock fixture served")) + factories(), + before, + "terminal authority must deny before the actual factory" ); - assert!( - logs[&second_key] - .iter() - .any(|line| line.contains("module panicked at")) - ); - assert!( - logs[&second_key] - .iter() - .all(|line| !line.contains("secret-token")) + assert_ne!( + flags + & if fault { + TB_NATIVE_TERMINAL_FAULT + } else { + TB_NATIVE_TERMINAL_STOP_INTENT + }, + 0 ); + assert_eq!(final_flags & TB_NATIVE_TERMINAL_PHYSICALLY_STOPPED, 0); } - second.shutdown(Duration::from_secs(3)).await; - broker_task.abort(); } diff --git a/crates/tinybus/src/module/host_tests.rs b/crates/tinybus/src/module/host_tests.rs index b7925af1..7c19bf25 100644 --- a/crates/tinybus/src/module/host_tests.rs +++ b/crates/tinybus/src/module/host_tests.rs @@ -746,6 +746,7 @@ fn a_sidecar_without_lazy_init_refuses_the_artifact_instead_of_loading_it_eagerl #[tokio::test] async fn a_module_whose_init_fails_is_terminal_and_is_never_initialized_again() { + let _test_guard = FAKE_MODULE_TEST_LOCK.lock().await; FAILED_INIT_COUNT.store(0, Ordering::Release); let bus = MemoryBus::new(); let broker = Broker::new(); @@ -1869,11 +1870,11 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { ); let _ = ModuleHost::new(Broker::new()).load_search_paths(); - let directory = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); - // Files inherit the private DACL, so these assertions exercise content gates. - #[cfg(windows)] - crate::module::cache::secure_release_cache(directory.path(), directory.path()); - let module = directory.path().join(if cfg!(windows) { + let root = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); + let directory = root.path().join("private"); + // Protect the artifact parent before files inherit the checkout's DACL. + super::super::windows_acl::create_private_dir_all(&directory).unwrap(); + let module = directory.join(if cfg!(windows) { "clock.dll" } else if cfg!(target_os = "macos") { "clock.dylib" @@ -1883,7 +1884,7 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { std::fs::write(&module, b"module bytes").unwrap(); let digest = crate::module::hash::file_hex(std::fs::File::open(&module).unwrap()).unwrap(); std::fs::write( - directory.path().join("modules.toml"), + directory.join("modules.toml"), format!( "clock.{} = \"{digest}\"\n", if cfg!(windows) { @@ -1897,7 +1898,7 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { ) .unwrap(); assert!(check_file(&module).is_ok()); - let text = directory.path().join("clock.txt"); + let text = directory.join("clock.txt"); std::fs::write(&text, b"module bytes").unwrap(); assert!( check_file(&text) @@ -1905,18 +1906,14 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { .to_string() .contains("extension is not loadable") ); - std::fs::write(directory.path().join("modules.toml"), "other = \"00\"\n").unwrap(); + std::fs::write(directory.join("modules.toml"), "other = \"00\"\n").unwrap(); assert!( check_file(&module) .unwrap_err() .to_string() .contains("absent") ); - std::fs::write( - directory.path().join("modules.toml"), - "clock = \"not-a-hash\"\n", - ) - .unwrap(); + std::fs::write(directory.join("modules.toml"), "clock = \"not-a-hash\"\n").unwrap(); assert!( check_file(&module) .unwrap_err() @@ -1931,10 +1928,10 @@ fn module_host_helpers_preserve_safe_names_states_and_allowlist_decisions() { #[test] fn an_allowlist_with_a_mismatched_hash_refuses_the_file() { - let directory = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); - // Files inherit the private DACL, so these assertions exercise content gates. - #[cfg(windows)] - crate::module::cache::secure_release_cache(directory.path(), directory.path()); + let root = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); + let directory = root.path().join("private"); + // Protect the artifact parent before files inherit the checkout's DACL. + super::super::windows_acl::create_private_dir_all(&directory).unwrap(); let extension = if cfg!(windows) { "dll" } else if cfg!(target_os = "macos") { @@ -1942,10 +1939,10 @@ fn an_allowlist_with_a_mismatched_hash_refuses_the_file() { } else { "so" }; - let path = directory.path().join(format!("clock.{extension}")); + let path = directory.join(format!("clock.{extension}")); std::fs::write(&path, b"not a module").unwrap(); std::fs::write( - directory.path().join("modules.toml"), + directory.join("modules.toml"), format!("clock.{extension} = \"{}\"\n", "0".repeat(64)), ) .unwrap(); diff --git a/crates/tinybus/src/module/runtime_tests.rs b/crates/tinybus/src/module/runtime_tests.rs index 978a789c..63274014 100644 --- a/crates/tinybus/src/module/runtime_tests.rs +++ b/crates/tinybus/src/module/runtime_tests.rs @@ -15,6 +15,7 @@ async fn setup_runs_before_the_runtime_is_handed_out() { let runtime = ModuleRuntime::start(&RuntimeNames::default(), Some(setup)).unwrap(); assert!(ran.load(Ordering::SeqCst)); assert!(runtime.host().list().is_empty()); + assert!(runtime.connection().unique_name().is_some()); } #[tokio::test] diff --git a/crates/tinybus/src/module/transport.rs b/crates/tinybus/src/module/transport.rs index 152fab0a..25aae37f 100644 --- a/crates/tinybus/src/module/transport.rs +++ b/crates/tinybus/src/module/transport.rs @@ -3,7 +3,7 @@ use std::collections::VecDeque; use std::ffi::c_void; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use std::sync::{Arc, Condvar, Mutex as StdMutex, Weak}; +use std::sync::{Arc, Condvar, Mutex as StdMutex, OnceLock, Weak}; use std::time::Duration; use async_trait::async_trait; @@ -74,6 +74,8 @@ pub(crate) struct ModuleTransport { native_lifecycle_pending: Arc, stop_task_spawned: Arc, callbacks: CallbackGate, + // One exact provider account. Its transport backlink is weak. + host_callbacks: OnceLock>, initialization_gate: StdMutex>, initialization_gate_changed: Notify, #[cfg(test)] @@ -290,6 +292,7 @@ impl ModuleTransport { native_lifecycle_pending: Arc::new(AtomicBool::new(false)), stop_task_spawned: Arc::new(AtomicBool::new(false)), callbacks: CallbackGate::default(), + host_callbacks: OnceLock::new(), initialization_gate: StdMutex::new(Some(true)), initialization_gate_changed: Notify::new(), #[cfg(test)] @@ -771,8 +774,25 @@ impl ModuleTransport { } } + pub(crate) fn bind_host_callbacks( + &self, + callbacks: Arc, + ) -> Result<()> { + self.host_callbacks + .set(callbacks) + .map_err(|_| Error::failed("module callback binding is already installed")) + } + pub(crate) fn native_lifecycle_pending(&self) -> bool { self.native_lifecycle_pending.load(Ordering::Acquire) + || (self.admission_closed() && !self.is_stopped()) + || self + .context + .physical + .lock() + .expect("physical state lock") + .job + .is_some() } pub(crate) fn is_ready(&self) -> bool { @@ -1056,6 +1076,11 @@ impl ModuleTransport { code } + #[cfg(test)] + pub(crate) fn native_terminal_state_for_test(&self) -> u32 { + unsafe { host_native_terminal_state(std::ptr::from_ref(self.context).cast_mut().cast()) } + } + #[cfg(test)] pub(crate) fn lifecycle_is_owned_for_test(&self) -> bool { self.lifecycle.try_lock().is_err() @@ -1269,12 +1294,14 @@ impl PhysicalJob { .expect("physical state lock"); } } - let guard = self - .guard - .lock() - .expect("physical guard lock") - .take() - .unwrap_or_else(|| transport.lifecycle.clone().blocking_lock_owned()); + let mut acquired = self.guard.lock().expect("physical guard lock").take(); + if let Some(callbacks) = transport.host_callbacks.get() { + // User futures may need lifecycle services. Retain this physical + // owner, but release native exclusion before their actual drain. + drop(acquired.take()); + callbacks.wait_drained(); + } + let guard = acquired.unwrap_or_else(|| transport.lifecycle.clone().blocking_lock_owned()); { let mut state = transport .context diff --git a/crates/tinybus/src/module/transport_tests.rs b/crates/tinybus/src/module/transport_tests.rs index df0494e8..4af6e634 100644 --- a/crates/tinybus/src/module/transport_tests.rs +++ b/crates/tinybus/src/module/transport_tests.rs @@ -364,7 +364,7 @@ async fn a_failed_initializer_is_reported_to_callers() { #[tokio::test(flavor = "current_thread")] async fn a_pending_call_is_failed_when_the_module_closes_its_delivery_queue() { - let _lock = VTABLE_TEST_LOCK + let lock = VTABLE_TEST_LOCK .get_or_init(|| tokio::sync::Mutex::new(())) .lock() .await; @@ -376,21 +376,122 @@ async fn a_pending_call_is_failed_when_the_module_closes_its_delivery_queue() { transport.defer_initialize(initialize_ok, host); transport.send(call()).await.unwrap(); unsafe { (host.ready)(host.host_ctx) }; - let reply = tokio::time::timeout(Duration::from_secs(1), transport.recv()) - .await - .unwrap() - .unwrap() - .unwrap(); - assert_eq!(reply.header.kind, MessageKind::Error); - assert!(transport.is_faulted()); - shutdown_entry().await; + let reply = tokio::time::timeout(Duration::from_secs(1), transport.recv()).await; + let faulted = transport.is_faulted(); + // A failed receive may have no fault-owned cleanup job. Commit cleanup + // before reporting the captured result; this does not clear an actual fault. + transport.request_terminal(false); + let shutdown_entered = tokio::time::timeout(Duration::from_secs(2), async { + while !SHUTDOWN_ENTERED.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await; + // Unblock the callback before observing physical completion, while the + // fixture lock still protects the callback flags and delivery result. SHUTDOWN_BLOCKED.store(false, Ordering::Release); wait_physical_cleanup(&transport).await; + // The release guard also resets shared flags; drop it before giving the + // fixture lock to another test. + drop(_release); + finish_closed_delivery_fixture(&transport, lock).await; + let reply = reply.unwrap().unwrap().unwrap(); + assert_eq!(reply.header.kind, MessageKind::Error); + assert!(faulted); + assert!( + shutdown_entered.is_ok(), + "fault cleanup must enter shutdown" + ); +} + +// Keep the shared vtable fixture serialized through its final global writes. +async fn finish_closed_delivery_fixture( + transport: &ModuleTransport, + lock: tokio::sync::MutexGuard<'_, ()>, +) { + wait_physical_cleanup(transport).await; assert!( transport.is_stopped(), "a failed-delivery fixture must finish shutdown before reusing callback flags" ); DELIVERY_CODE.store(TB_OK, Ordering::Release); + drop(lock); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_fault_reply_does_not_release_the_fixture_before_physical_cleanup() { + use std::future::Future; + + struct WorkerRelease(Option>); + impl Drop for WorkerRelease { + fn drop(&mut self) { + if let Some(release) = self.0.take() { + let _ = release.send(()); + } + } + } + + let lock = VTABLE_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; + SHUTDOWN_ENTERED.store(false, Ordering::Release); + SHUTDOWN_CALLS.store(0, Ordering::Release); + let (transport, host) = ModuleTransport::new("serialized-fault-cleanup".into(), vec![]); + transport.defer_initialize(initialize_ok, host); + transport.ensure_initialized().await.unwrap(); + let (started, started_rx) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); + let worker_release = WorkerRelease(Some(release)); + *transport.before_worker_lock.lock().unwrap() = Some(Box::new(move || { + let _ = started.send(()); + let _ = release_rx.recv(); + })); + DELIVERY_CODE.store(TB_CLOSED, Ordering::Release); + transport.send(call()).await.unwrap(); + unsafe { (host.ready)(host.host_ctx) }; + // The actual independently owned fault worker cannot acquire lifecycle yet. + let reply = tokio::time::timeout(Duration::from_secs(2), transport.recv()).await; + let started = tokio::time::timeout(Duration::from_secs(2), started_rx).await; + let faulted_before_cleanup = transport.is_faulted(); + let callback_pending = !SHUTDOWN_ENTERED.load(Ordering::Acquire) + && !matches!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Complete | CleanupState::Uncertain | CleanupState::NoCallbackRequired + ); + let mut completion = Box::pin(finish_closed_delivery_fixture(&transport, lock)); + let completed_early = + std::future::poll_fn(|cx| std::task::Poll::Ready(completion.as_mut().poll(cx).is_ready())) + .await; + let released_early = VTABLE_TEST_LOCK.get().unwrap().try_lock().is_ok(); + + // Release and observe the real callback before any regression assertion. + // The transport retains the actual context; cancelling an observer is not cleanup. + drop(worker_release); + wait_physical_cleanup(&transport).await; + let callback_entered = SHUTDOWN_ENTERED.load(Ordering::Acquire); + let callback_calls = SHUTDOWN_CALLS.load(Ordering::Acquire); + if !completed_early { + completion.await; + } + // Another queued fixture can acquire first; await our turn rather than + // treating legitimate contention after release as a failure. + let next_fixture = tokio::time::timeout( + Duration::from_secs(2), + VTABLE_TEST_LOCK.get().unwrap().lock(), + ) + .await; + assert!(matches!(reply, Ok(Ok(Some(ref reply))) if reply.header.kind == MessageKind::Error)); + assert!(matches!(started, Ok(Ok(())))); + assert!(faulted_before_cleanup && callback_pending); + assert!(callback_entered); + assert_eq!(callback_calls, 1); + assert!( + !completed_early, + "fault reply is not physical fixture completion" + ); + assert!(!released_early, "next fixture must wait for actual cleanup"); + assert!(next_fixture.is_ok()); } #[tokio::test] @@ -801,11 +902,11 @@ async fn stopping_initialized_module_before_ready_releases_observers_and_queued_ #[tokio::test] async fn ordinary_stop_wakes_backpressure_without_turning_closure_into_a_fault() { - let _release = ShutdownRelease; let _lock = VTABLE_TEST_LOCK .get_or_init(|| tokio::sync::Mutex::new(())) .lock() .await; + let _release = ShutdownRelease; SHUTDOWN_BLOCKED.store(false, Ordering::Release); DELIVERY_CODE.store(TB_BACKPRESSURE, Ordering::Release); DELIVERIES.store(0, Ordering::Release); @@ -1536,25 +1637,50 @@ unsafe impl Send for NativeBarrier {} unsafe impl Sync for NativeBarrier {} unsafe extern "C" fn native_entered(ctx: *mut c_void, host: *const TbHostVtable) { let barrier = unsafe { &*ctx.cast::() }; - let mut state = barrier.state.lock().unwrap(); + let mut state = barrier + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); state.0 = Some(unsafe { *host }); barrier.entered.notify_one(); while !state.1 { - state = barrier.changed.wait(state).unwrap(); + state = barrier + .changed + .wait(state) + .unwrap_or_else(std::sync::PoisonError::into_inner); } } unsafe extern "C" fn native_joined(ctx: *mut c_void) { let barrier = unsafe { &*ctx.cast::() }; - let mut state = barrier.state.lock().unwrap(); + let mut state = barrier + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); assert!(!state.2, "one actual native shutdown/join"); state.2 = true; barrier.joined.notify_one(); drop(state); - let gate = barrier.finish_gate.lock().unwrap().take(); + let gate = barrier + .finish_gate + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + // This artificial gate runs after the fixture's actual worker join. A failed + // Rust assertion may disconnect it; never unwind through the C callback. + finish_native_control(gate); +} +fn finish_native_control(gate: Option>) { if let Some(gate) = gate { - gate.recv().unwrap(); + let _ = gate.recv(); } } +#[test] +fn disconnected_artificial_finish_gate_does_not_unwind() { + let (release, receive) = std::sync::mpsc::channel(); + drop(release); + let result = std::panic::catch_unwind(|| finish_native_control(Some(receive))); + assert!(result.is_ok()); +} pub(crate) fn native_fixture() -> ( Arc, TbHostVtable, @@ -2186,7 +2312,15 @@ async fn rejected_observer_after_ordinary_stop_preserves_nonfault_cause() { #[tokio::test] async fn completed_shutdown_has_already_released_its_native_lifecycle_marker() { - let _guard = VTABLE_TEST_LOCK + struct WorkerRelease(Option>); + impl Drop for WorkerRelease { + fn drop(&mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } + } + let guard = VTABLE_TEST_LOCK .get_or_init(|| tokio::sync::Mutex::new(())) .lock() .await; @@ -2197,17 +2331,45 @@ async fn completed_shutdown_has_already_released_its_native_lifecycle_marker() { let published = Arc::new(Notify::new()); let entered = published.clone(); let (release, worker) = std::sync::mpsc::channel(); + let release = WorkerRelease(Some(release)); *transport.after_cleanup_publication.lock().unwrap() = Some(Box::new(move || { entered.notify_one(); - worker.recv().unwrap(); + let _ = worker.recv(); })); - let stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); - published.notified().await; - let result = stopping.await.unwrap(); + let mut stopping = tokio::spawn(transport.clone().stop(Duration::from_secs(2))); + let publication = tokio::time::timeout(Duration::from_secs(2), published.notified()).await; + let result = tokio::time::timeout(Duration::from_secs(2), &mut stopping).await; let still_pending = transport.native_lifecycle_pending(); - // Always release the worker before asserting, even against the unfixed code. - release.send(()).unwrap(); - assert_eq!(result.unwrap(), TB_OK); + drop(release); + transport.request_terminal(false); + let cleanup = tokio::time::timeout(Duration::from_secs(2), async { + loop { + let changed = transport.context.physical_notify.notified(); + if matches!( + transport.context.physical.lock().unwrap().cleanup, + CleanupState::Complete | CleanupState::NoCallbackRequired + ) { + break; + } + changed.await; + } + // Publication precedes native lifecycle unlock. Observe that actual + // exclusion release separately; this is not an OS-worker-exit claim. + let lifecycle = transport.lifecycle.clone().lock_owned().await; + drop(lifecycle); + }) + .await; + if cleanup.is_err() { + // Keep fixture exclusion with the actual transport/job if cleanup is + // unresolved. Neither timeout nor dropped async waiter reclaims it. + *transport.fixture_controller.lock().unwrap() = Some(Arc::new(guard)); + } + assert!(publication.is_ok(), "cleanup publication watchdog expired"); + assert!( + cleanup.is_ok(), + "actual cleanup/lifecycle release remains pending" + ); + assert_eq!(result.unwrap().unwrap().unwrap(), TB_OK); assert!( !still_pending, "completed cleanup still reports native work" diff --git a/docs/modules/module/README.md b/docs/modules/module/README.md index 164cd494..cad1f0a6 100644 --- a/docs/modules/module/README.md +++ b/docs/modules/module/README.md @@ -44,6 +44,78 @@ last initialized instance. Linked modules keep the embedding host's tracing and panic hook. The existing instance-wide last panic location is diagnostic state, not a proof of exact attribution among concurrently panicking methods. +## Opted-in terminal factories + +Configured modules can opt in with `module_export!(terminal_setup_factory = +factory, config = Config, ...)`; the static export macro accepts the same +arguments. The factory receives one `NativeInstanceLiveness` and returns its +setup callback plus `RuntimeHooks`. An unsupported or non-live native query +rejects initialization before the factory runs. Linked modules require the +same explicit host query and retain the embedding process's panic hook. The +factory owns cleanup of partial construction until it returns its hooks. It +must defer independent worker launch until setup, after hook registration; +reinitialization reuses the same controller and never replaces the hooks. + +A retained liveness view refers to that exact instance. Only a fresh, exactly +Live native query admits work; any observed terminal intent permanently closes +the view. `request_fault()` first reports genuine native fault. The view grants +no publication lease and does not turn local cancellation into native success. +Under its physical phase gate, the trusted controller must check immutable +instance/receiver binding and local terminal flags, sample the view, then +recheck local flags. A cached Live result cannot authorize a later phase; +subsequent intent cannot undo an operation already admitted. + +The request hook runs at most once and must only close controller admission and +perform bounded cancel/wake. It must not join, do I/O, wait for a gate, or reenter +the SDK. Finish runs independently at most once on the physical cleanup thread, +after startup ownership is published, request has returned, the sole initial +setup task has settled, and any admitted reinitializer has returned. Finish must +fence and join the controller's actual workers. `PhysicallyStopped` asserts that +work; a timeout or host shutdown return does not prove it. + +The SDK retains setup futures across cancellation, destroys them on the +physical cleanup thread, and drops callback captures outside resource locks. +Reinitialization polls its result before terminal cancellation when both are +ready. Repeated successful reinitialization retires its previous future through +one bounded cell on that same physical worker; the next setup starts only after +the worker confirms disposal. Callback admission uses bookkeeping exclusion, +with foreign invocation, request and destruction outside every resource lock. +The runtime is destroyed on the physical thread. Surviving SDK resources, +spawn failure, a hook panic, or unproved cleanup keep the owner uncertain and +anchored. Seven separate primary panic cells retain request, finish, initial +setup, reinitialize, factory, startup, and cleanup payloads. Native fault and +request completion precede any payload disposal. After the first cleanup +destructor panic, later resource owners remain attached rather than being +destroyed during cleanup unwinding. Once actual setup execution has settled, +the controller finish still runs once; retained or panicking SDK teardown keeps +the result uncertain even if that controller finish succeeds. A counted observer +retains the entire Connection through its last field and all cloned method +observers. A destructor unwind explicitly reports native Fault even after +diagnostic callbacks have been suppressed; a transport count or runtime timeout +cannot certify whole-Connection destruction. + +Each SDK image has 256 instance slots. The existing opaque `module_ctx` contains +a slot/generation address token, never a dereferenced pointer or extra ABI +allocation. Callback lookup acquires the exact State before releasing registry +exclusion. Successful teardown retires its registry anchor; only actual State +last-Drop releases the slot, so old views and surviving SDK resources count +against the limit. Generations never wrap: exhaustion permanently denies that +slot. Null, stale or unknown context tokens return `TB_CLOSED` for otherwise +valid terminal callbacks and cannot select a replacement. Legacy shutdown's +null-context behavior remains unchanged. +Tokens are private bookkeeping, not native authentication. + +The SDK's `__native-fixture-support` feature is private and disabled by default. +It exposes only compiled-fixture instrumentation, not production authority. + +This metadata bound does not bound existing host-context or mapped-library +retention. Libraries remain mapped until process exit, including a rejected +initialization whose independent SDK finish job is still running. A rejected +initialization without an accepted vtable cannot earn host PhysicalStopped +evidence. The native query remains authoritative after local Stop and before +any successful shutdown return or context retirement. Legacy exports retain +their existing lifecycle behavior. + ## Loading sequence 1. Check every directory component's ownership/mode, require a regular diff --git a/docs/modules/module/abi.md b/docs/modules/module/abi.md index 11927a5f..656429d4 100644 --- a/docs/modules/module/abi.md +++ b/docs/modules/module/abi.md @@ -18,7 +18,7 @@ support. The host initializes the output vtable's leading `size` field with its allocation capacity before calling init. A newer module reads only that frozen field first and writes no more than the advertised capacity, so the additive tail is safe in both compatibility directions. -The SDK bounds reinitialization setup to four seconds and returns `TB_TIMEOUT` +The legacy SDK bounds reinitialization setup to four seconds and returns `TB_TIMEOUT` after cancelling the setup future. The host serializes this callback with shutdown and retains the lifecycle lock even if its own outer deadline expires. The final host callback, `ready`, marks completion of asynchronous setup. This @@ -54,9 +54,31 @@ including independent workers after runtime shutdown or host destruction. Host destruction must leave a terminal tombstone; this context must never be recycled or reopened. Custom unsafe C hosts must truthfully uphold this contract, which cannot cryptographically authenticate an in-process host. -This tail defines an optional ABI capability; ordinary SDK hosts currently -supply `None`. It introduces no retain/release ABI or aggregate resource bound: -the existing process-lifetime host-context and library retention is unbounded. +This tail defines an optional ABI capability. Unsupported hosts supply `None`; +terminal factories reject them before factory evaluation. It introduces no +retain/release ABI or aggregate resource bound: the existing process-lifetime +host-context and library retention is unbounded. + +Opted-in SDK instances use the existing opaque `module_ctx` field for a private +slot/generation address token. No callback dereferences it and no ABI field or +context allocation is added. Each SDK image has 256 slots; callback lookup +clones the exact matching State under registry exclusion, then releases that +lock before instance operations. Successful resource teardown may retire the +anchor, but slot reuse waits for actual State last-Drop, including retained +liveness views and SDK resources. Generation exhaustion permanently denies the +slot instead of wrapping. Null, unknown or retired tokens return `TB_CLOSED` +when the callback's other arguments are valid; legacy shutdown retains its +existing null-context behavior. An old instance token cannot act on a +replacement. These tokens are not serialized or +logged and do not authenticate a native host. + +Terminal reinitialization uses result-first polling when completion and terminal +intent are simultaneously ready. Cancellation leaves its foreign future owned +by the instance until physical cleanup. Its runtime and terminal captures are +destroyed on the independent physical cleanup thread. Uncertainty remains +sticky and cannot be refunded by a timeout or shutdown return. Genuine native +Fault is sampled even after local Stop, and faulted published instances retain +their context anchor instead of earning successful retirement. ## Symbols diff --git a/docs/modules/module/callback-lifecycle-validation.md b/docs/modules/module/callback-lifecycle-validation.md index 24b9b38e..eb022f27 100644 --- a/docs/modules/module/callback-lifecycle-validation.md +++ b/docs/modules/module/callback-lifecycle-validation.md @@ -25,18 +25,32 @@ artifact cases, plus the existing helper, admission and umask controls. ## Implementation -The loaded record retains its exact `CallbackState`, including operation -accounting, after routing revocation removes its route. Stop revokes first and -then waits for admitted user futures with its remaining deadline, before native -shutdown. No loaded, router or callback-state lock is held across that wait. +The loaded record and transport retain one exact `CallbackState`, including +operation accounting, after routing revocation removes its route. The account +is bound before callback admission and cannot be replaced. Its backlink to the +transport remains weak. -A marker owned by the actual blocking native callback survives public waiter -cancellation or timeout. Enable/disable checks it under the same loaded mutex -that admits lifecycle transitions. The marker drops before the native lifecycle -mutex unlocks, so a finishing callback cannot clear a successor's marker. Stop -and reinitialize keep their canonical queued semantics. +Stop revokes routing and commits native stop intent before waiting for admitted +user futures with its remaining deadline. This closes native admission without +submitting physical shutdown. Drain timeout or public waiter cancellation keeps +stop intent pending; enable cannot restore admission. Once the actual provider +account drains, stop submits the existing physical owner. Ordinary stop intent +is not upgraded to a fault. -## Fresh acceptance +Last-host drop also uses that physical owner. Its OS worker revokes the exact +account and waits for its actual running count to reach zero before acquiring +the native lifecycle mutex or calling opaque shutdown. This wait has no runtime +dependency or deadline. A provider that never releases its future keeps the +physical job pending. Provider futures are destroyed before their running +accounts decrement, including when their runtime is destroyed. No provider code +or destructor runs under the accounting mutex. + +Reinitialize retains its marker in the actual blocking callback until it +returns. Enable/disable also checks committed stop intent and the retained +physical job, including uncertain cleanup. Public waiter completion or timeout +is not evidence of physical reclamation. + +## Upstream acceptance before integration Run 87340 completed with exit 0 on source `372a0ddd97d23daf58203d59b95b7be79fc3393a`. All 3,068 output lines were inspected: diff --git a/docs/plans/native-terminal-authority.md b/docs/plans/native-terminal-authority.md index 59fd4a9a..49d4a8ed 100644 --- a/docs/plans/native-terminal-authority.md +++ b/docs/plans/native-terminal-authority.md @@ -26,8 +26,12 @@ owner scope/generation/receiver, local flags, live native query, then local flag again. Query success is admission linearization; later intent cannot undo an OS operation already granted. Query alone is not a publication permit. -Normal stop commits intent only after acquiring actual lifecycle lock, before -handing it to retained physical task. Canceled/unacquired stop makes no commitment. +Host stop commits ordinary intent before its bounded provider-callback drain. +Drain timeout or cancellation preserves that intent and the loaded owner; native +physical handoff waits for actual provider completion. An entirely unpolled stop +has no effect. Direct transport stop commits after acquiring the native lifecycle +lock, before handing it to the retained physical task; canceling that direct wait +before acquisition makes no new commitment. Actual fault, rollback and Host Drop commit immediately, even behind opaque C init or reinit. Fault takes public-state precedence; normal stop never becomes Faulted. Fault cleanup must not create ordinary transition_from or restart terminal modules. diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index adea2aaf..9f1cd67a 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -1,22 +1,24 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." -cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --no-default-features --features modules,macros +cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --example module_terminal --no-default-features --features modules,macros,__native-fixture-support set -euo pipefail -module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_clock.so' -print -quit) -module_two=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_clock_two.so' -print -quit) -wrong_target=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_wrong_target.so' -print -quit) -context_module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_context.so' -print -quit) -context_eager_module=$(find target/debug/examples -maxdepth 1 -type f -name 'libmodule_context_eager.so' -print -quit) -host_callback_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) -controlled_init_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) -for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module"; do +target_dir=$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])') +module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f -name 'libmodule_clock.so' -print -quit) +module_two=$(find "$target_dir/debug/examples" -maxdepth 1 -type f -name 'libmodule_clock_two.so' -print -quit) +wrong_target=$(find "$target_dir/debug/examples" -maxdepth 1 -type f -name 'libmodule_wrong_target.so' -print -quit) +context_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f -name 'libmodule_context.so' -print -quit) +context_eager_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f -name 'libmodule_context_eager.so' -print -quit) +host_callback_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) +controlled_init_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) +terminal_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_terminal.so' -o -name 'libmodule_terminal.dylib' \) -print -quit) +for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module" "$terminal_module"; do if [ -z "$artifact" ] || [ ! -f "$artifact" ]; then echo "a module loader fixture is missing" >&2 exit 1 fi done -private="$PWD/target/private-module-tests/module-clock-two" +private="$PWD/target/private-module-coverage/module-clock-two" mkdir -p "$private" chmod 700 "$private" cp "$module_two" "$private/" @@ -27,6 +29,20 @@ else digest="$(shasum -a 256 "$private/$library" | awk '{print $1}')" fi printf '"%s" = "%s"\n' "$library" "$digest" > "$private/modules.toml" +# Terminal fixtures can retain uncertain owners; pin a private copy and run +# each SDK scenario in a fresh process so its retained state cannot leak. +terminal_private="$PWD/target/private-module-coverage/module-terminal" +mkdir -p "$terminal_private" +chmod 700 "$terminal_private" +cp "$terminal_module" "$terminal_private/" +terminal_library="$(basename "$terminal_module")" +if command -v sha256sum >/dev/null 2>&1; then + terminal_digest="$(sha256sum "$terminal_private/$terminal_library" | awk '{print $1}')" +else + terminal_digest="$(shasum -a 256 "$terminal_private/$terminal_library" | awk '{print $1}')" +fi +printf '"%s" = "%s"\n' "$terminal_library" "$terminal_digest" > "$terminal_private/modules.toml" +export TINYBUS_TEST_TERMINAL_MODULE="$terminal_private/$terminal_library" # Port traits declare contracts only; their transport implementations # exercise those contracts, but traits have no executable body to cover. # CLI/host/connection integration surfaces and the network release path are @@ -34,4 +50,12 @@ printf '"%s" = "%s"\n' "$library" "$digest" > "$private/modules.toml" # process/network branches are not meaningful per-file coverage. The # test-support module is integration-test glue whose platform-specific # dynamic-loader and timeout paths run in the dedicated OS jobs instead. -TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$PWD/$controlled_init_module" cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 -- --include-ignored +export TINYBUS_TEST_MODULE="$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$controlled_init_module" +cargo llvm-cov --workspace --all-features --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --no-report -- --include-ignored --skip terminal_runtime_tests::compiled_native_ + +terminal_tests=$(cargo test --locked -p tinybus-module --all-features --features tinybus/macros -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') +[ -n "$terminal_tests" ] || { echo "no compiled terminal scenarios found" >&2; exit 1; } +while IFS= read -r test; do + cargo llvm-cov test --no-report -p tinybus-module --all-features --features tinybus/macros "$test" -- --ignored --exact +done <<< "$terminal_tests" +cargo llvm-cov report --ignore-filename-regex 'crates/tinybus/src/ports/(listener|transport)\.rs$|crates/tinybus/src/module/github\.rs$|crates/tinybus/src/module/(host|mod)\.rs$|crates/tinybus/src/bin/(tinybus\.rs|tinybus/unix\.rs)$|crates/tinybus/src/connection\.rs$|crates/tinybus/src/test_support\.rs$' --fail-under-file-lines 90 diff --git a/scripts/test-e2e.ps1 b/scripts/test-e2e.ps1 index bbd3f966..8dfb9467 100644 --- a/scripts/test-e2e.ps1 +++ b/scripts/test-e2e.ps1 @@ -1,4 +1,7 @@ $ErrorActionPreference = 'Stop' +$targetDirectory = (cargo metadata --format-version 1 --no-deps | ConvertFrom-Json).target_directory +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +$examples = Join-Path $targetDirectory debug/examples $private = Join-Path $PWD target/private-module-tests New-Item -ItemType Directory -Force $private | Out-Null $identity = [System.Security.Principal.WindowsIdentity]::GetCurrent() @@ -15,15 +18,16 @@ $rule = [System.Security.AccessControl.FileSystemAccessRule]::new( ) $acl.AddAccessRule($rule) Set-Acl -LiteralPath $private -AclObject $acl -Copy-Item target/debug/examples/module_clock.dll $private -Copy-Item target/debug/examples/module_context.dll $private -Copy-Item target/debug/examples/module_context_eager.dll $private -Copy-Item target/debug/examples/module_host_callback.dll $private -Copy-Item target/debug/examples/module_controlled_init.dll $private -Copy-Item target/debug/examples/module_wrong_target.dll $private +Copy-Item (Join-Path $examples module_clock.dll) $private +Copy-Item (Join-Path $examples module_context.dll) $private +Copy-Item (Join-Path $examples module_context_eager.dll) $private +Copy-Item (Join-Path $examples module_host_callback.dll) $private +Copy-Item (Join-Path $examples module_controlled_init.dll) $private +Copy-Item (Join-Path $examples module_terminal.dll) $private +Copy-Item (Join-Path $examples module_wrong_target.dll) $private # Admit every intentional fixture by its actual built bytes so later # checks, including the wrong-target gate, can exercise their contract. -$fixtureAllowlist = @('module_clock.dll', 'module_context.dll', 'module_context_eager.dll', 'module_host_callback.dll', 'module_controlled_init.dll', 'module_wrong_target.dll') | ForEach-Object { +$fixtureAllowlist = @('module_clock.dll', 'module_context.dll', 'module_context_eager.dll', 'module_host_callback.dll', 'module_controlled_init.dll', 'module_terminal.dll', 'module_wrong_target.dll') | ForEach-Object { $digest = (Get-FileHash -LiteralPath (Join-Path $private $_) -Algorithm SHA256).Hash.ToLowerInvariant() '"' + $_ + '" = "' + $digest + '"' } @@ -31,14 +35,14 @@ $fixtureAllowlist | Set-Content -LiteralPath (Join-Path $private modules.toml) $clockTwoDirectory = Join-Path $private module-clock-two New-Item -ItemType Directory -Force $clockTwoDirectory | Out-Null Set-Acl -LiteralPath $clockTwoDirectory -AclObject $acl -Copy-Item target/debug/examples/module_clock_two.dll $clockTwoDirectory +Copy-Item (Join-Path $examples module_clock_two.dll) $clockTwoDirectory $clockTwoHash = (Get-FileHash -LiteralPath (Join-Path $clockTwoDirectory module_clock_two.dll) -Algorithm SHA256).Hash.ToLowerInvariant() '"module_clock_two.dll" = "' + $clockTwoHash + '"' | Set-Content -LiteralPath (Join-Path $clockTwoDirectory modules.toml) -NoNewline $mixed = Join-Path $private mixed New-Item -ItemType Directory -Force $mixed | Out-Null Set-Acl -LiteralPath $mixed -AclObject $acl -Copy-Item target/debug/examples/module_clock.dll $mixed -Copy-Item target/debug/examples/module_wrong_target.dll $mixed +Copy-Item (Join-Path $examples module_clock.dll) $mixed +Copy-Item (Join-Path $examples module_wrong_target.dll) $mixed $module = (Get-ChildItem "$private/module_clock.dll" | Select-Object -First 1).FullName $moduleTwo = (Get-ChildItem "$clockTwoDirectory/module_clock_two.dll" | Select-Object -First 1).FullName $wrongTarget = (Get-ChildItem "$private/module_wrong_target.dll" | Select-Object -First 1).FullName @@ -47,13 +51,23 @@ $env:TMP = $private $env:TINYBUS_TEST_CONTEXT_MODULE = (Get-Item "$private/module_context.dll").FullName $env:TINYBUS_TEST_CONTEXT_EAGER_MODULE = (Get-Item "$private/module_context_eager.dll").FullName $env:TINYBUS_TEST_CONTROLLED_INIT_MODULE = (Get-Item "$private/module_controlled_init.dll").FullName +$env:TINYBUS_TEST_TERMINAL_MODULE = (Get-Item "$private/module_terminal.dll").FullName $env:TINYBUS_TEST_HOST_CALLBACK_MODULE = (Get-Item "$private/module_host_callback.dll").FullName $env:TINYBUS_TEST_MODULE = $module $env:TINYBUS_TEST_MODULE_TWO = $moduleTwo $env:TINYBUS_TEST_WRONG_TARGET = $wrongTarget $env:TINYBUS_TEST_MODULE_DIRECTORY = $mixed -cargo test --locked --no-default-features --features modules,macros,test-support -- --ignored +cargo test --locked --no-default-features --workspace --features modules,macros,test-support,__native-fixture-support -- --ignored --skip terminal_runtime_tests::compiled_native_ +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +# Retained fixture owners require a fresh process for each SDK scenario. +$testList = cargo test --locked -p tinybus-module --features __native-fixture-support,tinybus/macros -- --list --ignored if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +foreach ($line in $testList) { + if ($line -match '^(terminal_runtime_tests::compiled_native_.*): test$') { + cargo test --locked -p tinybus-module --features __native-fixture-support,tinybus/macros $Matches[1] -- --ignored --exact + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } +} # Directory admission against the real ACLs: a bundle Everyone can # write falls back to the verified release cache. cargo test --locked --no-default-features --features modules,macros admission_tests -- --include-ignored diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh index 72579373..8973c7c5 100755 --- a/scripts/test-e2e.sh +++ b/scripts/test-e2e.sh @@ -1,20 +1,22 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." -cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --no-default-features --features modules,macros +cargo build --locked --example module_clock --example module_clock_two --example module_wrong_target --example module_context --example module_context_eager --example module_host_callback --example module_controlled_init --example module_terminal --no-default-features --features modules,macros,__native-fixture-support cargo test --locked --no-default-features --features modules,macros,test-support test_support::tests case "${RUNNER_OS:-$(uname -s)}" in Windows|MINGW*|MSYS*|CYGWIN*) pwsh -NoProfile -File scripts/test-e2e.ps1 ;; *) set -euo pipefail -module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_clock.so' -o -name 'libmodule_clock.dylib' \) -print -quit) -module_two=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_clock_two.so' -o -name 'libmodule_clock_two.dylib' \) -print -quit) -wrong_target=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_wrong_target.so' -o -name 'libmodule_wrong_target.dylib' \) -print -quit) -context_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_context.so' -o -name 'libmodule_context.dylib' \) -print -quit) -context_eager_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_context_eager.so' -o -name 'libmodule_context_eager.dylib' \) -print -quit) -host_callback_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) -controlled_init_module=$(find target/debug/examples -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) -for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module"; do +target_dir=$(cargo metadata --format-version 1 --no-deps | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])') +module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_clock.so' -o -name 'libmodule_clock.dylib' \) -print -quit) +module_two=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_clock_two.so' -o -name 'libmodule_clock_two.dylib' \) -print -quit) +wrong_target=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_wrong_target.so' -o -name 'libmodule_wrong_target.dylib' \) -print -quit) +context_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_context.so' -o -name 'libmodule_context.dylib' \) -print -quit) +context_eager_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_context_eager.so' -o -name 'libmodule_context_eager.dylib' \) -print -quit) +host_callback_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_host_callback.so' -o -name 'libmodule_host_callback.dylib' \) -print -quit) +controlled_init_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_controlled_init.so' -o -name 'libmodule_controlled_init.dylib' \) -print -quit) +terminal_module=$(find "$target_dir/debug/examples" -maxdepth 1 -type f \( -name 'libmodule_terminal.so' -o -name 'libmodule_terminal.dylib' \) -print -quit) +for artifact in "$module" "$module_two" "$wrong_target" "$context_module" "$context_eager_module" "$host_callback_module" "$controlled_init_module" "$terminal_module"; do if [ -z "$artifact" ] || [ ! -f "$artifact" ]; then echo "a module example artifact is missing; the loader tests would not exercise it" >&2 exit 1 @@ -31,7 +33,28 @@ else digest="$(shasum -a 256 "$private/$library" | awk '{print $1}')" fi printf '"%s" = "%s"\n' "$library" "$digest" > "$private/modules.toml" -TINYBUS_TEST_MODULE="$PWD/$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$PWD/$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$PWD/$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$PWD/$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$PWD/$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$PWD/$controlled_init_module" cargo test --locked --no-default-features --features modules,macros,test-support -- --ignored +# Terminal fixtures can retain uncertain owners; pin a private copy and run +# each SDK scenario in a fresh process so its retained state cannot leak. +terminal_private="$PWD/target/private-module-tests/module-terminal" +mkdir -p "$terminal_private" +chmod 700 "$terminal_private" +cp "$terminal_module" "$terminal_private/" +terminal_library="$(basename "$terminal_module")" +if command -v sha256sum >/dev/null 2>&1; then + terminal_digest="$(sha256sum "$terminal_private/$terminal_library" | awk '{print $1}')" +else + terminal_digest="$(shasum -a 256 "$terminal_private/$terminal_library" | awk '{print $1}')" +fi +printf '"%s" = "%s"\n' "$terminal_library" "$terminal_digest" > "$terminal_private/modules.toml" +export TINYBUS_TEST_TERMINAL_MODULE="$terminal_private/$terminal_library" +export TINYBUS_TEST_MODULE="$module" TINYBUS_TEST_MODULE_TWO="$private/$library" TINYBUS_TEST_WRONG_TARGET="$wrong_target" TINYBUS_TEST_CONTEXT_MODULE="$context_module" TINYBUS_TEST_CONTEXT_EAGER_MODULE="$context_eager_module" TINYBUS_TEST_HOST_CALLBACK_MODULE="$host_callback_module" TINYBUS_TEST_CONTROLLED_INIT_MODULE="$controlled_init_module" +cargo test --locked --no-default-features --workspace --features modules,macros,test-support,__native-fixture-support -- --ignored --skip terminal_runtime_tests::compiled_native_ + +terminal_tests=$(cargo test --locked -p tinybus-module --features __native-fixture-support,tinybus/macros -- --list --ignored | sed -n 's/^\(terminal_runtime_tests::compiled_native_.*\): test$/\1/p') +[ -n "$terminal_tests" ] || { echo "no compiled terminal scenarios found" >&2; exit 1; } +while IFS= read -r test; do + cargo test --locked -p tinybus-module --features __native-fixture-support,tinybus/macros "$test" -- --ignored --exact +done <<< "$terminal_tests" cargo test --locked --no-default-features --features modules,macros admission_tests -- --include-ignored TINYBUS_TEST_UMASK=1 cargo test --locked --no-default-features --features modules,macros permissive_umask -- --test-threads=1