diff --git a/changelog.d/8596-transitive-leaf.md b/changelog.d/8596-transitive-leaf.md new file mode 100644 index 0000000000..12a749db21 --- /dev/null +++ b/changelog.d/8596-transitive-leaf.md @@ -0,0 +1 @@ +perf(gc): reduce native-root safepoint density with a whole-module Perry-GC effect closure (#8596). Direct calls to generated functions are now marked `gc-leaf-function` when every transitive path stays within proven non-collecting runtime helpers and generated callees; pure recursive SCCs are supported. Allocation/poll paths, indirect calls, unknown externals, and cross-module calls remain statepoints. The proof and annotations are shared by whole-module text emission, split codegen units, and native LLVM construction (including `invoke` inside `try`). Shadow-frame bookkeeping helpers are also classified non-collecting; their only allocation is the raw Rust shadow-buffer `Vec`, which cannot trigger Perry GC. This is the sound polling-style reduction available with LLVM statepoints: a caller edge whose callee may collect must retain its relocation map so moving GC can rewrite the suspended caller frame. diff --git a/crates/perry-codegen/src/dialect/eh.rs b/crates/perry-codegen/src/dialect/eh.rs index 04f28549fa..11b04067fe 100644 --- a/crates/perry-codegen/src/dialect/eh.rs +++ b/crates/perry-codegen/src/dialect/eh.rs @@ -52,6 +52,7 @@ impl<'ctx, 'm> FnReader<'ctx, 'm> { let callee = &after[..paren]; let close = rmatch_paren(after, paren)?; let args_str = &after[paren + 1..close]; + let trailing_attr = after[close + 1..].trim(); // `build_indirect_invoke` takes basic values (not metadata enums // like the call path), so collect both shapes once. @@ -85,6 +86,19 @@ impl<'ctx, 'm> FnReader<'ctx, 'm> { if preserve_none { site.set_call_convention(super::LLVM_CC_PRESERVE_NONE); } + match trailing_attr { + "" => {} + // #8596: whole-module GC-effect closure can prove a direct + // generated callee transitively non-collecting even inside a try. + // The textual path places the call-site attribute before + // `to label`; reproduce it in the C-API path or native units gain + // statepoints the text units do not have. + "\"gc-leaf-function\"" => site.add_attribute( + inkwell::attributes::AttributeLoc::Function, + self.ctx.create_string_attribute("gc-leaf-function", ""), + ), + other => bail!("unknown invoke callsite attribute `{other}`"), + } // An invoke terminates its block; the emitted text continues in // the inline continuation label, which arrives as the next line. match site.try_as_basic_value() { diff --git a/crates/perry-codegen/src/dialect/tests.rs b/crates/perry-codegen/src/dialect/tests.rs index 36121796a5..9905c46288 100644 --- a/crates/perry-codegen/src/dialect/tests.rs +++ b/crates/perry-codegen/src/dialect/tests.rs @@ -381,3 +381,39 @@ fn preserve_none_constructs_on_define_call_and_invoke() { "invoke site lost its calling convention:\n{printed}" ); } + +/// #8596: a transitive-leaf direct call inside `try` is an invoke, and LLVM's +/// call-site attribute sits between the argument list and `to label`. The +/// split-module native reader must carry it onto the CallBase or RS4GC silently +/// restores a statepoint that the text path removed. +#[test] +fn gc_leaf_attribute_constructs_on_invoke() { + let ctx = Context::create(); + let skeleton = "declare void @pure()\n\ + declare i32 @perry_eh_personality(i32, i32, i64, ptr, ptr)\n"; + let module = crate::inprocess::parse_ir_text(&ctx, skeleton, "leaf_invoke_skel") + .expect("skeleton parses"); + let function = "define void @trycaller() personality ptr @perry_eh_personality {\n\ + entry:\n\ + \x20 invoke void @pure() \"gc-leaf-function\" to label %ok unwind label %pad\n\ + ok:\n\ + \x20 ret void\n\ + pad:\n\ + \x20 %lp = landingpad { ptr, i32 } catch ptr null\n\ + \x20 ret void\n\ + }\n"; + predeclare_function_from_text(&ctx, &module, function).expect("predeclare"); + add_function_from_text(&ctx, &module, function).unwrap_or_else(|e| panic!("{e:#}")); + module + .verify() + .unwrap_or_else(|e| panic!("verifier rejected native module:\n{}", e.to_string())); + let printed = module.print_to_string().to_string(); + let invoke = printed + .lines() + .find(|line| line.contains("invoke void @pure")) + .unwrap_or_else(|| panic!("no invoke in constructed module:\n{printed}")); + assert!( + invoke.contains("#0") && printed.contains("attributes #0 = { \"gc-leaf-function\" }"), + "invoke lost its gc-leaf-function call-site attribute:\n{printed}" + ); +} diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 3807c36395..32fc22e7a2 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -895,6 +895,14 @@ impl LlFunction { } pub fn to_ir(&self) -> String { + self.to_ir_with_gc_leaf_callees(&HashSet::new()) + } + + /// Render with the module's transitive Perry-GC leaf closure available at + /// direct call sites. Standalone function tests use [`Self::to_ir`] and an + /// empty set; module and codegen-unit renderers compute the whole-module + /// fixed point before serializing any function. + pub(crate) fn to_ir_with_gc_leaf_callees(&self, gc_leaf_callees: &HashSet) -> String { let mut ir = self.define_header(false); ir.push('\n'); self.for_each_final_line::(&mut |line| { @@ -921,6 +929,18 @@ impl LlFunction { ir }; + // #8596: LLVM needs a statepoint at a caller edge exactly when the + // transitive callee can reach collection. The whole-module analysis + // proves direct generated callees that cannot; stamp those edges after + // root lowering (which separately handles audited runtime helpers). + // Unknown, indirect, cross-module and collecting callees remain + // unmarked and therefore remain statepoints. + let ir = if self.stack_map_requested && !gc_leaf_callees.is_empty() { + crate::gc_call_effects::annotate_transitive_leaf_calls(&ir, gc_leaf_callees) + } else { + ir + }; + // RS4GC uses the unwind destination's landing pad **as the token** for // the relocates it inserts on the exceptional edge, so // `statepoint-example` requires that pad to be `landingpad token`. diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index 3c4a89c81f..ac65c3f397 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -9,6 +9,11 @@ //! auditing the complete runtime call graph for `gc_check_trigger`, //! `js_gc_collect`, `js_gc_loop_safepoint`, or another route into collection. +use std::collections::{HashMap, HashSet}; + +use crate::function::{FinalItem, LlFunction}; +use crate::inst::LlInst; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum GcCallEffect { CannotCollect, @@ -55,6 +60,21 @@ pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { | "js_gc_temp_root_get" | "js_gc_temp_root_set" | "js_gc_temp_root_truncate" + // Heap-shadow-frame bookkeeping. These helpers touch only the + // thread-local shadow buffer; growth is a raw Rust Vec allocation, + // and slot writes may run the incremental-mark root barrier, neither + // of which can enter Perry's collector. Native-root functions consume + // bind/set calls before RS4GC, while #8583-spilled functions retain + // them. Classifying both forms lets the module call-graph closure prove + // an otherwise-leaf spilled callee without pretending its frame + // maintenance is a safepoint. All six are in the root-dominance + // checker's NONCOLLECTING authority. + | "js_shadow_frame_enter" + | "js_shadow_frame_push" + | "js_shadow_frame_pop" + | "js_shadow_state_addr" + | "js_shadow_slot_bind" + | "js_shadow_slot_set" // `gc/barrier.rs`: remembered-set / incremental-marking maintenance. | "js_write_barrier" | "js_write_barrier_slot" @@ -236,6 +256,250 @@ pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { } } +/// Whether a direct external call is a Perry-GC leaf in this compile. +/// +/// `AllocNoReentry` is deliberately conditional: without the strict +/// safepoint-only contract those helpers may collect synchronously at their +/// allocation site, so every caller frame on the stack still needs a +/// statepoint at the edge that reached them. +fn external_callee_cannot_collect(name: &str) -> bool { + name.starts_with("llvm.") + || match classify_direct_callee(name) { + GcCallEffect::CannotCollect => true, + GcCallEffect::AllocNoReentry => { + crate::codegen::helpers::gc_safepoint_only_contract_enabled() + } + GcCallEffect::Unknown => false, + } +} + +/// The direct callee token and its argument-list opening parenthesis. +/// +/// Perry's closed IR dialect emits unquoted `[-A-Za-z0-9_.$]` symbols. Search +/// for the first `%name(`/`@name(` token after the call opcode rather than the +/// first `(`: return types such as `ptr addrspace(1)` contain parentheses too. +/// Choosing the first sigil also fails closed for an indirect call whose +/// arguments later contain a direct-function constant. +fn direct_callee_span(line: &str) -> Option<(&str, usize)> { + let trimmed = line.trim_start(); + let leading = line.len() - trimmed.len(); + // Prefer invoke before searching for `call`: a later argument or inline + // constant may contain those bytes, but it is never the opcode of an + // invoke line. + let opcode = if let Some(rest) = trimmed.strip_prefix("invoke ") { + trimmed.len() - rest.len() + } else if let Some(pos) = trimmed.find(" = invoke ") { + pos + " = invoke ".len() + } else if let Some(pos) = trimmed.find("call ") { + pos + "call ".len() + } else { + return None; + }; + let tail = &trimmed[opcode..]; + let bytes = tail.as_bytes(); + let mut i = 0usize; + while i < bytes.len() { + if matches!(bytes[i], b'@' | b'%') { + let sigil = bytes[i]; + let start = i + 1; + let mut end = start; + while end < bytes.len() + && (bytes[end].is_ascii_alphanumeric() + || matches!(bytes[end], b'_' | b'.' | b'$' | b'-')) + { + end += 1; + } + if end > start && bytes.get(end) == Some(&b'(') { + if sigil == b'%' { + return None; + } + return Some((&tail[start..end], leading + opcode + end)); + } + i = end.max(i + 1); + } else { + i += 1; + } + } + None +} + +fn line_is_call_like(line: &str) -> bool { + let t = line.trim_start(); + t.starts_with("call ") + || t.starts_with("tail call ") + || t.starts_with("musttail call ") + || t.starts_with("notail call ") + || t.contains(" = call ") + || t.contains(" = tail call ") + || t.contains(" = musttail call ") + || t.contains(" = notail call ") + || t.starts_with("invoke ") + || t.contains(" = invoke ") +} + +fn matching_call_paren(line: &str, open: usize) -> Option { + let mut depth = 0usize; + let mut quoted = false; + let mut escaped = false; + for (offset, ch) in line[open..].char_indices() { + if quoted { + if escaped { + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if ch == '"' { + quoted = false; + } + continue; + } + match ch { + '"' => quoted = true, + '(' => depth += 1, + ')' => { + depth = depth.checked_sub(1)?; + if depth == 0 { + return Some(open + offset); + } + } + _ => {} + } + } + None +} + +/// Add LLVM's call-site leaf marker to direct calls of `known_leaf_callees`. +/// +/// This runs after Perry's native-root lowering, which already annotates the +/// audited runtime-helper table. It handles both `call` and `invoke`; for an +/// invoke the attribute belongs between `@callee(args)` and `to label`. +pub(crate) fn annotate_transitive_leaf_calls( + ir: &str, + known_leaf_callees: &HashSet, +) -> String { + if known_leaf_callees.is_empty() { + return ir.to_string(); + } + let mut out = String::with_capacity(ir.len()); + for line in ir.lines() { + let rewritten = (line_is_call_like(line) && !line.contains(" asm ")) + .then(|| direct_callee_span(line)) + .flatten() + .and_then(|(callee, open)| { + if !known_leaf_callees.contains(callee) || line.contains("\"gc-leaf-function\"") { + return None; + } + let close = matching_call_paren(line, open)?; + let mut marked = String::with_capacity(line.len() + 19); + marked.push_str(&line[..=close]); + marked.push_str(" \"gc-leaf-function\""); + marked.push_str(&line[close + 1..]); + Some(marked) + }); + out.push_str(rewritten.as_deref().unwrap_or(line)); + out.push('\n'); + } + out +} + +#[derive(Default)] +struct FunctionEffects { + internal_callees: HashSet, + has_collecting_edge: bool, +} + +fn note_direct_callee(effects: &mut FunctionEffects, callee: &str, defined: &HashSet<&str>) { + if defined.contains(callee) { + effects.internal_callees.insert(callee.to_string()); + } else if !external_callee_cannot_collect(callee) { + effects.has_collecting_edge = true; + } +} + +fn note_text_effects(line: &str, effects: &mut FunctionEffects, defined: &HashSet<&str>) { + if !line_is_call_like(line) || line.contains("\"gc-leaf-function\"") || line.contains(" asm ") { + return; + } + match direct_callee_span(line) { + Some((callee, _)) => note_direct_callee(effects, callee, defined), + None => effects.has_collecting_edge = true, + } +} + +fn effects_of(function: &LlFunction, defined: &HashSet<&str>) -> FunctionEffects { + let mut effects = FunctionEffects::default(); + function + .for_each_final_item::(&mut |item| { + match item { + FinalItem::Inst(LlInst::Call { callee, .. }) => { + note_direct_callee(&mut effects, callee, defined) + } + FinalItem::Inst(LlInst::CallIndirect { .. }) => effects.has_collecting_edge = true, + FinalItem::Inst(LlInst::AsmBarrier) => {} + FinalItem::Inst(LlInst::Raw(line)) => { + note_text_effects(line, &mut effects, defined) + } + FinalItem::Text(line) => note_text_effects(line, &mut effects, defined), + FinalItem::Label(_) | FinalItem::Blank | FinalItem::Inst(_) => {} + } + Ok(()) + }) + .unwrap_or_else(|e| match e {}); + effects +} + +/// Compute the largest sound set of module-defined functions that cannot +/// reach Perry's collector. +/// +/// Start with every definition as a candidate and remove functions with an +/// unknown/indirect/collecting external edge, then propagate removal backwards +/// through direct calls. This greatest-fixed-point formulation admits pure +/// recursive SCCs while rejecting an SCC as soon as any member can allocate, +/// poll, throw through an allocating helper, call indirectly, or leave the +/// module through an unaudited symbol. +/// +/// A caller suspended below a collecting callee still needs a statepoint even +/// when collection begins only at an allocation or loop poll: the moving +/// collector must find and rewrite that caller's frame. Consequently this set +/// is the safe part of polling-style density reduction; calls outside it must +/// remain statepoints. +pub(crate) fn transitive_leaf_functions(functions: &[&LlFunction]) -> HashSet { + let defined: HashSet<&str> = functions.iter().map(|f| f.name.as_str()).collect(); + let effects: HashMap<&str, FunctionEffects> = functions + .iter() + .map(|f| (f.name.as_str(), effects_of(f, &defined))) + .collect(); + + let mut collecting: HashSet<&str> = effects + .iter() + .filter_map(|(&name, effect)| effect.has_collecting_edge.then_some(name)) + .collect(); + // Reverse edges make propagation O(functions + calls). Re-scanning every + // function once per newly-unsafe layer is quadratic on a long generated + // call chain -- exactly the scale this optimization is meant to help. + let mut callers: HashMap<&str, Vec<&str>> = HashMap::new(); + for (&caller, effect) in &effects { + for callee in &effect.internal_callees { + callers.entry(callee.as_str()).or_default().push(caller); + } + } + let mut work: Vec<&str> = collecting.iter().copied().collect(); + while let Some(callee) = work.pop() { + if let Some(direct_callers) = callers.get(callee) { + for &caller in direct_callers { + if collecting.insert(caller) { + work.push(caller); + } + } + } + } + + defined + .into_iter() + .filter(|name| !collecting.contains(name)) + .map(str::to_string) + .collect() +} + #[cfg(test)] mod tests { use super::*; @@ -436,6 +700,12 @@ mod tests { fn audited_runtime_bookkeeping_cannot_collect() { for name in [ "js_gc_temp_root_push", + "js_shadow_frame_enter", + "js_shadow_frame_push", + "js_shadow_frame_pop", + "js_shadow_state_addr", + "js_shadow_slot_bind", + "js_shadow_slot_set", "js_write_barrier_root_nanbox", "js_gc_note_slot_layout", "js_typed_feedback_record_guard_pass", @@ -600,4 +870,159 @@ mod tests { ); } } + + fn void_function(name: &str, calls: &[&str]) -> LlFunction { + let mut f = LlFunction::new(name, crate::types::VOID, vec![]); + let entry = f.create_block("entry"); + for callee in calls { + entry.call_void(callee, &[]); + } + entry.ret_void(); + f + } + + /// #8596: the greatest fixed point admits a pure recursive component, but + /// one collecting exit poisons every direct caller that can reach it. + #[test] + fn transitive_leaf_closure_handles_recursion_and_collecting_exits() { + let leaf = void_function("leaf", &["js_nanbox_pointer"]); + let wrapper = void_function("wrapper", &["leaf"]); + let recursive_a = void_function("recursive_a", &["recursive_b"]); + let recursive_b = void_function("recursive_b", &["recursive_a"]); + let allocating = void_function("allocating", &["js_array_alloc"]); + let reaches_allocating = void_function("reaches_allocating", &["allocating"]); + let functions = [ + &leaf, + &wrapper, + &recursive_a, + &recursive_b, + &allocating, + &reaches_allocating, + ]; + + let safe = transitive_leaf_functions(&functions); + for name in ["leaf", "wrapper", "recursive_a", "recursive_b"] { + assert!(safe.contains(name), "{name} should be transitively leaf"); + } + for name in ["allocating", "reaches_allocating"] { + assert!( + !safe.contains(name), + "{name} reaches Perry allocation and must remain a safepoint callee" + ); + } + } + + #[test] + fn indirect_and_unknown_external_edges_fail_closed() { + let unknown = void_function("unknown", &["cross_module_function"]); + let mut indirect = LlFunction::new("indirect", crate::types::VOID, vec![]); + let entry = indirect.create_block("entry"); + entry.call_indirect(crate::types::I64, "%callback", &[]); + entry.ret_void(); + let functions = [&unknown, &indirect]; + + let safe = transitive_leaf_functions(&functions); + assert!( + safe.is_empty(), + "unknown and indirect calls must fail closed" + ); + } + + #[test] + fn annotates_call_and_invoke_at_the_llvm_attribute_position() { + let known = HashSet::from(["pure".to_string()]); + let ir = " %a = call ptr addrspace(1) @pure(ptr addrspace(1) %p)\n\ + %b = invoke preserve_nonecc double @pure(double %x) to label %ok unwind label %pad\n\ + %c = call double @collecting()\n\ + %d = call double %callback(ptr @pure)\n\ + ; call void @pure() is documentation, not an instruction\n"; + let marked = annotate_transitive_leaf_calls(ir, &known); + assert!(marked.contains( + "%a = call ptr addrspace(1) @pure(ptr addrspace(1) %p) \"gc-leaf-function\"" + )); + assert!(marked.contains( + "%b = invoke preserve_nonecc double @pure(double %x) \"gc-leaf-function\" to label %ok unwind label %pad" + )); + assert!(marked.contains("%c = call double @collecting()\n")); + assert!(marked.contains("%d = call double %callback(ptr @pure)\n")); + assert_eq!( + marked.matches("\"gc-leaf-function\"").count(), + 2, + "only the two proven direct calls may be annotated:\n{marked}" + ); + } + + /// End-to-end emission witness: the analysis is module-wide and the leaf + /// set reaches a rooted caller's final IR. The allocating sibling is the + /// discriminating control and must remain unmarked for RS4GC to rewrite. + #[test] + fn module_marks_only_transitively_noncollecting_generated_calls() { + let _native = crate::codegen::helpers::NativeRootsPin::native(); + let mut module = crate::module::LlModule::new(crate::codegen::default_target_triple()); + module.declare_function("js_array_alloc", crate::types::I64, &[crate::types::I32]); + module.declare_function( + "js_shadow_slot_bind", + crate::types::VOID, + &[crate::types::I32, crate::types::PTR], + ); + + let pure = module.define_function("pure_generated", crate::types::VOID, vec![]); + pure.create_block("entry").ret_void(); + + let allocating = module.define_function("allocating_generated", crate::types::VOID, vec![]); + let entry = allocating.create_block("entry"); + entry.call( + crate::types::I64, + "js_array_alloc", + &[(crate::types::I32, "0")], + ); + entry.ret_void(); + + let caller = module.define_function("rooted_caller", crate::types::VOID, vec![]); + caller.enable_shadow_frame(0); + let slot = caller.reserve_shadow_slot().expect("reserve native root"); + let root = caller.alloca_entry(crate::types::I64); + caller.entry_allocas_push_store(crate::types::I64, "0", &root); + caller.entry_setup_call_void( + "js_shadow_slot_bind", + &[ + (crate::types::I32, &slot.to_string()), + (crate::types::PTR, &root), + ], + ); + let entry = caller.create_block("entry"); + entry.call_void("pure_generated", &[]); + entry.call_void("allocating_generated", &[]); + entry.ret_void(); + + let ir = module.to_ir(); + assert!(ir.contains("call void @pure_generated() \"gc-leaf-function\"")); + assert!( + ir.contains("call void @allocating_generated()") + && !ir.contains("call void @allocating_generated() \"gc-leaf-function\""), + "allocating generated callee must remain a statepoint edge:\n{ir}" + ); + + #[cfg(feature = "llvm-inprocess")] + { + let target = crate::codegen::default_target_triple(); + let rewritten = crate::inprocess::statepoint_rewritten_ir( + &ir, + &target, + "transitive_leaf_generated_calls", + ) + .expect("module-wide leaf witness must survive RS4GC"); + assert!( + rewritten.contains("call void @pure_generated()"), + "proven leaf call was unexpectedly rewritten:\n{rewritten}" + ); + assert!( + rewritten.lines().any(|line| { + line.contains("@llvm.experimental.gc.statepoint") + && line.contains("@allocating_generated") + }), + "collecting generated call did not become a statepoint:\n{rewritten}" + ); + } + } } diff --git a/crates/perry-codegen/src/module.rs b/crates/perry-codegen/src/module.rs index c004993874..dc81409128 100644 --- a/crates/perry-codegen/src/module.rs +++ b/crates/perry-codegen/src/module.rs @@ -13,6 +13,7 @@ use std::cell::RefCell; use std::collections::{BTreeMap, HashMap, HashSet}; use std::rc::Rc; +use std::sync::Arc; use crate::block::FpFlags; use crate::function::LlFunction; @@ -299,7 +300,14 @@ pub(crate) fn declare_line_for(f: &LlFunction) -> String { /// `private` definition so cross-unit calls can bind to it. Names are /// module-prefixed and unique, so promotion never collides. pub(crate) fn render_fn_external(f: &LlFunction) -> String { - let ir = f.to_ir(); + render_fn_external_with_gc_leaf_callees(f, &HashSet::new()) +} + +pub(crate) fn render_fn_external_with_gc_leaf_callees( + f: &LlFunction, + gc_leaf_callees: &HashSet, +) -> String { + let ir = f.to_ir_with_gc_leaf_callees(gc_leaf_callees); if f.linkage == "internal" || f.linkage == "private" { return ir.replacen(&format!("define {} ", f.linkage), "define ", 1); } @@ -789,6 +797,11 @@ impl LlModule { ir.push('\n'); let funcs = self.deduped_function_refs(); + let gc_leaf_callees = if crate::codegen::helpers::native_stack_roots_enabled() { + crate::gc_call_effects::transitive_leaf_functions(&funcs) + } else { + HashSet::new() + }; // Skip any `declare` whose name is also `define`d in this module — // LLVM rejects declare+define for the same symbol. @@ -806,7 +819,7 @@ impl LlModule { ir.push('\n'); for func in &funcs { - ir.push_str(&func.to_ir()); + ir.push_str(&func.to_ir_with_gc_leaf_callees(&gc_leaf_callees)); ir.push('\n'); } @@ -917,11 +930,17 @@ impl LlModule { /// into one object, keeping `compile_module`'s single-object API. pub(crate) fn codegen_unit_parts(&self, n: usize) -> Vec> { let funcs = self.deduped_function_refs(); + let gc_leaf_callees = Arc::new(if crate::codegen::helpers::native_stack_roots_enabled() { + crate::gc_call_effects::transitive_leaf_functions(&funcs) + } else { + HashSet::new() + }); if n <= 1 || funcs.len() <= 1 { return vec![CodegenUnitPart { pre: String::new(), post: String::new(), funcs, + gc_leaf_callees, }]; } let n = n.min(funcs.len()); @@ -1155,6 +1174,7 @@ impl LlModule { pre, post: unit_posts[bi].clone(), funcs: bucket, + gc_leaf_callees: Arc::clone(&gc_leaf_callees), }); } parts @@ -1167,7 +1187,7 @@ impl LlModule { /// function graph as soon as its immutable worker payload exists instead /// of retaining the whole `LlModule` until every LLVM unit has finished. pub(crate) fn into_codegen_unit_parts(mut self, n: usize) -> Vec { - let layouts: Vec<(String, String, Vec)> = self + let layouts: Vec<(String, String, Vec, Arc>)> = self .codegen_unit_parts(n) .into_iter() .map(|part| { @@ -1175,6 +1195,7 @@ impl LlModule { part.pre, part.post, part.funcs.iter().map(|func| func.name.clone()).collect(), + part.gc_leaf_callees, ) }) .collect(); @@ -1187,7 +1208,7 @@ impl LlModule { layouts .into_iter() - .map(|(pre, post, names)| OwnedCodegenUnitPart { + .map(|(pre, post, names, gc_leaf_callees)| OwnedCodegenUnitPart { pre, post, funcs: names @@ -1198,6 +1219,7 @@ impl LlModule { .expect("borrowed codegen partition named an owned function") }) .collect(), + gc_leaf_callees, }) .collect() } @@ -1215,7 +1237,10 @@ impl LlModule { .map(|part| { let mut ir = part.pre; for func in &part.funcs { - ir.push_str(&render_fn_external(func)); + ir.push_str(&render_fn_external_with_gc_leaf_callees( + func, + &part.gc_leaf_callees, + )); ir.push('\n'); } ir.push_str(&part.post); @@ -1233,12 +1258,14 @@ pub(crate) struct CodegenUnitPart<'m> { pub pre: String, pub post: String, pub funcs: Vec<&'m LlFunction>, + pub gc_leaf_callees: Arc>, } pub(crate) struct OwnedCodegenUnitPart { pub pre: String, pub post: String, pub funcs: Vec, + pub gc_leaf_callees: Arc>, } #[cfg(test)] diff --git a/crates/perry-codegen/src/native_emit.rs b/crates/perry-codegen/src/native_emit.rs index 2010ee7090..cd9b52113d 100644 --- a/crates/perry-codegen/src/native_emit.rs +++ b/crates/perry-codegen/src/native_emit.rs @@ -89,7 +89,9 @@ fn build_native_module<'ctx>(context: &'ctx Context, llmod: &LlModule) -> Result skeleton.push_str(&format!("declare {} @{}({})\n", f.return_type, f.name, tys)); } let module = crate::inprocess::parse_ir_text(context, &skeleton, "perry_native_module")?; - let (typed_insts, raw_insts) = stream_functions(context, &module, &funcs, false)?; + let gc_leaf_callees = crate::gc_call_effects::transitive_leaf_functions(&funcs); + let (typed_insts, raw_insts) = + stream_functions(context, &module, &funcs, false, &gc_leaf_callees)?; log::debug!( "perry-codegen: native construction built {} functions, {} typed + {} raw instructions \ (ratchet: raw -> 0), skeleton {} bytes", @@ -108,6 +110,7 @@ fn stream_functions<'ctx>( module: &Module<'ctx>, funcs: &[&crate::function::LlFunction], force_external: bool, + gc_leaf_callees: &std::collections::HashSet, ) -> Result<(usize, usize)> { let mut typed_insts = 0usize; let mut raw_insts = 0usize; @@ -123,7 +126,7 @@ fn stream_functions<'ctx>( // This remains native construction: only one finalized function // is materialized and fed through the closed dialect line reader, // never parsed as module-scale IR. - let fn_text = f.to_ir(); + let fn_text = f.to_ir_with_gc_leaf_callees(gc_leaf_callees); for line in fn_text.lines().skip(1) { stream.line(line).map_err(|e| { anyhow!( @@ -170,7 +173,12 @@ fn freeze_unit( part: crate::module::OwnedCodegenUnitPart, external_declarations: &[(String, String)], ) -> Result { - let crate::module::OwnedCodegenUnitPart { pre, post, funcs } = part; + let crate::module::OwnedCodegenUnitPart { + pre, + post, + funcs, + gc_leaf_callees, + } = part; let mut skeleton = format!("{pre}{post}"); // Text units minimize declarations with a rendered-reference scan. Typed // instructions can name helpers without passing through that textual scan @@ -201,7 +209,10 @@ fn freeze_unit( // no inkwell builders. Let LLVM's in-process assembly parser build // only these exceptional functions; all ordinary bodies remain on // the typed C-API path and never become text. - skeleton.push_str(&crate::module::render_fn_external(&f)); + skeleton.push_str(&crate::module::render_fn_external_with_gc_leaf_callees( + &f, + &gc_leaf_callees, + )); skeleton.push('\n'); continue; } @@ -213,7 +224,7 @@ fn freeze_unit( // owned lines so worker threads still receive an immutable payload // and the module-scale text graph is never retained. items.extend( - f.to_ir() + f.to_ir_with_gc_leaf_callees(&gc_leaf_callees) .lines() .skip(1) .filter(|line| *line != "}") @@ -882,6 +893,80 @@ mod tests { ); } + /// #8596: whole-module generated-callee effects must reach both emission + /// transports. The text path spells the string attribute inline; LLVM's + /// C API prints it through an attribute group. RS4GC is the final arbiter: + /// both forms must leave `pure_generated` direct and wrap `may_collect`. + #[test] + fn transitive_generated_leaf_calls_match_text_and_native_construction() { + let _native = crate::codegen::helpers::NativeRootsPin::native(); + let mut module = LlModule::new(crate::codegen::default_target_triple()); + module.declare_function("js_shadow_slot_bind", VOID, &[I32, PTR]); + module.declare_function("may_collect", VOID, &[]); + + let pure = module.define_function("pure_generated", VOID, vec![]); + pure.create_block("entry").ret_void(); + + let caller = module.define_function("rooted_leaf_caller", VOID, vec![]); + caller.enable_shadow_frame(0); + let slot = caller.reserve_shadow_slot().expect("reserve native root"); + let root = caller.alloca_entry(I64); + caller.entry_allocas_push_store(I64, "0", &root); + caller.entry_setup_call_void( + "js_shadow_slot_bind", + &[(I32, &slot.to_string()), (PTR, &root)], + ); + let entry = caller.create_block("entry"); + entry.call_void("pure_generated", &[]); + entry.call_void("may_collect", &[]); + entry.ret_void(); + + let text_ir = module.to_ir(); + let context = Context::create(); + let native_ir = build_native_module(&context, &module) + .expect("native transitive-leaf witness constructs") + .print_to_string() + .to_string(); + assert!( + text_ir.contains("call void @pure_generated() \"gc-leaf-function\""), + "text path lost transitive leaf marker:\n{text_ir}" + ); + assert!( + native_ir.contains("\"gc-leaf-function\""), + "native path lost transitive leaf marker:\n{native_ir}" + ); + let units = module.render_codegen_units(2); + assert_eq!(units.len(), 2, "fixture must split into two real units"); + assert!( + units + .iter() + .any(|unit| unit.contains("call void @pure_generated() \"gc-leaf-function\"")), + "split text units lost the whole-module leaf closure:\n{}", + units.join("\n--- unit ---\n") + ); + + let target = crate::codegen::default_target_triple(); + for (arm, ir) in [("text", text_ir), ("native", native_ir)] { + let rewritten = crate::inprocess::statepoint_rewritten_ir( + &ir, + &target, + &format!("transitive_leaf_{arm}"), + ) + .unwrap_or_else(|e| panic!("{arm} transitive-leaf witness failed RS4GC: {e:#}")); + assert!( + rewritten.contains("call void @pure_generated()"), + "{arm} path statepointed a proven leaf call:\n{rewritten}" + ); + assert!( + rewritten.lines().any(|line| { + line.contains("@llvm.experimental.gc.statepoint") + && line.contains("@may_collect") + }), + "{arm} path failed to statepoint the collecting control:\n{rewritten}" + ); + } + } + fn compact_gc_map_section_name() -> &'static [u8] { if cfg!(target_os = "macos") { b"__perry_gcmap" diff --git a/crates/perry-codegen/src/native_root_coverage/mechanics.rs b/crates/perry-codegen/src/native_root_coverage/mechanics.rs index b54d8e6034..24de4b8f76 100644 --- a/crates/perry-codegen/src/native_root_coverage/mechanics.rs +++ b/crates/perry-codegen/src/native_root_coverage/mechanics.rs @@ -336,7 +336,15 @@ fn no_entry_module_root_is_live_before_the_gc_is_initialized() { vec![ let_stmt(1, "a", Expr::MapNew), let_stmt(2, "b", Expr::MapNew), - console_log(vec![Expr::LocalGet(1), Expr::LocalGet(2)]), + // Keep the string initializer non-empty: #8596 can prove an + // empty initializer transitively leaf, in which case it is an + // ordinary call rather than a statepoint and this test would + // stop observing the boundary whose ordering it verifies. + console_log(vec![ + Expr::LocalGet(1), + Expr::LocalGet(2), + Expr::String("root-order-control".to_string()), + ]), ], ); let ir = native_ir(&module, target, true); diff --git a/docs/statepoint-gc-experiment.md b/docs/statepoint-gc-experiment.md index b91f39225f..43cb2d6e8e 100644 --- a/docs/statepoint-gc-experiment.md +++ b/docs/statepoint-gc-experiment.md @@ -1053,3 +1053,30 @@ Closing the last axis therefore needs the root set to shrink, not the encoding: 221 KB of map for 154k roots is already near this format's floor. That is the repsel-promotion lever the earlier projection named, and it is still the outstanding work. + +## Safepoint density and the caller-frame constraint (2026-08-24) + +A polling design cannot soundly mark every ordinary call as +`gc-leaf-function` under LLVM statepoints. If `A` calls `B`, and `B` reaches an +allocation or loop poll that starts moving collection, `A` is suspended at its +call to `B`. The collector must find and rewrite `A`'s live managed values at +that return PC. Omitting the statepoint on `A -> B` would remove exactly that +caller-frame relocation map; putting a poll only inside `B` does not recreate +it. VM poll points reduce where collection may begin, but every active caller +edge beneath such a poll still needs an oop/relocation map. + +Perry therefore applies the maximal local reduction that preserves this +constraint: compute a whole-module, greatest-fixed-point GC-effect closure and +mark a direct generated call leaf only when its callee cannot transitively +reach collection. The proof admits mutually recursive pure components. It +fails closed on any allocation or poll helper, indirect call, unknown external, +or cross-module call, and propagates that result back through callers. Runtime +helpers remain governed by the audited `GcCallEffect` table. + +The closure is computed before codegen-unit partitioning and carried into every +unit, so a safe direct edge remains leaf even when caller and callee are emitted +into different objects. Textual and native LLVM construction consume the same +set; the native dialect also preserves the marker on `invoke` edges inside +`try`. Calls outside the proven set remain ordinary RS4GC safepoints. Reducing +those further requires a different frame representation (for example, spilling +caller roots to a shadow frame), not merely moving the collection trigger.