diff --git a/changelog.d/8586-rs4gc-budget-assert.md b/changelog.d/8586-rs4gc-budget-assert.md new file mode 100644 index 0000000000..60f98a6a92 --- /dev/null +++ b/changelog.d/8586-rs4gc-budget-assert.md @@ -0,0 +1,5 @@ +### Changed + +- `PERRY_LL_PREOPT_OPTNONE_INSTRS` is removed (#8583). It stamped `optnone` before `rewrite-statepoints-for-gc`, which makes the pass manager skip `mem2reg`/`sccp` while RS4GC still runs, so a demoted function's root allocas were never promoted and the collector never saw them. The cap defaulted to 0, so no shipped build was affected; a test now pins that an `optnone` function loses every root under the rewrite. +- `PERRY_LL_RS4GC_MAX_INSTRS` (default 1.5 Mi): after `rewrite-statepoints-for-gc`, a function whose body exceeds the per-function budget fails its codegen unit with the function's name and its sizes before and after the rewrite, instead of entering an optimizer pipeline that is super-linear on statepoint relocation fan-out and would not finish. This is an assertion, not a fallback — no function is ever demoted and the requested optimization level applies to every function. `` raises it, `warn:` only warns, `0` disables. Both caches key on it. +- `PERRY_CODEGEN_UNIT_TIMINGS` now reports, per codegen unit, the widest function by estimated IR before LLVM starts, and after compile the instruction totals and widest function before and after RS4GC, the growth factor, and rewrite/optimize/emit times. diff --git a/changelog.d/8587-root-spill.md b/changelog.d/8587-root-spill.md new file mode 100644 index 0000000000..9ce5caba4a --- /dev/null +++ b/changelog.d/8587-root-spill.md @@ -0,0 +1,3 @@ +### Changed + +- Native GC-root spilling (#8583): a function whose estimated statepoint relocation count — `live_root_slots × safepoints` — exceeds `PERRY_ROOT_SPILL_RELOCATIONS` (default 4,000,000) keeps its GC roots in a heap shadow frame instead of native statepoints. `rewrite-statepoints-for-gc` adds one relocation per live root per safepoint, so a minified-bundle entry function (measured: 795 root slots × ~106k safepoints, grown 439k → 6.5M instructions under RS4GC) drove the `-Os` middle-end super-linear and did not finish; the same unit optimizes in ~5s once that one function is spilled. The function is still compiled at the requested optimization level — only its root representation changes — and its roots stay precise: the runtime already scans shadow-frame and stack-map roots in one walk, and the frame pointer is kept so the FP-chain walker steps over the spilled frame. Each spilled function is reported at default verbosity. `PERRY_ROOT_SPILL_RELOCATIONS=0` disables spilling (every function on native statepoints, the previous behavior). diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index d5e6a1543d..05fbfb3f43 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -612,6 +612,12 @@ pub(super) fn compile_closure( // which spans the body. let capture_root_slots = u32::from(captures_this || enclosing_class.is_some()) + u32::from(captures_new_target); + crate::codegen::helpers::maybe_spill_roots_to_shadow_frame( + lf, + &llvm_name, + m.len() + capture_root_slots as usize, + body, + ); lf.enable_shadow_frame(m.len() as u32 + capture_root_slots); m } else { diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 313675d96e..a0a2b91e54 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -554,6 +554,12 @@ pub(super) fn compile_function( cross_module.flat_const_arrays.keys().copied().collect(); let m = crate::collectors::collect_pointer_typed_locals(&f.params, &f.body, &flat_const_ids); + crate::codegen::helpers::maybe_spill_roots_to_shadow_frame( + lf, + &llvm_name, + m.len(), + &f.body, + ); lf.enable_shadow_frame(m.len() as u32); m } else { diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index 0f54dcb522..88ccc1f869 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -357,6 +357,74 @@ pub(crate) fn inline_hot_small_max_call_sites() -> u32 { }) } +/// #8583: statepoint relocation estimate above which a function keeps its GC +/// roots in a shadow frame instead of native statepoints. +/// +/// `rewrite-statepoints-for-gc` adds one relocation per GC value live across +/// each safepoint, so the optimizer's post-rewrite cost scales with +/// `live_roots × safepoints`. Past a point that fan-out makes the `-Os`/`-O3` +/// middle-end super-linear and the compile does not finish (the Claude Code +/// bundle's 68 MB entry body measured 795 root slots × ~106k safepoints ≈ 8.4e7 +/// and grew 439k → 6.5M instructions under RS4GC; without RS4GC the same unit +/// optimized at `-Os` in ~5s). Real functions sit orders of magnitude below +/// this: hundreds of call sites times tens of slots is ~1e4–1e5. The default +/// is set well under the measured pathological point and well over ordinary +/// code, and the post-RS4GC instruction-budget assertion (#8583, inprocess.rs) +/// backstops any function the estimate misses. +/// +/// `PERRY_ROOT_SPILL_RELOCATIONS=` overrides it; `0` disables spilling +/// (every function stays on native statepoints, the pre-#8583 behavior). +const DEFAULT_ROOT_SPILL_RELOCATIONS: usize = 4_000_000; + +fn root_spill_relocation_threshold() -> usize { + std::env::var("PERRY_ROOT_SPILL_RELOCATIONS") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(DEFAULT_ROOT_SPILL_RELOCATIONS) +} + +/// The relocation estimate for a function with `slot_count` GC-root slots and +/// a body containing `safepoint_sites` call-like expressions. Saturating so a +/// pathological product cannot wrap. +pub(crate) fn root_relocation_estimate(slot_count: usize, safepoint_sites: usize) -> usize { + slot_count.saturating_mul(safepoint_sites) +} + +/// Decide whether `func` should spill its roots to the shadow frame, and if so +/// mark it (BEFORE its `enable_*_shadow_frame` call) and report it. Only +/// meaningful under native stack-map roots — the shadow frame is already the +/// lowering otherwise. Reporting is at default verbosity because #8421 requires +/// that a change to how a function is compiled is never silent; the message +/// states that the optimization level is unchanged. +pub(super) fn maybe_spill_roots_to_shadow_frame( + func: &mut crate::function::LlFunction, + fn_name: &str, + slot_count: usize, + body: &[perry_hir::Stmt], +) { + if !native_stack_roots_enabled() { + return; + } + let threshold = root_spill_relocation_threshold(); + if threshold == 0 { + return; + } + let sites = crate::collectors::count_safepoint_sites(body); + let estimate = root_relocation_estimate(slot_count, sites); + if estimate <= threshold { + return; + } + func.request_shadow_frame_spill(); + eprintln!( + "perry: `{fn_name}` keeps its {slot_count} GC roots in a shadow frame instead of \ + statepoints: an estimated {estimate} relocations ({slot_count} roots × {sites} \ + safepoints) would make rewrite-statepoints-for-gc fan-out super-linear in the \ + optimizer (> {threshold}). The function is still compiled at the requested \ + optimization level; only its GC-root representation changes, and its roots stay \ + precise (#8583). Override with PERRY_ROOT_SPILL_RELOCATIONS." + ); +} + pub(super) fn enable_module_init_shadow_frame( func: &mut crate::function::LlFunction, stmts: &[perry_hir::Stmt], @@ -368,6 +436,10 @@ pub(super) fn enable_module_init_shadow_frame( let shadow_slot_map = crate::collectors::collect_pointer_typed_locals(&[], stmts, flat_const_ids); + // #8583: the module-entry body is the minified-bundle IIFE — the function + // that fans out catastrophically under RS4GC. Decide its root lowering + // before the frame is built. + maybe_spill_roots_to_shadow_frame(func, "main", shadow_slot_map.len(), stmts); func.enable_post_init_shadow_frame(shadow_slot_map.len() as u32); let shadow_slot_clears_after_stmt = crate::collectors::collect_shadow_slot_clear_points(stmts, &shadow_slot_map); diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index c1363c5bba..58b9d790c5 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -319,6 +319,12 @@ pub(super) fn compile_method( &method.body, &flat_const_ids, ); + crate::codegen::helpers::maybe_spill_roots_to_shadow_frame( + lf, + &llvm_name, + m.len() + 1, + &method.body, + ); lf.enable_shadow_frame(m.len() as u32 + 1); m } else { @@ -1392,6 +1398,12 @@ pub(super) fn compile_static_method( cross_module.flat_const_arrays.keys().copied().collect(); let m = crate::collectors::collect_pointer_typed_locals(&f.params, &f.body, &flat_const_ids); + crate::codegen::helpers::maybe_spill_roots_to_shadow_frame( + lf, + &llvm_name, + m.len() + 1, + &f.body, + ); lf.enable_shadow_frame(m.len() as u32 + 1); m } else { diff --git a/crates/perry-codegen/src/collectors/mod.rs b/crates/perry-codegen/src/collectors/mod.rs index 23a2ce34f2..010c4a513f 100644 --- a/crates/perry-codegen/src/collectors/mod.rs +++ b/crates/perry-codegen/src/collectors/mod.rs @@ -42,6 +42,7 @@ mod ptr_shape_report; mod ptr_shape_returns; mod refs; mod repsel_benefit; +mod safepoint_sites; mod scalar_method_dispatch; mod scalar_methods; mod shadow_slots; @@ -93,6 +94,7 @@ pub(crate) use ptr_shape_returns::collect_exported_return_shapes; pub(crate) use refs::{ collect_let_ids, collect_ref_ids_in_expr, collect_ref_ids_in_stmts, is_clamp_call, }; +pub(crate) use safepoint_sites::count_safepoint_sites; pub(crate) use scalar_method_dispatch::{ collect_module_dispatch_facts, mark_unstable_scalar_method_receivers, ModuleDispatchFacts, }; diff --git a/crates/perry-codegen/src/collectors/safepoint_sites.rs b/crates/perry-codegen/src/collectors/safepoint_sites.rs new file mode 100644 index 0000000000..c87cd77e46 --- /dev/null +++ b/crates/perry-codegen/src/collectors/safepoint_sites.rs @@ -0,0 +1,219 @@ +//! Count the GC safepoints in a function body (#8583). +//! +//! `rewrite-statepoints-for-gc` inserts, at every safepoint, one relocation +//! per GC value live across it — so the optimizer's post-rewrite work grows +//! with `live_roots × safepoints`. A function whose product is large enough +//! makes the `-Os`/`-O3` middle-end super-linear: the 68 MB minified entry +//! body of the Claude Code bundle measured 795 root slots × ~106k safepoints +//! and grew 439k → 6.5M instructions under RS4GC, and a single `-Os` pass on +//! the result did not finish in practical time (#8583). +//! `codegen/helpers::maybe_spill_roots_to_shadow_frame` multiplies this count +//! by the function's root-slot count and, past a threshold, keeps that +//! function's roots in a shadow frame instead of statepoints. +//! +//! A safepoint is any call-like expression: a call can re-enter the runtime +//! and collect. The count is an over-approximation biased toward spilling — +//! a false positive is a shadow frame on a function that would have been fine +//! (cheap; the shadow lowering is the pre-#7370 default), while a false +//! negative would let relocation fan-out reach the optimizer. Nested closures +//! are NOT counted: each compiles to its own `LlFunction` with its own frame, +//! so its safepoints belong to it (`walk_expr_children` does not descend into +//! a closure's body, only its parameter defaults). + +use perry_hir::{Expr, Stmt}; + +/// Total call-like expressions reachable from `stmts` without descending into +/// nested closures. +pub fn count_safepoint_sites(stmts: &[Stmt]) -> usize { + let mut n = 0usize; + for s in stmts { + count_in_stmt(s, &mut n); + } + n +} + +/// A call-like expression is a potential safepoint: anything whose lowering +/// emits a call that can re-enter the runtime. Nodes not listed contribute +/// nothing themselves but are still recursed into, so adding a new call +/// variant can only make the estimate more conservative (a possible +/// under-count that the post-RS4GC instruction-budget assertion backstops), +/// never wrong in a way that hides a fan-out. +fn is_safepoint(e: &Expr) -> bool { + matches!( + e, + Expr::Call { .. } + | Expr::CallSpread { .. } + | Expr::NativeMethodCall { .. } + | Expr::StaticMethodCall { .. } + | Expr::SuperCall(_) + | Expr::SuperCallSpread(_) + | Expr::SuperMethodCall { .. } + | Expr::SuperMethodCallSpread { .. } + | Expr::ObjectSuperMethodCall { .. } + | Expr::New { .. } + | Expr::NewDynamic { .. } + | Expr::NewDynamicSpread { .. } + | Expr::Await(_) + | Expr::Yield { .. } + | Expr::AsyncFirstCall { .. } + ) +} + +fn count_in_expr(e: &Expr, n: &mut usize) { + if is_safepoint(e) { + *n += 1; + } + // Generic recursion into direct sub-expressions. `walk_expr_children` does + // not descend into a closure's statement body (only its param defaults), + // which is exactly the boundary we want: a nested closure is a separate + // frame and its safepoints are not this function's. + perry_hir::walker::walk_expr_children(e, &mut |child| count_in_expr(child, n)); +} + +fn count_in_stmt(s: &Stmt, n: &mut usize) { + match s { + Stmt::Let { init: Some(e), .. } + | Stmt::Expr(e) + | Stmt::Throw(e) + | Stmt::Return(Some(e)) => count_in_expr(e, n), + Stmt::Let { init: None, .. } | Stmt::Return(None) => {} + Stmt::If { + condition, + then_branch, + else_branch, + } => { + count_in_expr(condition, n); + for st in then_branch { + count_in_stmt(st, n); + } + if let Some(else_branch) = else_branch { + for st in else_branch { + count_in_stmt(st, n); + } + } + } + Stmt::While { condition, body } | Stmt::DoWhile { body, condition } => { + count_in_expr(condition, n); + for st in body { + count_in_stmt(st, n); + } + } + Stmt::For { + init, + condition, + update, + body, + } => { + if let Some(init) = init { + count_in_stmt(init, n); + } + if let Some(condition) = condition { + count_in_expr(condition, n); + } + if let Some(update) = update { + count_in_expr(update, n); + } + for st in body { + count_in_stmt(st, n); + } + } + Stmt::Labeled { body, .. } => count_in_stmt(body, n), + Stmt::Try { + body, + catch, + finally, + } => { + for st in body { + count_in_stmt(st, n); + } + if let Some(catch) = catch { + for st in &catch.body { + count_in_stmt(st, n); + } + } + if let Some(finally) = finally { + for st in finally { + count_in_stmt(st, n); + } + } + } + Stmt::Switch { + discriminant, + cases, + } => { + count_in_expr(discriminant, n); + for c in cases { + if let Some(t) = &c.test { + count_in_expr(t, n); + } + for st in &c.body { + count_in_stmt(st, n); + } + } + } + // No expression children. + Stmt::Break + | Stmt::Continue + | Stmt::LabeledBreak(_) + | Stmt::LabeledContinue(_) + | Stmt::PreallocateBoxes(_) + | Stmt::PreallocateTdzBoxes(_) + | Stmt::ReleaseBoxes(_) => {} + } +} + +#[cfg(test)] +mod tests { + use super::count_safepoint_sites; + use perry_hir::types::Type; + use perry_hir::{Expr, Stmt}; + + fn call(args: Vec) -> Expr { + Expr::Call { + callee: Box::new(Expr::Undefined), + args, + type_args: vec![], + byte_offset: 0, + } + } + + fn empty_closure(body: Vec) -> Expr { + Expr::Closure { + func_id: 0, + params: vec![], + return_type: Type::Any, + body, + captures: vec![], + mutable_captures: vec![], + captures_this: false, + captures_new_target: false, + enclosing_class: None, + is_arrow: false, + is_async: false, + is_generator: false, + is_strict: false, + } + } + + #[test] + fn counts_calls_across_control_flow_but_not_into_closures() { + let stmts = vec![ + Stmt::Expr(call(vec![])), + Stmt::While { + condition: Expr::Bool(true), + body: vec![Stmt::Expr(call(vec![]))], + }, + // A call buried in a nested closure body must NOT be counted. + Stmt::Expr(empty_closure(vec![Stmt::Expr(call(vec![]))])), + Stmt::Return(Some(call(vec![]))), + ]; + assert_eq!(count_safepoint_sites(&stmts), 3); + } + + #[test] + fn call_arguments_are_themselves_safepoints() { + // f(g(), h()) is three calls. + let nested = call(vec![call(vec![]), call(vec![])]); + assert_eq!(count_safepoint_sites(&[Stmt::Expr(nested)]), 3); + } +} diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 6759cd63bc..f8ff3f4f4e 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -152,6 +152,18 @@ pub struct LlFunction { /// final IR pass resolves these indices to the native allocas named by /// `js_shadow_slot_bind` calls, removes the calls, and emits stack maps. stack_map_slot_count: u32, + /// #8583: force this function onto the heap-backed shadow frame even when + /// native stack-map roots are the build default. Set for a function whose + /// estimated statepoint relocation count (`live_roots × safepoints`) would + /// make `rewrite-statepoints-for-gc` fan-out super-linear in the optimizer + /// (`codegen/helpers::maybe_spill_roots_to_shadow_frame`). The shadow-frame + /// lowering is the pre-#7370 default, walked by the same runtime root scan + /// as stack maps, so a spilled function's roots stay precise — it simply + /// carries no `gc "statepoint-example"` strategy and RS4GC skips it. The + /// frame pointer is unaffected (kept for every native-roots build), so the + /// FP-chain walker steps over the spilled frame exactly as it does the + /// runtime's own. + force_shadow_frame: bool, /// Runtime hooks emitted immediately before each non-pointer `ret`. /// Entry/module-init functions use this for process-level diagnostics /// that must run regardless of which block reaches the normal epilogue. @@ -257,6 +269,7 @@ impl LlFunction { shadow_frame_slot_count: 0, stack_map_requested: false, stack_map_slot_count: 0, + force_shadow_frame: false, pre_return_void_calls: Vec::new(), } } @@ -282,6 +295,19 @@ impl LlFunction { /// into every caller's hot loop. Skip the frame entirely; the /// to_ir() rewrite pass keys off `shadow_frame_slot.is_some()`, /// so no matching pop is emitted either. + /// #8583: route this function's precise roots through the heap shadow + /// frame instead of native statepoints. Must be called BEFORE + /// `enable_shadow_frame` / `enable_post_init_shadow_frame` so the frame is + /// built in shadow form. No effect once a frame has been emitted. + pub fn request_shadow_frame_spill(&mut self) { + self.force_shadow_frame = true; + } + + /// Whether this function spills its roots to the shadow frame (#8583). + pub fn spills_roots_to_shadow_frame(&self) -> bool { + self.force_shadow_frame + } + pub fn enable_shadow_frame(&mut self, slot_count: u32) { self.enable_shadow_frame_inner(slot_count, false); } @@ -297,7 +323,7 @@ impl LlFunction { } fn enable_shadow_frame_inner(&mut self, slot_count: u32, post_init: bool) { - if crate::codegen::helpers::native_stack_roots_enabled() { + if crate::codegen::helpers::native_stack_roots_enabled() && !self.force_shadow_frame { self.shadow_frame_requested = true; self.shadow_frame_post_init_region = post_init; self.stack_map_requested = slot_count != 0; @@ -391,7 +417,7 @@ impl LlFunction { if !self.shadow_frame_requested { return None; } - if crate::codegen::helpers::native_stack_roots_enabled() { + if crate::codegen::helpers::native_stack_roots_enabled() && !self.force_shadow_frame { let idx = self.stack_map_slot_count; self.stack_map_slot_count += 1; self.stack_map_requested = true; @@ -826,12 +852,19 @@ impl LlFunction { // #7174: the `!has_try` exclusion is gone with the field. Try/catch no // longer lowers to setjmp/longjmp (#7302), so nothing can jump past a // `gc.relocate` any more and statepoints cover every function. - let gc_strategy = - if self.stack_map_requested && crate::codegen::helpers::native_stack_roots_enabled() { - " gc \"statepoint-example\"" - } else { - "" - }; + // A spilled function (#8583) keeps precise roots in the shadow frame, + // so it must NOT carry the statepoint strategy — RS4GC would then run + // on it and reintroduce the relocation fan-out the spill avoids. Its + // `stack_map_requested` is already false (enable_shadow_frame_inner + // took the shadow branch), so this is belt-and-braces. + let gc_strategy = if self.stack_map_requested + && !self.force_shadow_frame + && crate::codegen::helpers::native_stack_roots_enabled() + { + " gc \"statepoint-example\"" + } else { + "" + }; // Invoke-EH (#7302): functions containing landing/funclet pads name // their personality on the define line. LLVM's grammar orders these // `[fn attrs] [gc] [personality]`, so the strategy precedes it. @@ -1201,6 +1234,92 @@ mod define_header_tests { } } + /// #8583 root spilling: under the native-roots build, a function that + /// requested a shadow-frame spill BEFORE `enable_shadow_frame` must take + /// the heap shadow lowering — no `gc "statepoint-example"` strategy (so + /// RS4GC skips it and cannot fan out its relocations) — while a sibling + /// that did not request the spill keeps native statepoints. The frame + /// pointer is kept regardless, so the FP-chain root walker still steps + /// over the spilled frame. + #[test] + fn a_spilled_function_takes_the_shadow_lowering_while_its_sibling_keeps_statepoints() { + use crate::codegen::helpers::NativeRootsPin; + use crate::types::{I64, PTR}; + const STRATEGY: &str = "gc \"statepoint-example\""; + const FRAME_PTR: &str = "\"frame-pointer\"=\"non-leaf\""; + + // Build a function with one bound root across a call: enough for both + // lowerings to have real content to render. + fn rooted(spill: bool) -> LlFunction { + let mut f = LlFunction::new("perry_fn_probe", crate::types::VOID, vec![]); + if spill { + f.request_shadow_frame_spill(); + } + f.enable_shadow_frame(0); + let idx = f.reserve_shadow_slot().expect("a frame yields a root slot"); + let root = f.alloca_entry(I64); + f.entry_allocas_push_store(I64, "0", &root); + f.entry_setup_call_void( + "js_shadow_slot_bind", + &[(crate::types::I32, &idx.to_string()), (PTR, &root)], + ); + let entry = f.create_block("entry"); + let _ = entry.call(I64, "may_collect", &[]); + entry.ret_void(); + f + } + + let _native = NativeRootsPin::native(); + + let native = rooted(false); + assert_eq!( + native.stack_map_slot_count, 1, + "the un-spilled sibling must take the stack-map path" + ); + let native_hdr = native.define_header(false); + assert!( + native_hdr.contains(STRATEGY), + "the un-spilled sibling must carry the statepoint strategy:\n{native_hdr}" + ); + + let spilled = rooted(true); + assert!(spilled.spills_roots_to_shadow_frame()); + assert_eq!( + spilled.stack_map_slot_count, 0, + "a spilled function must NOT take the stack-map path even under the \ + native-roots pin — that is the whole point of the spill" + ); + let spilled_hdr = spilled.define_header(false); + assert!( + !spilled_hdr.contains(STRATEGY), + "a spilled function must NOT claim the statepoint strategy, or RS4GC \ + would run on it and reintroduce the relocation fan-out:\n{spilled_hdr}" + ); + assert!( + spilled_hdr.contains(FRAME_PTR) && native_hdr.contains(FRAME_PTR), + "both lowerings keep the non-leaf frame pointer so the FP-chain \ + walker can step over the frame:\nspilled: {spilled_hdr}\nnative: {native_hdr}" + ); + + // The spilled function builds the heap shadow frame (`js_shadow_frame_enter` + // + retained `js_shadow_slot_bind`); the native sibling has neither — its + // roots are stack-map slots that RS4GC lowers, and the bind calls are + // stripped. + let spilled_ir = spilled.to_ir(); + assert!( + spilled_ir.contains("@js_shadow_frame_enter") + && spilled_ir.contains("@js_shadow_slot_bind"), + "a spilled function keeps the runtime shadow frame:\n{spilled_ir}" + ); + let native_ir = native.to_ir(); + assert!( + !native_ir.contains("@js_shadow_frame_enter") + && !native_ir.contains("@js_shadow_slot_bind"), + "the native sibling has no heap shadow frame; its roots are stack-map \ + slots and its binds are lowered away:\n{native_ir}" + ); + } + /// `force_external` drops only the linkage keyword. The codegen-unit path /// depends on that and on nothing else changing. #[test] diff --git a/crates/perry-codegen/src/inprocess.rs b/crates/perry-codegen/src/inprocess.rs index 972a48eb42..7e5ee5ce47 100644 --- a/crates/perry-codegen/src/inprocess.rs +++ b/crates/perry-codegen/src/inprocess.rs @@ -21,7 +21,6 @@ use std::ffi::CString; use std::sync::Once; use anyhow::{anyhow, Result}; -use inkwell::attributes::{Attribute, AttributeLoc}; use inkwell::context::Context; use inkwell::memory_buffer::MemoryBuffer; use inkwell::passes::PassBuilderOptions; @@ -205,6 +204,7 @@ pub fn compile_ll_to_object_inprocess( &mllvm, emit_asm, native_roots, + None, ) } @@ -314,6 +314,24 @@ pub(crate) fn optimize_and_emit_module( effective_target: &str, clang_style_args: &[String], native_roots: bool, +) -> Result> { + optimize_and_emit_module_with_stats( + module, + effective_target, + clang_style_args, + native_roots, + None, + ) +} + +/// [`optimize_and_emit_module`] that also fills `stats` (sizes before and +/// after RS4GC, widest functions, phase times) for the per-unit report. +pub(crate) fn optimize_and_emit_module_with_stats( + module: &inkwell::module::Module<'_>, + effective_target: &str, + clang_style_args: &[String], + native_roots: bool, + stats: Option<&mut UnitCodegenStats>, ) -> Result> { let (opt, mcpu_native, explicit_cpu, mllvm, emit_asm) = interpret_plan_args(clang_style_args)?; optimize_and_emit( @@ -325,92 +343,191 @@ pub(crate) fn optimize_and_emit_module( &mllvm, emit_asm, native_roots, + stats, ) } -/// Optional pre-optimization escape hatch for unusually large generated -/// functions. -/// -/// Dense generated bundles often contain one parser/table initializer that is -/// large enough to make the `-O1+` middle-end super-linear, alongside hundreds -/// of ordinary functions that benefit substantially from `-Os`. Routing the -/// whole codegen unit to `-O0` keeps compilation bounded but also bloats every -/// ordinary sibling. When this cap is non-zero, only functions above it are -/// stamped `optnone`+`noinline` before the module pipeline runs. This makes -/// `PERRY_LL_SIZE_OPT=1` a practical hybrid mode instead of an all-or-nothing -/// gamble on the largest function in each unit. -/// -/// Disabled by default while the threshold is calibrated across the bundle -/// corpus. `PERRY_LL_PREOPT_OPTNONE_INSTRS=N` enables it; `0` disables it. -const DEFAULT_PREOPT_OPTNONE_INSTRS: usize = 0; - -fn preopt_optnone_instr_cap() -> usize { - std::env::var("PERRY_LL_PREOPT_OPTNONE_INSTRS") - .ok() - .and_then(|s| s.trim().parse::().ok()) - .unwrap_or(DEFAULT_PREOPT_OPTNONE_INSTRS) -} - -fn stamp_function_optnone(function: inkwell::values::FunctionValue<'_>) { - let context = function.get_type().get_context(); - let optnone_kind = Attribute::get_named_enum_kind_id("optnone"); - let noinline_kind = Attribute::get_named_enum_kind_id("noinline"); - // `alwaysinline` and `noinline` are verifier-incompatible. Generated - // functions do not normally carry it, but the opt-in must remain safe for - // imported/generated IR that does. - function.remove_enum_attribute( - AttributeLoc::Function, - Attribute::get_named_enum_kind_id("alwaysinline"), - ); - function.remove_enum_attribute( - AttributeLoc::Function, - Attribute::get_named_enum_kind_id("inlinehint"), - ); - function.add_attribute( - AttributeLoc::Function, - context.create_enum_attribute(optnone_kind, 0), - ); - function.add_attribute( - AttributeLoc::Function, - context.create_enum_attribute(noinline_kind, 0), - ); +/// Per-unit facts the backend learns while it works: instruction totals and +/// the widest function before and after `rewrite-statepoints-for-gc`, and the +/// time each phase took. `native_emit` prints one line per unit from these +/// under `PERRY_CODEGEN_UNIT_TIMINGS`, so a build that is stuck in LLVM names +/// the function it is stuck on instead of a unit number (#8583). +#[derive(Debug, Default, Clone)] +pub struct UnitCodegenStats { + pub functions: usize, + pub pre_rewrite_instructions: usize, + pub pre_rewrite_widest: Option<(String, usize)>, + pub post_rewrite_instructions: usize, + pub post_rewrite_widest: Option<(String, usize)>, + pub rewrite_secs: f64, + pub optimize_secs: f64, + pub emit_secs: f64, } -fn function_instruction_count(function: inkwell::values::FunctionValue<'_>, cap: usize) -> usize { +fn function_instruction_count(function: inkwell::values::FunctionValue<'_>) -> usize { let mut instrs = 0usize; - 'body: for bb in function.get_basic_blocks() { + for bb in function.get_basic_blocks() { let mut inst = bb.get_first_instruction(); while let Some(i) = inst { instrs += 1; - if instrs > cap { - break 'body; - } inst = i.get_next_instruction(); } } instrs } -/// Demote large functions before the ordinary optimization pipeline while -/// leaving every smaller sibling eligible for the unit's requested opt level. -fn demote_preoptimization_bloated_functions(module: &inkwell::module::Module<'_>, cap: usize) { - if cap == 0 { - return; +/// (defined functions, total instructions, widest function) for a module. +/// One linear walk through the C API; a few milliseconds per ordinary unit. +fn module_instruction_census( + module: &inkwell::module::Module<'_>, +) -> (usize, usize, Option<(String, usize)>) { + let mut functions = 0usize; + let mut total = 0usize; + let mut widest: Option<(String, usize)> = None; + let mut function = module.get_first_function(); + while let Some(f) = function { + if f.count_basic_blocks() > 0 { + functions += 1; + let n = function_instruction_count(f); + total += n; + if widest.as_ref().is_none_or(|(_, w)| n > *w) { + widest = Some((f.get_name().to_string_lossy().into_owned(), n)); + } + } + function = f.get_next_function(); + } + (functions, total, widest) +} + +/// Instruction budget for ONE function after `rewrite-statepoints-for-gc`. +/// +/// This is an assertion about the estimate that keeps relocation fan-out out +/// of LLVM's input (#8583), not an optimization policy: a function past it is +/// refused loudly, never demoted. The #8421 contract — every function is +/// optimized at the level the plan asked for — stays intact; what this adds +/// is that an estimator miss fails in seconds with the function's name and +/// sizes instead of hanging the build for hours. +/// +/// Calibrated between the two measured points of #8128 on the Next 16.3.0 +/// production bundle: the largest post-rewrite function that finished +/// comfortably at `-Os` was ~413k instructions, and the one that ran more +/// than 65 CPU-minutes without finishing was ~2.1M. 1.5 Mi sits between them +/// with margin on both sides. `PERRY_LL_RS4GC_MAX_INSTRS=` raises or +/// lowers it, `warn:` only warns, and `0`/`off` disables the check. +const DEFAULT_RS4GC_MAX_INSTRS: usize = 1_572_864; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RewriteBudget { + Off, + Error(usize), + Warn(usize), +} + +fn parse_rewrite_budget(value: Option<&str>) -> RewriteBudget { + match value.map(str::trim) { + None | Some("") => RewriteBudget::Error(DEFAULT_RS4GC_MAX_INSTRS), + Some("0") | Some("off") | Some("false") => RewriteBudget::Off, + Some(v) => { + if let Some(n) = v.strip_prefix("warn:") { + match n.trim().parse::() { + Ok(0) => RewriteBudget::Off, + Ok(n) => RewriteBudget::Warn(n), + Err(_) => RewriteBudget::Warn(DEFAULT_RS4GC_MAX_INSTRS), + } + } else { + match v.parse::() { + Ok(n) => RewriteBudget::Error(n), + Err(_) => RewriteBudget::Error(DEFAULT_RS4GC_MAX_INSTRS), + } + } + } } +} + +fn rs4gc_instruction_budget() -> RewriteBudget { + parse_rewrite_budget(std::env::var("PERRY_LL_RS4GC_MAX_INSTRS").ok().as_deref()) +} + +/// Every defined function whose post-rewrite body exceeds `cap`. +fn rs4gc_budget_violations( + module: &inkwell::module::Module<'_>, + cap: usize, +) -> Vec<(String, usize)> { + let mut over = Vec::new(); let mut function = module.get_first_function(); while let Some(f) = function { - if function_instruction_count(f, cap) > cap { - stamp_function_optnone(f); - eprintln!( - "perry: `{}` exceeds {} pre-optimization instructions; compiling only this \ - function unoptimized (optnone) while its siblings keep the module's size \ - optimization. Override with PERRY_LL_PREOPT_OPTNONE_INSTRS.", - f.get_name().to_string_lossy(), - cap, + if f.count_basic_blocks() > 0 { + let n = function_instruction_count(f); + if n > cap { + over.push((f.get_name().to_string_lossy().into_owned(), n)); + } + } + function = f.get_next_function(); + } + over +} + +fn rewrite_budget_message(name: &str, post: usize, cap: usize, pre: Option) -> String { + let before = pre + .map(|n| format!(" (it was {n} before the rewrite)")) + .unwrap_or_default(); + format!( + "rewrite-statepoints-for-gc grew `{name}` to {post} instructions{before}; the \ + per-function budget is {cap}. LLVM's optimizer is super-linear on statepoint \ + relocation fan-out of this size and the compile would not finish in practical \ + time, so the unit is refused instead of being left to hang. Perry does not lower \ + the optimization level for it: the fix is to keep this function's GC roots out \ + of the relocation set or to split it (#8583). Override with \ + PERRY_LL_RS4GC_MAX_INSTRS= (raise), =warn: (warn only) or =0 (disable)." + ) +} + +/// Apply [`RewriteBudget`] to a rewritten module. `pre` gives each function's +/// pre-rewrite size for the message, when the caller took a census. +fn enforce_rs4gc_instruction_budget( + module: &inkwell::module::Module<'_>, + budget: RewriteBudget, + pre: &std::collections::HashMap, +) -> Result<()> { + let (cap, fatal) = match budget { + RewriteBudget::Off => return Ok(()), + RewriteBudget::Error(cap) => (cap, true), + RewriteBudget::Warn(cap) => (cap, false), + }; + let over = rs4gc_budget_violations(module, cap); + if over.is_empty() { + return Ok(()); + } + let messages: Vec = over + .iter() + .map(|(name, post)| rewrite_budget_message(name, *post, cap, pre.get(name).copied())) + .collect(); + if fatal { + return Err(anyhow!("{}", messages.join("\n"))); + } + for m in messages { + eprintln!("perry: warning: {m}"); + } + Ok(()) +} + +/// Per-function pre-rewrite sizes, for the budget message. Only the names +/// are retained, so this is a few bytes per function, not per instruction. +fn pre_rewrite_sizes( + module: &inkwell::module::Module<'_>, +) -> std::collections::HashMap { + let mut sizes = std::collections::HashMap::new(); + let mut function = module.get_first_function(); + while let Some(f) = function { + if f.count_basic_blocks() > 0 { + sizes.insert( + f.get_name().to_string_lossy().into_owned(), + function_instruction_count(f), ); } function = f.get_next_function(); } + sizes } fn optimize_and_emit( @@ -422,6 +539,7 @@ fn optimize_and_emit( mllvm: &[String], emit_asm: bool, native_roots: bool, + mut stats: Option<&mut UnitCodegenStats>, ) -> Result> { global_init(mllvm); announce(); @@ -473,12 +591,6 @@ fn optimize_and_emit( module.set_triple(&triple); module.set_data_layout(&tm.get_target_data().get_data_layout()); - // Opt-in hybrid size optimization for generated bundles: protect only the - // pathological bodies before entering the requested module pipeline. - if opt != '0' { - demote_preoptimization_bloated_functions(module, preopt_optnone_instr_cap()); - } - // RS4GC must run BEFORE the optimization pipeline, and — critically — in // this process, against this LLVM. // @@ -496,6 +608,23 @@ fn optimize_and_emit( // `try` is one — 26% of the gap suite (128 of 479 files) contains a `try`, // which the explicit bridge refuses outright (#7327/#7330). if native_roots { + // Sizes before the rewrite: the budget message below names them, and + // the per-unit report compares them with the post-rewrite census. + let budget = rs4gc_instruction_budget(); + let pre_sizes = if budget == RewriteBudget::Off && stats.is_none() { + std::collections::HashMap::new() + } else { + pre_rewrite_sizes(module) + }; + if let Some(stats) = stats.as_deref_mut() { + stats.functions = pre_sizes.len(); + stats.pre_rewrite_instructions = pre_sizes.values().sum(); + stats.pre_rewrite_widest = pre_sizes + .iter() + .max_by_key(|(_, n)| **n) + .map(|(name, n)| (name.clone(), *n)); + } + let rewrite_started = std::time::Instant::now(); module .run_passes(STATEPOINT_REWRITE_PASSES, &tm, PassBuilderOptions::create()) .map_err(|e| { @@ -518,6 +647,14 @@ fn optimize_and_emit( e.to_string() ) })?; + if let Some(stats) = stats.as_deref_mut() { + stats.rewrite_secs = rewrite_started.elapsed().as_secs_f64(); + let (_, total, widest) = module_instruction_census(module); + stats.post_rewrite_instructions = total; + stats.post_rewrite_widest = widest; + } + // The relocation-fan-out assertion (#8583): refuse, never demote. + enforce_rs4gc_instruction_budget(module, budget, &pre_sizes)?; } let pipeline = match opt { @@ -528,18 +665,26 @@ fn optimize_and_emit( 'z' => "default", _ => "default", }; + let optimize_started = std::time::Instant::now(); module .run_passes(pipeline, &tm, PassBuilderOptions::create()) .map_err(|e| anyhow!("pass pipeline `{pipeline}` failed:\n{}", e.to_string()))?; + if let Some(stats) = stats.as_deref_mut() { + stats.optimize_secs = optimize_started.elapsed().as_secs_f64(); + } let kind = if emit_asm { FileType::Assembly } else { FileType::Object }; + let emit_started = std::time::Instant::now(); let obj = tm - .write_to_memory_buffer(&module, kind) + .write_to_memory_buffer(module, kind) .map_err(|e| anyhow!("{kind:?} emission failed:\n{}", e.to_string()))?; + if let Some(stats) = stats { + stats.emit_secs = emit_started.elapsed().as_secs_f64(); + } Ok(obj.as_slice().to_vec()) } @@ -608,44 +753,123 @@ mod tests { } #[test] - fn preoptimization_bloated_function_is_demoted_without_demoting_its_sibling() { + fn rewrite_budget_spellings() { + assert_eq!( + parse_rewrite_budget(None), + RewriteBudget::Error(DEFAULT_RS4GC_MAX_INSTRS) + ); + assert_eq!(parse_rewrite_budget(Some("0")), RewriteBudget::Off); + assert_eq!(parse_rewrite_budget(Some("off")), RewriteBudget::Off); + assert_eq!( + parse_rewrite_budget(Some(" 250000 ")), + RewriteBudget::Error(250_000) + ); + assert_eq!( + parse_rewrite_budget(Some("warn:4096")), + RewriteBudget::Warn(4096) + ); + assert_eq!(parse_rewrite_budget(Some("warn:0")), RewriteBudget::Off); + // Unparsable values keep the default rather than silently disabling. + assert_eq!( + parse_rewrite_budget(Some("lots")), + RewriteBudget::Error(DEFAULT_RS4GC_MAX_INSTRS) + ); + } + + /// Six gc values live across forty safepoints: ~60 instructions before + /// `rewrite-statepoints-for-gc`, a few hundred after (each statepoint + /// relocates every live value). A budget between the two is exceeded + /// only by the post-rewrite module — which is the property the + /// assertion exists for. Counting BEFORE the rewrite (the #8421 + /// replacement knob's mistake) would make `after` empty and fail here. + fn relocation_fanout_fixture() -> String { + let mut ir = String::from( + "declare i64 @may_collect()\n\n\ + define i64 @f(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5) gc \"statepoint-example\" {\n\ + entry:\n", + ); + for i in 0..6 { + ir.push_str(&format!( + " %p{i} = inttoptr i64 %a{i} to ptr addrspace(1)\n" + )); + } + for c in 0..40 { + ir.push_str(&format!(" %c{c} = call i64 @may_collect()\n")); + } + for i in 0..6 { + ir.push_str(&format!( + " %b{i} = ptrtoint ptr addrspace(1) %p{i} to i64\n" + )); + } + ir.push_str( + " %s0 = add i64 %b0, %b1\n %s1 = add i64 %s0, %b2\n %s2 = add i64 %s1, %b3\n\ + \x20 %s3 = add i64 %s2, %b4\n %s4 = add i64 %s3, %b5\n %s5 = add i64 %s4, %c0\n\ + \x20 %s6 = add i64 %s5, %c39\n ret i64 %s6\n}\n", + ); + ir + } + + #[test] + fn rs4gc_budget_fires_only_on_the_rewritten_module() { global_init(&[]); + let target = "arm64-apple-darwin"; + let fixture = relocation_fanout_fixture(); + let rewritten = statepoint_rewritten_ir(&fixture, target, "fanout_budget") + .expect("fan-out fixture must run RS4GC"); + let context = Context::create(); - let ir = "define i64 @big(i64 %a) {\n\ - entry:\n\ - \x20 %x1 = add i64 %a, 1\n\ - \x20 %x2 = add i64 %x1, 1\n\ - \x20 %x3 = add i64 %x2, 1\n\ - \x20 %x4 = add i64 %x3, 1\n\ - \x20 %x5 = add i64 %x4, 1\n\ - \x20 ret i64 %x5\n\ - }\n\ - define i64 @small(i64 %a) {\n\ - entry:\n\ - \x20 %x1 = add i64 %a, 1\n\ - \x20 ret i64 %x1\n\ - }\n"; - let module = - parse_ir_text(&context, ir, "preopt_optnone_demotion").expect("fixture parses"); - demote_preoptimization_bloated_functions(&module, 4); - - let optnone_kind = Attribute::get_named_enum_kind_id("optnone"); - let big = module.get_function("big").expect("big exists"); - let small = module.get_function("small").expect("small exists"); + let before = parse_ir_text(&context, &fixture, "fanout_before").expect("fixture parses"); + let after = parse_ir_text(&context, &rewritten, "fanout_after").expect("rewritten parses"); + let pre = pre_rewrite_sizes(&before); + let pre_f = pre["f"]; + let (_, post_total, post_widest) = module_instruction_census(&after); + let post_f = post_widest.as_ref().map(|(_, n)| *n).unwrap_or(0); assert!( - big.get_enum_attribute(AttributeLoc::Function, optnone_kind) - .is_some(), - "a function past the pre-optimization cap must be stamped optnone" + post_f > 3 * pre_f, + "fixture must grow under relocation fan-out (pre {pre_f}, post {post_f}):\n{rewritten}" ); + assert_eq!(post_total, post_f, "one defined function"); + let cap = pre_f + (post_f - pre_f) / 2; + assert!( - small - .get_enum_attribute(AttributeLoc::Function, optnone_kind) - .is_none(), - "an ordinary sibling must keep the module optimization pipeline" + rs4gc_budget_violations(&before, cap).is_empty(), + "the pre-rewrite module is under the budget by construction" ); - module - .verify() - .expect("optnone+noinline must remain verifier-valid"); + let over = rs4gc_budget_violations(&after, cap); + assert_eq!( + over.len(), + 1, + "exactly the rewritten body is over: {over:?}" + ); + assert_eq!(over[0].0, "f"); + assert_eq!(over[0].1, post_f); + + let err = enforce_rs4gc_instruction_budget(&after, RewriteBudget::Error(cap), &pre) + .expect_err("the default spelling refuses the unit"); + let msg = format!("{err:#}"); + for needle in [ + "`f`", + &format!("to {post_f} instructions"), + &format!("it was {pre_f} before"), + &format!("budget is {cap}"), + "PERRY_LL_RS4GC_MAX_INSTRS", + "#8583", + ] { + assert!( + msg.contains(needle), + "message must carry {needle:?}:\n{msg}" + ); + } + assert!( + !msg.contains("optnone"), + "the budget is an assertion, never a demotion:\n{msg}" + ); + enforce_rs4gc_instruction_budget(&after, RewriteBudget::Warn(cap), &pre) + .expect("warn spelling does not refuse"); + enforce_rs4gc_instruction_budget(&after, RewriteBudget::Off, &pre) + .expect("off spelling does not refuse"); + enforce_rs4gc_instruction_budget(&after, RewriteBudget::Error(post_f), &pre) + .expect("a budget at the exact size is not exceeded"); } fn constant_fold_order_fixture(folded: bool) -> String { diff --git a/crates/perry-codegen/src/native_emit.rs b/crates/perry-codegen/src/native_emit.rs index 284468bcdb..fd51c8a8b1 100644 --- a/crates/perry-codegen/src/native_emit.rs +++ b/crates/perry-codegen/src/native_emit.rs @@ -291,10 +291,11 @@ pub fn compile_module_units_native( let target_triple = llmod.target_triple.clone(); let owned_module = std::mem::replace(llmod, LlModule::new(target_triple)); let parts = owned_module.into_codegen_unit_parts(n); + let unit_timings = std::env::var("PERRY_CODEGEN_UNIT_TIMINGS").is_ok(); let show_progress = matches!( std::env::var("PERRY_CODEGEN_PROGRESS").as_deref(), Ok("1" | "all") - ) || std::env::var("PERRY_CODEGEN_UNIT_TIMINGS").is_ok(); + ) || unit_timings; let unit_total = parts.len(); // Root lowering was selected while the module was produced. Preserve that // exact backend choice across the worker boundary instead of re-reading @@ -322,13 +323,39 @@ pub fn compile_module_units_native( .map_err(|e| anyhow!("unit {i}: {e:#}"))?; debug_dump(&module, &format!("{module_prefix}.unit{i}")); let (effective_target, args) = crate::linker::native_plan_args(target, native_roots); - let unit_bytes = crate::inprocess::optimize_and_emit_module( + let mut stats = crate::inprocess::UnitCodegenStats::default(); + let unit_bytes = crate::inprocess::optimize_and_emit_module_with_stats( &module, &effective_target, &args, native_roots, + unit_timings.then_some(&mut stats), ) .map_err(|e| anyhow!("unit {i}: {e:#}"))?; + if unit_timings { + let widest = |w: &Option<(String, usize)>| { + w.as_ref() + .map(|(name, n)| format!("{name} {n}")) + .unwrap_or_else(|| "-".to_string()) + }; + let growth = if stats.pre_rewrite_instructions > 0 { + stats.post_rewrite_instructions as f64 / stats.pre_rewrite_instructions as f64 + } else { + 0.0 + }; + eprintln!( + "[perry] codegen: {module_prefix}: unit {}/{unit_total}: {} fns; pre-RS4GC {} instrs (widest {}); post-RS4GC {} instrs (x{growth:.1}; widest {}); rs4gc {:.1}s, opt {:.1}s, emit {:.1}s", + i + 1, + stats.functions, + stats.pre_rewrite_instructions, + widest(&stats.pre_rewrite_widest), + stats.post_rewrite_instructions, + widest(&stats.post_rewrite_widest), + stats.rewrite_secs, + stats.optimize_secs, + stats.emit_secs, + ); + } let obj = crate::linker::finish_native_emission(unit_bytes, &effective_target, &args) .map_err(|e| anyhow!("unit {i}: {e:#}"))?; log::debug!( @@ -420,6 +447,22 @@ pub fn compile_module_units_native( // dropping that multi-gigabyte graph afterwards added a several-minute // single-threaded destructor tail on the full Claude Code bundle. for (i, part) in parts.into_iter().enumerate() { + if unit_timings { + // Name the widest body before LLVM ever sees it: the one + // irreducible function in a bundle is the one that sets the + // unit's time and memory, and a stuck unit number alone does + // not say which (#8583). + if let Some(widest) = part.funcs.iter().max_by_key(|f| f.estimated_ir_bytes()) { + eprintln!( + "[perry] codegen: {module_prefix}: unit {}/{unit_total}: {} fns, ~{:.1} MiB estimated IR, widest {} (~{:.1} MiB)", + i + 1, + part.funcs.len(), + part.funcs.iter().map(|f| f.estimated_ir_bytes()).sum::() as f64 / 1_048_576.0, + widest.name, + widest.estimated_ir_bytes() as f64 / 1_048_576.0 + ); + } + } let unit = freeze_unit(part, &external_declarations); if sender.send((i, unit)).is_err() { break; @@ -672,6 +715,46 @@ mod tests { } } + /// #8583: `optnone` stamped BEFORE `rewrite-statepoints-for-gc` is not a + /// compile-time escape hatch, it is a rooting bug. The new pass manager + /// skips `mem2reg`/`sccp` on an `optnone` function while RS4GC (a module + /// pass keyed on the `gc` attribute) still runs, so the root allocas are + /// never promoted and the collector never sees them: no `gc-live` operand + /// bundle, no relocation. This is why the pre-rewrite + /// `PERRY_LL_PREOPT_OPTNONE_INSTRS` knob was removed rather than + /// calibrated, and why any future size policy must run AFTER the rewrite. + #[test] + fn optnone_before_rs4gc_hides_every_root_from_the_collector() { + let _native = crate::codegen::helpers::NativeRootsPin::native(); + let module = precise_root_fixture(false); + let target = crate::codegen::default_target_triple(); + let text_ir = module.to_ir(); + assert!( + text_ir.contains("gc \"statepoint-example\" {"), + "fixture must carry the GC strategy:\n{text_ir}" + ); + // Control: the same fixture without optnone roots and relocates. + assert_dynamic_root_survives_rs4gc(&module, "optnone_control"); + + let demoted = text_ir.replace( + "gc \"statepoint-example\" {", + "optnone noinline gc \"statepoint-example\" {", + ); + let rewritten = + crate::inprocess::statepoint_rewritten_ir(&demoted, &target, "optnone_before_rs4gc") + .expect("optnone fixture must still run RS4GC"); + assert!( + !rewritten.contains("\"gc-live\"(ptr addrspace(1)"), + "an optnone function kept its roots visible to RS4GC, so the pre-rewrite \ + demotion would be sound after all and this test (and the knob's removal) \ + needs revisiting:\n{rewritten}" + ); + assert!( + rewritten.contains("= alloca ptr addrspace(1)"), + "the root allocas should survive unpromoted under optnone:\n{rewritten}" + ); + } + /// #8121, emission half. The sibling pair in `inprocess::tests` proves the /// LLVM mechanism (RS4GC breaks an unmarked inline-asm barrier, and /// `gc-leaf-function` stops it) using hand-written IR, so it would still diff --git a/crates/perry/src/commands/compile/build_cache.rs b/crates/perry/src/commands/compile/build_cache.rs index f185a15653..7118d0b3b7 100644 --- a/crates/perry/src/commands/compile/build_cache.rs +++ b/crates/perry/src/commands/compile/build_cache.rs @@ -39,10 +39,15 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ "PERRY_WRITE_BARRIERS", "PERRY_SHADOW_STACK", "PERRY_RS4GC", - // Explicit hybrid size mode changes both the module optimization policy - // and which unusually large functions skip the middle-end. + // `-Os` vs `-O3` for every native module. "PERRY_LL_SIZE_OPT", - "PERRY_LL_PREOPT_OPTNONE_INSTRS", + // The post-RS4GC per-function instruction budget (#8583): a unit that one + // setting refuses must not be served from a build another accepted. + "PERRY_LL_RS4GC_MAX_INSTRS", + // #8583: the relocation estimate above which a function spills its GC roots + // to a shadow frame. It changes which functions carry statepoints, so it + // changes the generated code and must be a cache input. + "PERRY_ROOT_SPILL_RELOCATIONS", "PERRY_GC_SAFEPOINT_ONLY", "PERRY_INLINE_SHADOW_SLOT", "PERRY_DISABLE_BUFFER_FAST_PATH", diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index e83860d767..450af15f70 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -841,9 +841,18 @@ fn compute_object_cache_key_with_env( "env_ll_size_opt", env_var("PERRY_LL_SIZE_OPT").as_deref().unwrap_or(""), ); + // #8583: the post-RS4GC instruction budget decides whether a unit is + // refused; two settings must never share a cached object. h.field( - "env_ll_preopt_optnone_instrs", - env_var("PERRY_LL_PREOPT_OPTNONE_INSTRS") + "env_ll_rs4gc_max_instrs", + env_var("PERRY_LL_RS4GC_MAX_INSTRS") + .as_deref() + .unwrap_or(""), + ); + // #8583: root-spill threshold changes which functions carry statepoints. + h.field( + "env_root_spill_relocations", + env_var("PERRY_ROOT_SPILL_RELOCATIONS") .as_deref() .unwrap_or(""), ); diff --git a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs index b21dc6654b..7ab7dcaa53 100644 --- a/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs +++ b/crates/perry/src/commands/compile/object_cache/object_cache_tests.rs @@ -621,7 +621,8 @@ fn key_changes_with_codegen_env_vars() { "PERRY_SHADOW_STACK", "PERRY_RS4GC", "PERRY_LL_SIZE_OPT", - "PERRY_LL_PREOPT_OPTNONE_INSTRS", + "PERRY_LL_RS4GC_MAX_INSTRS", + "PERRY_ROOT_SPILL_RELOCATIONS", "PERRY_GC_SAFEPOINT_ONLY", "PERRY_DISABLE_BUFFER_FAST_PATH", "PERRY_VERIFY_NATIVE_REGIONS", diff --git a/crates/perry/tests/gc_root_spill_mixed_frames_8583.rs b/crates/perry/tests/gc_root_spill_mixed_frames_8583.rs new file mode 100644 index 0000000000..2d1f89f349 --- /dev/null +++ b/crates/perry/tests/gc_root_spill_mixed_frames_8583.rs @@ -0,0 +1,185 @@ +//! #8583 root spilling — mixed statepoint / shadow-frame stacks are GC-correct. +//! +//! Root spilling lets one function keep its GC roots in a heap shadow frame +//! (the pre-#7370 lowering) while the rest of the program keeps native +//! statepoint roots, so a minified-bundle entry function whose relocation +//! fan-out would hang the optimizer stays compilable. The soundness question +//! it raises is new: a single call stack now carries BOTH kinds of frame, and +//! a moving minor must find and rewrite the live roots in each. Nothing on +//! `main` exercised that combination before this feature. +//! +//! This is a differential test with no node oracle. The same program is +//! compiled twice from identical source: +//! +//! * `PERRY_ROOT_SPILL_RELOCATIONS=0` — spilling disabled, every function on +//! native statepoints (the pre-#8583 lowering); +//! * `PERRY_ROOT_SPILL_RELOCATIONS=1` — spill anything with a root and a +//! call, so `run`/`make`/`main` take the shadow frame while the call-free +//! accessor `leaf` stays on statepoints — a genuinely mixed stack. +//! +//! Both binaries run under every moving-collector configuration and must +//! produce byte-identical output. If the spilled frame's roots were invisible +//! to the collector, a relocating minor would leave a stale pointer and the +//! checksum would diverge (or the run would crash) in the `=1` arm only. +//! +//! The `=1` compile is also checked to have actually spilled (its stderr names +//! the shadow-frame functions), so a future change that stops spilling turns +//! this test into a tautology loudly rather than silently. + +use std::path::PathBuf; +use std::process::Command; + +fn perry_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_perry")) +} + +/// `leaf` reads a field and makes no call: at `PERRY_ROOT_SPILL_RELOCATIONS=1` +/// its estimate is `slots × 0 = 0`, so it stays on native statepoints while its +/// callers spill. `run` holds `a`/`b`/`keep` live across allocating calls, so a +/// minor that fires inside `make` must find those roots in `run`'s shadow frame +/// and the `leaf` argument in `leaf`'s statepoint frame on the same stack. +const SOURCE: &str = r#" +function leaf(o: { v: number }): number { + return o.v; +} + +function make(i: number): { v: number } { + return { v: i }; +} + +function run(): number { + let acc = 0; + const keep: { v: number }[] = []; + for (let i = 0; i < 40000; i++) { + const a = make(i); + const b = make(i * 2); + acc = (acc + leaf(a) + leaf(b)) | 0; + if (i % 7 === 0) keep.push(a); + if (keep.length > 128) keep.shift(); + } + let s = 0; + for (const k of keep) s = (s + leaf(k)) | 0; + return (acc + s) | 0; +} + +console.log("r:" + run()); +"#; + +/// Collector knobs cleared before each run so a developer's exported kill +/// switch cannot turn every arm into the never-relocates control (mirrors +/// `gc_closure_self_pointer_root_7055`). +const GC_ENV_OVERRIDES: &[&str] = &[ + "PERRY_GEN_GC", + "PERRY_GC_SCAVENGE", + "PERRY_GC_SCAVENGE_NURSERY_MB", + "PERRY_GC_MOVING_SAFEPOINT", + "PERRY_GC_MOVING_LOOP_POLLS", + "PERRY_GC_FORCE_EVACUATE", + "PERRY_CONSERVATIVE_STACK_SCAN", + "PERRY_WRITE_BARRIERS", + "PERRY_GC_INCREMENTAL", + "PERRY_GC_HEAP_LIMIT", +]; + +fn compile(dir: &std::path::Path, spill_threshold: &str) -> (PathBuf, String) { + let entry = dir.join("main.ts"); + let output = dir.join(format!("bin_spill_{spill_threshold}")); + std::fs::write(&entry, SOURCE).expect("write entry"); + let out = Command::new(perry_bin()) + .current_dir(dir) + .arg("compile") + .arg(&entry) + .arg("-o") + .arg(&output) + .arg("--no-cache") + .env("PERRY_ROOT_SPILL_RELOCATIONS", spill_threshold) + .output() + .expect("run perry compile"); + assert!( + out.status.success(), + "perry compile (PERRY_ROOT_SPILL_RELOCATIONS={spill_threshold}) failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr), + ); + (output, String::from_utf8_lossy(&out.stderr).into_owned()) +} + +fn run_arms(binary: &std::path::Path, dir: &std::path::Path, label: &str) -> String { + let mut arms: Vec> = vec![vec![]]; + for mb in ["1", "2", "4", "8"] { + arms.push(vec![("PERRY_GC_SCAVENGE_NURSERY_MB", mb)]); + } + arms.push(vec![("PERRY_GEN_GC", "0")]); + + let mut first: Option = None; + for arm in &arms { + let mut cmd = Command::new(binary); + cmd.current_dir(dir); + for key in GC_ENV_OVERRIDES { + cmd.env_remove(key); + } + for (k, v) in arm { + cmd.env(k, v); + } + let run = cmd.output().expect("run compiled binary"); + let arm_label = if arm.is_empty() { + format!("{label}/default") + } else { + format!( + "{label}/{}", + arm.iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join(" ") + ) + }; + assert!( + run.status.success(), + "[{arm_label}] compiled binary failed (exit {:?})\nstderr:\n{}", + run.status.code(), + String::from_utf8_lossy(&run.stderr), + ); + let stdout = String::from_utf8_lossy(&run.stdout).into_owned(); + match &first { + None => first = Some(stdout), + Some(f) => assert_eq!( + &stdout, f, + "[{arm_label}] output differs between collector arms — a moving \ + minor left a stale root in this configuration" + ), + } + } + first.expect("at least one arm ran") +} + +#[test] +fn mixed_statepoint_and_shadow_frames_survive_a_relocating_minor() { + let dir = tempfile::tempdir().expect("tempdir"); + + // All-native reference and aggressively-spilled arm, from identical source. + let (native_bin, _native_err) = compile(dir.path(), "0"); + let (spilled_bin, spilled_err) = compile(dir.path(), "1"); + + // The spilled compile must have actually spilled, or the differential below + // proves nothing. The report names each shadow-framed function at default + // verbosity (#8421: the change is never silent). + assert!( + spilled_err.contains("keeps its") && spilled_err.contains("GC roots in a shadow frame"), + "PERRY_ROOT_SPILL_RELOCATIONS=1 was expected to spill at least one \ + function, but the compile reported none:\nstderr:\n{spilled_err}" + ); + + let native_out = run_arms(&native_bin, dir.path(), "native"); + let spilled_out = run_arms(&spilled_bin, dir.path(), "spilled"); + + assert!( + native_out.starts_with("r:"), + "unexpected program output: {native_out:?}" + ); + assert_eq!( + native_out, spilled_out, + "mixed statepoint/shadow-frame stacks (spilled) diverged from the \ + all-statepoint build (native): a live root in a spilled frame was not \ + found or not rewritten by a moving minor (#8583)" + ); +}