From 7c8162ff68a60f5ec46d9f1a6ba8bbd8e0d7c589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 22 Aug 2026 09:07:43 +0200 Subject: [PATCH 1/4] perf(codegen): inline admitted helpers before statepoints --- .../src/codegen/index_method_clone_tests.rs | 13 ++ crates/perry-codegen/src/codegen/method.rs | 3 + crates/perry-codegen/src/function.rs | 47 ++++++- crates/perry-codegen/src/inprocess.rs | 119 +++++++++++++++--- crates/perry-codegen/src/linker.rs | 26 +++- 5 files changed, 181 insertions(+), 27 deletions(-) diff --git a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs index 60532b0f85..3761ae175e 100644 --- a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs +++ b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs @@ -200,12 +200,25 @@ fn function_body(ir: &str, definition_contains: &str) -> String { #[test] fn proven_index_routes_to_live_clone_while_unproven_index_keeps_public_fallback() { + let _native = crate::codegen::helpers::NativeRootsPin::native(); let ir = emit(); let clone_symbol = "perry_method_index_method_clone_ts__Reader__read$idx_u31_12"; let clone = function_body(&ir, &format!("@{clone_symbol}(")); let public_symbol = "perry_method_index_method_clone_ts__Reader__read"; let public = function_body(&ir, &format!("@{public_symbol}(")); + assert!( + clone.lines().next().is_some_and(|line| line.contains(" alwaysinline ")), + "the proven index clone must be admitted before RS4GC turns its call into a statepoint:\n{clone}" + ); + assert!( + public + .lines() + .next() + .is_some_and(|line| !line.contains(" alwaysinline ")), + "the public fallback must not consume the scoped pre-statepoint code-size budget:\n{public}" + ); + assert!( clone.contains("fptosi double %arg12 to i32") && clone.contains("arr.guard.deref") diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index c1363c5bba..3041666574 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -306,6 +306,9 @@ pub(super) fn compile_method( { lf.linkage = "internal".to_string(); } + if is_index_clone { + lf.pre_statepoint_inline = true; + } // gh #6206 / #6081: methods were compiled WITHOUT a shadow frame — same // exact-roots liveness hole as closures (see compile_closure). One extra diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 6759cd63bc..71d56352f1 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -28,6 +28,12 @@ pub struct LlFunction { /// function at every call site, exposing integer operations to the /// caller's optimizer context (critical for vectorization of clamp patterns). pub force_inline: bool, + /// Admit this function to the unconditional inliner that runs before + /// RewriteStatepointsForGC. RS4GC turns calls into statepoints, after + /// which LLVM cannot honor `alwaysinline`; keeping this separate from the + /// general small-function policy gives the early pass an explicit code-size + /// budget instead of activating every historical hint at once. + pub pre_statepoint_inline: bool, /// When true, keep a small routing wrapper as an optimization boundary. /// Used when inlining would duplicate guarded fast/fallback call graphs. pub no_inline: bool, @@ -237,6 +243,7 @@ impl LlFunction { params, linkage: String::new(), force_inline: false, + pre_statepoint_inline: false, no_inline: false, inline_hint: false, hot_loop_callee: false, @@ -805,11 +812,12 @@ impl LlFunction { String::new() }; - let attrs = if self.force_inline { + let rs4gc = crate::codegen::helpers::rs4gc_enabled(); + let attrs = if self.pre_statepoint_inline || (self.force_inline && !rs4gc) { " alwaysinline" } else if self.no_inline { " noinline" - } else if self.inline_hint { + } else if self.inline_hint || self.force_inline { " inlinehint" } else { "" @@ -1127,6 +1135,41 @@ mod define_header_tests { } } + #[test] + fn pre_statepoint_inline_has_an_explicit_native_roots_budget() { + use crate::codegen::helpers::NativeRootsPin; + + { + let _shadow = NativeRootsPin::shadow(); + let mut ordinary = probe(); + ordinary.force_inline = true; + assert!(ordinary.define_header(false).contains(" alwaysinline")); + } + + { + let _native = NativeRootsPin::native(); + let mut ordinary = probe(); + ordinary.force_inline = true; + let ordinary_header = ordinary.define_header(false); + assert!(ordinary_header.contains(" inlinehint")); + assert!( + !ordinary_header.contains(" alwaysinline"), + "the early pass must not activate every historical force-inline hint: \ + {ordinary_header}" + ); + + let mut admitted = probe(); + admitted.pre_statepoint_inline = true; + let admitted_header = admitted.define_header(false); + assert!(admitted_header.contains(" alwaysinline")); + assert!( + !admitted_header.contains(" inlinehint"), + "an explicitly admitted function needs an unconditional attribute: \ + {admitted_header}" + ); + } + } + /// The property that was actually lost, asserted directly (#7982) — in /// **both** lowerings, neither of them dark. /// diff --git a/crates/perry-codegen/src/inprocess.rs b/crates/perry-codegen/src/inprocess.rs index 972a48eb42..a26e64f9e2 100644 --- a/crates/perry-codegen/src/inprocess.rs +++ b/crates/perry-codegen/src/inprocess.rs @@ -30,25 +30,7 @@ use inkwell::targets::{ }; use inkwell::OptimizationLevel; -/// The pass string that inserts every statepoint, relocation and -/// downstream-use rewrite — i.e. the whole native-roots lowering, after -/// codegen has retyped its root allocas to `ptr addrspace(1)`. -/// -/// Named rather than spelled inline because `native_root_coverage` (#7502) -/// runs it too, and a coverage suite that spelled its own pass list would keep -/// passing against a pipeline production had stopped using. `mem2reg` is not -/// incidental company: RS4GC tracks `addrspace(1)` **SSA values**, not memory, -/// so a root alloca that survives promotion is a root the collector never sees. -// SCCP—not InstCombine—is before RS4GC deliberately (#8065). Native C-API construction -// folds constants as instructions are built, while whole-module text parsing -// retains the equivalent instruction graph. If RS4GC sees those two shapes -// before canonicalization, their live-root ordering can differ and reach both -// machine code and the compact GC map. The ordinary optimization pipeline is -// too late: statepoints and relocations have already been assigned by then. -// The narrower SCCP preserves dynamic pointer round trips which InstCombine -// can erase, so the positive live-root witness remains visible to RS4GC. -pub(crate) const STATEPOINT_REWRITE_PASSES: &str = - "function(mem2reg,sccp),rewrite-statepoints-for-gc"; +use crate::linker::STATEPOINT_REWRITE_PASSES; /// Test seam (#7502): parse `ll_text`, run [`STATEPOINT_REWRITE_PASSES`] for /// `effective_target`, and return the rewritten IR. @@ -773,6 +755,105 @@ mod tests { ); } + #[test] + fn rs4gc_honors_alwaysinline_before_rewriting_calls() { + let target = crate::codegen::default_target_triple(); + let ir = r#" +declare ptr addrspace(1) @alloc() + +define internal ptr addrspace(1) @leaf(ptr addrspace(1) %p) alwaysinline gc "statepoint-example" { +entry: + %unused = call ptr addrspace(1) @alloc() + ret ptr addrspace(1) %p +} + +define ptr addrspace(1) @caller(ptr addrspace(1) %p) gc "statepoint-example" { +entry: + %result = call ptr addrspace(1) @leaf(ptr addrspace(1) %p) + ret ptr addrspace(1) %result +} +"#; + + const PRE_FIX_PASSES: &str = "function(mem2reg,sccp),rewrite-statepoints-for-gc"; + let before = + statepoint_rewritten_ir_with_passes(ir, &target, "alwaysinline_before", PRE_FIX_PASSES) + .expect("negative control rewrites the fixture"); + assert!( + before.lines().any(|line| { + line.contains("@llvm.experimental.gc.statepoint") && line.contains("@leaf") + }), + "negative control must leave the alwaysinline call as a statepoint:\n{before}" + ); + + let after = statepoint_rewritten_ir(ir, &target, "alwaysinline_after") + .expect("shipped pipeline rewrites the inlined fixture"); + assert!( + !after.contains("@leaf"), + "alwaysinline callee and call must disappear before RS4GC:\n{after}" + ); + let live_bundle = after + .lines() + .find(|line| line.contains("@llvm.experimental.gc.statepoint")) + .unwrap_or_else(|| panic!("inlined allocation must remain a statepoint:\n{after}")); + assert!( + live_bundle.contains("\"gc-live\"") && live_bundle.contains("%p"), + "caller root must stay live through the inlined allocation:\n{after}" + ); + assert!( + after.contains("@llvm.experimental.gc.relocate") + && after.contains("ret ptr addrspace(1)"), + "caller must return the relocated root after the inlined allocation:\n{after}" + ); + } + + #[test] + fn rs4gc_rewrites_inlined_invoke_and_preserves_exception_edge() { + let target = crate::codegen::default_target_triple(); + let ir = r#" +declare ptr addrspace(1) @alloc() +declare i32 @perry_eh_personality(...) + +define internal ptr addrspace(1) @leaf(ptr addrspace(1) %p) alwaysinline gc "statepoint-example" personality ptr @perry_eh_personality { +entry: + %unused = invoke ptr addrspace(1) @alloc() + to label %ok unwind label %exception +ok: + ret ptr addrspace(1) %p +exception: + %landing = landingpad token cleanup + ret ptr addrspace(1) %p +} + +define ptr addrspace(1) @caller(ptr addrspace(1) %p) gc "statepoint-example" personality ptr @perry_eh_personality { +entry: + %result = call ptr addrspace(1) @leaf(ptr addrspace(1) %p) + ret ptr addrspace(1) %result +} +"#; + + let after = statepoint_rewritten_ir(ir, &target, "alwaysinline_invoke") + .expect("shipped pipeline rewrites an invoke in an inlined callee"); + assert!( + !after.contains("@leaf"), + "alwaysinline invoke callee must disappear before RS4GC:\n{after}" + ); + assert!( + after.lines().any(|line| { + line.contains("invoke token") && line.contains("@llvm.experimental.gc.statepoint") + }), + "inlined invoke must become a statepoint while retaining its unwind edge:\n{after}" + ); + assert!( + after.contains("landingpad token") + && after.lines().any(|line| line.trim() == "cleanup"), + "statepoint invoke must retain a verifier-valid exceptional pad:\n{after}" + ); + assert!( + after.contains("@llvm.experimental.gc.relocate"), + "both edges must relocate the caller root after the inlined invoke:\n{after}" + ); + } + /// Layer-2 readiness (#7174, engine-plan layer 0 -> 2): the in-process /// pipeline can schedule `RewriteStatepointsForGC` at the pinned LLVM — /// no `opt` subprocess, no version-skewed toolchain. This is the exact diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index ef0a2bb5a4..a90ba9ff46 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -22,6 +22,21 @@ use linker_temp::{ reap_stale_llvm_scratch_once, FailedScratch, FailureRetention, PROCESS_FAILURE_RETENTION, }; +/// The shared pass string that inserts every statepoint, relocation and +/// downstream-use rewrite after codegen retypes root allocas to +/// `ptr addrspace(1)`. +/// +/// `always-inline` must run first. Once RS4GC rewrites a call to a statepoint, +/// LLVM's normal optimization pipeline can no longer honor the callee's +/// `alwaysinline` attribute. The function passes are also load-bearing: +/// `mem2reg` exposes root allocas as SSA values for RS4GC, while SCCP converges +/// textual and native-C-API constant folding before root liveness is assigned. +/// +/// Both the external and in-process backends consume this constant so their +/// native-roots correctness pipelines cannot drift. +pub(crate) const STATEPOINT_REWRITE_PASSES: &str = + "always-inline,function(mem2reg,sccp),rewrite-statepoints-for-gc"; + /// Cached result of the pre-flight clang probe — evaluated once per process. /// `Some(default_triple)` if the probe succeeded, `None` if it failed. static CLANG_PROBE: OnceLock> = OnceLock::new(); @@ -499,12 +514,10 @@ fn maybe_rs4gc_preprocess(ll_text: &str, native_roots: bool) -> Result Result Date: Sat, 22 Aug 2026 09:45:57 +0200 Subject: [PATCH 2/4] docs(changelog): record pre-statepoint inline budget --- changelog.d/8588-pre-statepoint-inline.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog.d/8588-pre-statepoint-inline.md diff --git a/changelog.d/8588-pre-statepoint-inline.md b/changelog.d/8588-pre-statepoint-inline.md new file mode 100644 index 0000000000..da908747c0 --- /dev/null +++ b/changelog.d/8588-pre-statepoint-inline.md @@ -0,0 +1,10 @@ +Added a scoped pre-statepoint inline budget for helpers that must cross Perry's +RS4GC boundary. The proven nonnegative-index method clone is admitted before +calls become statepoints, while historical force-inline sites remain ordinary +LLVM hints under native roots; non-RS4GC behavior is unchanged. + +On the `codehz/ecs` 10,000-entity query, an 11-pair contended-host A/B against +current `main` reduced paired medians by 2.22% for read-only iteration and 1.92% +for accumulation. The candidate won 10/11 and 11/11 pairs respectively, shrank +the executable by 148,664 bytes (1.264%), and all 22 processes passed the query +assertions and exact 50,005,000 accumulation oracle. From 173661fd124827efaf4c148cede38fdb45cb9d89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 22 Aug 2026 10:15:09 +0200 Subject: [PATCH 3/4] test(codegen): expect inlined method shape guards --- .../tests/native_proof_regressions.rs | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/crates/perry-codegen/tests/native_proof_regressions.rs b/crates/perry-codegen/tests/native_proof_regressions.rs index a3de43bbdf..edb39054ee 100644 --- a/crates/perry-codegen/tests/native_proof_regressions.rs +++ b/crates/perry-codegen/tests/native_proof_regressions.rs @@ -151,6 +151,11 @@ fn compile_ir_for_module_with_opts(module: Module, opts: CompileOptions) -> anyh Ok(String::from_utf8(compile_module(&module, opts)?)?) } +fn contains_inline_direct_method_shape_guard(ir: &str) -> bool { + ir.contains("method_direct.inline_deref") + && ir.contains("load atomic i8, ptr @PERRY_CLASS_PROTOTYPE_FAST_GUARDS_INVALIDATED acquire") +} + fn compile_artifact_json(name: &str, body: Vec) -> serde_json::Value { compile_artifact_json_for_module(module(name, body)) } @@ -11561,7 +11566,7 @@ fn typed_i32_method_clone_emits_internal_clone_and_guarded_direct_call() { "public method wrapper should guard/unbox Int32 args and box raw i32 at the ABI edge:\n{wrapper_ir}" ); assert!( - caller_ir.contains("call i32 @js_method_direct_shape_guard") + contains_inline_direct_method_shape_guard(caller_ir) && caller_ir.contains("typed_i32_method.fast") && caller_ir.contains("typed_i32_method.generic") && caller_ir.contains("call i32 @js_typed_i32_arg_guard") @@ -11751,7 +11756,7 @@ fn typed_string_method_clone_emits_internal_clone_and_guarded_direct_call() { "public method wrapper should guard/unbox string args, call the raw clone, box the result, and keep generic fallback:\n{wrapper_ir}" ); assert!( - caller_ir.contains("call i32 @js_method_direct_shape_guard") + contains_inline_direct_method_shape_guard(caller_ir) && caller_ir.contains("typed_string_method.fast") && caller_ir.contains("typed_string_method.generic") && caller_ir.contains("call i32 @js_typed_string_arg_guard") @@ -11912,10 +11917,7 @@ fn typed_i1_method_clone_emits_internal_clone_and_guarded_direct_call() { )), "generic method ABI body must remain emitted separately:\n{ir}" ); - assert!( - ir.contains("call i32 @js_method_direct_shape_guard"), - "{ir}" - ); + assert!(contains_inline_direct_method_shape_guard(&ir), "{ir}"); assert!(ir.contains("call i32 @js_typed_i1_arg_guard"), "{ir}"); assert!(ir.contains("call i32 @js_typed_i1_arg_to_raw"), "{ir}"); assert!( @@ -12058,7 +12060,7 @@ fn typed_i1_numeric_predicate_method_uses_f64_params_and_guarded_direct_call() { "public method wrapper should guard/unbox f64 args before the i1 clone:\n{wrapper_ir}" ); assert!( - caller_ir.contains("call i32 @js_method_direct_shape_guard") + contains_inline_direct_method_shape_guard(caller_ir) && caller_ir.contains("call i32 @js_typed_f64_arg_guard") && caller_ir.contains("call double @js_typed_f64_arg_to_raw") && caller_ir.contains(&format!("call i1 @{typed}(double ")), @@ -12173,10 +12175,7 @@ fn typed_f64_method_clone_emits_internal_clone_and_guarded_direct_call() { )), "generic method ABI body must remain emitted separately:\n{ir}" ); - assert!( - ir.contains("call i32 @js_method_direct_shape_guard"), - "{ir}" - ); + assert!(contains_inline_direct_method_shape_guard(&ir), "{ir}"); assert!(ir.contains("call i32 @js_typed_f64_arg_guard"), "{ir}"); assert!(ir.contains("call double @js_typed_f64_arg_to_raw"), "{ir}"); assert!( From 7d762642754228379eea52fb7afcd4fd7338585a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 22 Aug 2026 11:34:18 +0200 Subject: [PATCH 4/4] test(codegen): prove relocated inline return flow --- crates/perry-codegen/src/inprocess.rs | 56 ++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/crates/perry-codegen/src/inprocess.rs b/crates/perry-codegen/src/inprocess.rs index a26e64f9e2..fe9556bb9a 100644 --- a/crates/perry-codegen/src/inprocess.rs +++ b/crates/perry-codegen/src/inprocess.rs @@ -529,6 +529,23 @@ fn optimize_and_emit( mod tests { use super::*; + fn relocation_results(ir: &str) -> std::collections::HashSet<&str> { + ir.lines() + .filter(|line| line.contains("@llvm.experimental.gc.relocate")) + .filter_map(|line| line.trim().split_once(" = ").map(|(result, _)| result)) + .collect() + } + + fn returned_gc_pointers(ir: &str) -> Vec<&str> { + ir.lines() + .filter_map(|line| { + line.trim() + .strip_prefix("ret ptr addrspace(1) ") + .and_then(|value| value.split_whitespace().next()) + }) + .collect() + } + fn asm_barrier_fixture(leaf_attr: &str) -> String { format!( "declare i64 @may_collect()\n\n\ @@ -799,10 +816,16 @@ entry: live_bundle.contains("\"gc-live\"") && live_bundle.contains("%p"), "caller root must stay live through the inlined allocation:\n{after}" ); + let relocation_results = relocation_results(&after); + let returned_pointers = returned_gc_pointers(&after); + assert_eq!( + returned_pointers.len(), + 1, + "fixture must retain exactly one return edge after inlining:\n{after}" + ); assert!( - after.contains("@llvm.experimental.gc.relocate") - && after.contains("ret ptr addrspace(1)"), - "caller must return the relocated root after the inlined allocation:\n{after}" + relocation_results.contains(returned_pointers[0]), + "caller must return the gc.relocate result, not the pre-statepoint root:\n{after}" ); } @@ -848,9 +871,32 @@ entry: && after.lines().any(|line| line.trim() == "cleanup"), "statepoint invoke must retain a verifier-valid exceptional pad:\n{after}" ); + let relocation_results = relocation_results(&after); + let returned_pointers = returned_gc_pointers(&after); + assert_eq!( + returned_pointers.len(), + 1, + "inlined invoke fixture must retain one merged return edge:\n{after}" + ); + assert_eq!( + relocation_results.len(), + 2, + "normal and exceptional continuations must each relocate the root:\n{after}" + ); + let return_phi = after + .lines() + .find(|line| { + line.trim().starts_with(returned_pointers[0]) + && line.contains(" = phi ptr addrspace(1) ") + }) + .unwrap_or_else(|| { + panic!("invoke continuations must merge through the returned phi:\n{after}") + }); assert!( - after.contains("@llvm.experimental.gc.relocate"), - "both edges must relocate the caller root after the inlined invoke:\n{after}" + relocation_results + .iter() + .all(|relocated| return_phi.contains(*relocated)), + "returned phi must merge both gc.relocate results, not the pre-statepoint root:\n{after}" ); }