perf(codegen): inline admitted helpers before statepoints

changelog.d/8588-pre-statepoint-inline.md

@@ -0,0 +1,10 @@
+Added a scoped pre-statepoint inline budget for helpers that must cross Perry's
+RS4GC boundary. The proven nonnegative-index method clone is admitted before
+calls become statepoints, while historical force-inline sites remain ordinary
+LLVM hints under native roots; non-RS4GC behavior is unchanged.
+
+On the `codehz/ecs` 10,000-entity query, an 11-pair contended-host A/B against
+current `main` reduced paired medians by 2.22% for read-only iteration and 1.92%
+for accumulation. The candidate won 10/11 and 11/11 pairs respectively, shrank
+the executable by 148,664 bytes (1.264%), and all 22 processes passed the query
+assertions and exact 50,005,000 accumulation oracle.

crates/perry-codegen/src/codegen/index_method_clone_tests.rs

@@ -200,12 +200,25 @@ fn function_body(ir: &str, definition_contains: &str) -> String {
 
 #[test]
 fn proven_index_routes_to_live_clone_while_unproven_index_keeps_public_fallback() {
+    let _native = crate::codegen::helpers::NativeRootsPin::native();
     let ir = emit();
     let clone_symbol = "perry_method_index_method_clone_ts__Reader__read$idx_u31_12";
     let clone = function_body(&ir, &format!("@{clone_symbol}("));
     let public_symbol = "perry_method_index_method_clone_ts__Reader__read";
     let public = function_body(&ir, &format!("@{public_symbol}("));
 
+    assert!(
+        clone.lines().next().is_some_and(|line| line.contains(" alwaysinline ")),
+        "the proven index clone must be admitted before RS4GC turns its call into a statepoint:\n{clone}"
+    );
+    assert!(
+        public
+            .lines()
+            .next()
+            .is_some_and(|line| !line.contains(" alwaysinline ")),
+        "the public fallback must not consume the scoped pre-statepoint code-size budget:\n{public}"
+    );
+
     assert!(
         clone.contains("fptosi double %arg12 to i32")
             && clone.contains("arr.guard.deref")

crates/perry-codegen/src/codegen/method.rs

@@ -306,6 +306,9 @@ pub(super) fn compile_method(
     {
         lf.linkage = "internal".to_string();
     }
+    if is_index_clone {
+        lf.pre_statepoint_inline = true;
+    }
 
     // gh #6206 / #6081: methods were compiled WITHOUT a shadow frame — same
     // exact-roots liveness hole as closures (see compile_closure). One extra

crates/perry-codegen/src/function.rs

@@ -28,6 +28,12 @@ pub struct LlFunction {
     /// function at every call site, exposing integer operations to the
     /// caller's optimizer context (critical for vectorization of clamp patterns).
     pub force_inline: bool,
+    /// Admit this function to the unconditional inliner that runs before
+    /// RewriteStatepointsForGC. RS4GC turns calls into statepoints, after
+    /// which LLVM cannot honor `alwaysinline`; keeping this separate from the
+    /// general small-function policy gives the early pass an explicit code-size
+    /// budget instead of activating every historical hint at once.
+    pub pre_statepoint_inline: bool,
     /// When true, keep a small routing wrapper as an optimization boundary.
     /// Used when inlining would duplicate guarded fast/fallback call graphs.
     pub no_inline: bool,
@@ -237,6 +243,7 @@ impl LlFunction {
             params,
             linkage: String::new(),
             force_inline: false,
+            pre_statepoint_inline: false,
             no_inline: false,
             inline_hint: false,
             hot_loop_callee: false,
@@ -805,11 +812,12 @@ impl LlFunction {
             String::new()
         };
 
-        let attrs = if self.force_inline {
+        let rs4gc = crate::codegen::helpers::rs4gc_enabled();
+        let attrs = if self.pre_statepoint_inline || (self.force_inline && !rs4gc) {
             " alwaysinline"
         } else if self.no_inline {
             " noinline"
-        } else if self.inline_hint {
+        } else if self.inline_hint || self.force_inline {
             " inlinehint"
         } else {
             ""
@@ -1127,6 +1135,41 @@ mod define_header_tests {
         }
     }
 
+    #[test]
+    fn pre_statepoint_inline_has_an_explicit_native_roots_budget() {
+        use crate::codegen::helpers::NativeRootsPin;
+
+        {
+            let _shadow = NativeRootsPin::shadow();
+            let mut ordinary = probe();
+            ordinary.force_inline = true;
+            assert!(ordinary.define_header(false).contains(" alwaysinline"));
+        }
+
+        {
+            let _native = NativeRootsPin::native();
+            let mut ordinary = probe();
+            ordinary.force_inline = true;
+            let ordinary_header = ordinary.define_header(false);
+            assert!(ordinary_header.contains(" inlinehint"));
+            assert!(
+                !ordinary_header.contains(" alwaysinline"),
+                "the early pass must not activate every historical force-inline hint: \
+                 {ordinary_header}"
+            );
+
+            let mut admitted = probe();
+            admitted.pre_statepoint_inline = true;
+            let admitted_header = admitted.define_header(false);
+            assert!(admitted_header.contains(" alwaysinline"));
+            assert!(
+                !admitted_header.contains(" inlinehint"),
+                "an explicitly admitted function needs an unconditional attribute: \
+                 {admitted_header}"
+            );
+        }
+    }
+
     /// The property that was actually lost, asserted directly (#7982) — in
     /// **both** lowerings, neither of them dark.
     ///

crates/perry-codegen/src/inprocess.rs

@@ -30,25 +30,7 @@ use inkwell::targets::{
 };
 use inkwell::OptimizationLevel;
 
-/// The pass string that inserts every statepoint, relocation and
-/// downstream-use rewrite — i.e. the whole native-roots lowering, after
-/// codegen has retyped its root allocas to `ptr addrspace(1)`.
-///
-/// Named rather than spelled inline because `native_root_coverage` (#7502)
-/// runs it too, and a coverage suite that spelled its own pass list would keep
-/// passing against a pipeline production had stopped using. `mem2reg` is not
-/// incidental company: RS4GC tracks `addrspace(1)` **SSA values**, not memory,
-/// so a root alloca that survives promotion is a root the collector never sees.
-// SCCP—not InstCombine—is before RS4GC deliberately (#8065). Native C-API construction
-// folds constants as instructions are built, while whole-module text parsing
-// retains the equivalent instruction graph. If RS4GC sees those two shapes
-// before canonicalization, their live-root ordering can differ and reach both
-// machine code and the compact GC map. The ordinary optimization pipeline is
-// too late: statepoints and relocations have already been assigned by then.
-// The narrower SCCP preserves dynamic pointer round trips which InstCombine
-// can erase, so the positive live-root witness remains visible to RS4GC.
-pub(crate) const STATEPOINT_REWRITE_PASSES: &str =
-    "function(mem2reg,sccp),rewrite-statepoints-for-gc";
+use crate::linker::STATEPOINT_REWRITE_PASSES;
 
 /// Test seam (#7502): parse `ll_text`, run [`STATEPOINT_REWRITE_PASSES`] for
 /// `effective_target`, and return the rewritten IR.
@@ -547,6 +529,23 @@ fn optimize_and_emit(
 mod tests {
     use super::*;
 
+    fn relocation_results(ir: &str) -> std::collections::HashSet<&str> {
+        ir.lines()
+            .filter(|line| line.contains("@llvm.experimental.gc.relocate"))
+            .filter_map(|line| line.trim().split_once(" = ").map(|(result, _)| result))
+            .collect()
+    }
+
+    fn returned_gc_pointers(ir: &str) -> Vec<&str> {
+        ir.lines()
+            .filter_map(|line| {
+                line.trim()
+                    .strip_prefix("ret ptr addrspace(1) ")
+                    .and_then(|value| value.split_whitespace().next())
+            })
+            .collect()
+    }
+
     fn asm_barrier_fixture(leaf_attr: &str) -> String {
         format!(
             "declare i64 @may_collect()\n\n\
@@ -773,6 +772,134 @@ mod tests {
         );
     }
 
+    #[test]
+    fn rs4gc_honors_alwaysinline_before_rewriting_calls() {
+        let target = crate::codegen::default_target_triple();
+        let ir = r#"
+declare ptr addrspace(1) @alloc()
+
+define internal ptr addrspace(1) @leaf(ptr addrspace(1) %p) alwaysinline gc "statepoint-example" {
+entry:
+  %unused = call ptr addrspace(1) @alloc()
+  ret ptr addrspace(1) %p
+}
+
+define ptr addrspace(1) @caller(ptr addrspace(1) %p) gc "statepoint-example" {
+entry:
+  %result = call ptr addrspace(1) @leaf(ptr addrspace(1) %p)
+  ret ptr addrspace(1) %result
+}
+"#;
+
+        const PRE_FIX_PASSES: &str = "function(mem2reg,sccp),rewrite-statepoints-for-gc";
+        let before =
+            statepoint_rewritten_ir_with_passes(ir, &target, "alwaysinline_before", PRE_FIX_PASSES)
+                .expect("negative control rewrites the fixture");
+        assert!(
+            before.lines().any(|line| {
+                line.contains("@llvm.experimental.gc.statepoint") && line.contains("@leaf")
+            }),
+            "negative control must leave the alwaysinline call as a statepoint:\n{before}"
+        );
+
+        let after = statepoint_rewritten_ir(ir, &target, "alwaysinline_after")
+            .expect("shipped pipeline rewrites the inlined fixture");
+        assert!(
+            !after.contains("@leaf"),
+            "alwaysinline callee and call must disappear before RS4GC:\n{after}"
+        );
+        let live_bundle = after
+            .lines()
+            .find(|line| line.contains("@llvm.experimental.gc.statepoint"))
+            .unwrap_or_else(|| panic!("inlined allocation must remain a statepoint:\n{after}"));
+        assert!(
+            live_bundle.contains("\"gc-live\"") && live_bundle.contains("%p"),
+            "caller root must stay live through the inlined allocation:\n{after}"
+        );
+        let relocation_results = relocation_results(&after);
+        let returned_pointers = returned_gc_pointers(&after);
+        assert_eq!(
+            returned_pointers.len(),
+            1,
+            "fixture must retain exactly one return edge after inlining:\n{after}"
+        );
+        assert!(
+            relocation_results.contains(returned_pointers[0]),
+            "caller must return the gc.relocate result, not the pre-statepoint root:\n{after}"
+        );
+    }
+
+    #[test]
+    fn rs4gc_rewrites_inlined_invoke_and_preserves_exception_edge() {
+        let target = crate::codegen::default_target_triple();
+        let ir = r#"
+declare ptr addrspace(1) @alloc()
+declare i32 @perry_eh_personality(...)
+
+define internal ptr addrspace(1) @leaf(ptr addrspace(1) %p) alwaysinline gc "statepoint-example" personality ptr @perry_eh_personality {
+entry:
+  %unused = invoke ptr addrspace(1) @alloc()
+      to label %ok unwind label %exception
+ok:
+  ret ptr addrspace(1) %p
+exception:
+  %landing = landingpad token cleanup
+  ret ptr addrspace(1) %p
+}
+
+define ptr addrspace(1) @caller(ptr addrspace(1) %p) gc "statepoint-example" personality ptr @perry_eh_personality {
+entry:
+  %result = call ptr addrspace(1) @leaf(ptr addrspace(1) %p)
+  ret ptr addrspace(1) %result
+}
+"#;
+
+        let after = statepoint_rewritten_ir(ir, &target, "alwaysinline_invoke")
+            .expect("shipped pipeline rewrites an invoke in an inlined callee");
+        assert!(
+            !after.contains("@leaf"),
+            "alwaysinline invoke callee must disappear before RS4GC:\n{after}"
+        );
+        assert!(
+            after.lines().any(|line| {
+                line.contains("invoke token") && line.contains("@llvm.experimental.gc.statepoint")
+            }),
+            "inlined invoke must become a statepoint while retaining its unwind edge:\n{after}"
+        );
+        assert!(
+            after.contains("landingpad token")
+                && after.lines().any(|line| line.trim() == "cleanup"),
+            "statepoint invoke must retain a verifier-valid exceptional pad:\n{after}"
+        );
+        let relocation_results = relocation_results(&after);
+        let returned_pointers = returned_gc_pointers(&after);
+        assert_eq!(
+            returned_pointers.len(),
+            1,
+            "inlined invoke fixture must retain one merged return edge:\n{after}"
+        );
+        assert_eq!(
+            relocation_results.len(),
+            2,
+            "normal and exceptional continuations must each relocate the root:\n{after}"
+        );
+        let return_phi = after
+            .lines()
+            .find(|line| {
+                line.trim().starts_with(returned_pointers[0])
+                    && line.contains(" = phi ptr addrspace(1) ")
+            })
+            .unwrap_or_else(|| {
+                panic!("invoke continuations must merge through the returned phi:\n{after}")
+            });
+        assert!(
+            relocation_results
+                .iter()
+                .all(|relocated| return_phi.contains(*relocated)),
+            "returned phi must merge both gc.relocate results, not the pre-statepoint root:\n{after}"
+        );
+    }
+
     /// Layer-2 readiness (#7174, engine-plan layer 0 -> 2): the in-process
     /// pipeline can schedule `RewriteStatepointsForGC` at the pinned LLVM —
     /// no `opt` subprocess, no version-skewed toolchain. This is the exact

crates/perry-codegen/src/linker.rs

@@ -22,6 +22,21 @@ use linker_temp::{
     reap_stale_llvm_scratch_once, FailedScratch, FailureRetention, PROCESS_FAILURE_RETENTION,
 };
 
+/// The shared pass string that inserts every statepoint, relocation and
+/// downstream-use rewrite after codegen retypes root allocas to
+/// `ptr addrspace(1)`.
+///
+/// `always-inline` must run first. Once RS4GC rewrites a call to a statepoint,
+/// LLVM's normal optimization pipeline can no longer honor the callee's
+/// `alwaysinline` attribute. The function passes are also load-bearing:
+/// `mem2reg` exposes root allocas as SSA values for RS4GC, while SCCP converges
+/// textual and native-C-API constant folding before root liveness is assigned.
+///
+/// Both the external and in-process backends consume this constant so their
+/// native-roots correctness pipelines cannot drift.
+pub(crate) const STATEPOINT_REWRITE_PASSES: &str =
+    "always-inline,function(mem2reg,sccp),rewrite-statepoints-for-gc";
+
 /// Cached result of the pre-flight clang probe — evaluated once per process.
 /// `Some(default_triple)` if the probe succeeded, `None` if it failed.
 static CLANG_PROBE: OnceLock<Option<String>> = OnceLock::new();
@@ -499,12 +514,10 @@ fn maybe_rs4gc_preprocess(ll_text: &str, native_roots: bool) -> Result<Option<St
             "PERRY_RS4GC=1 requires an LLVM `opt` binary: set PERRY_LLVM_OPT, \
              install Homebrew LLVM, or put `opt` on PATH",
         )?;
+    let passes_arg = format!("-passes={STATEPOINT_REWRITE_PASSES}");
     let mut child = Command::new(&opt)
-        .args([
-            "-passes=function(mem2reg),rewrite-statepoints-for-gc",
-            "-S",
-            "-",
-        ])
+        .arg(&passes_arg)
+        .args(["-S", "-"])
         .stdin(std::process::Stdio::piped())
         .stdout(std::process::Stdio::piped())
         .stderr(std::process::Stdio::piped())
@@ -531,12 +544,13 @@ fn maybe_rs4gc_preprocess(ll_text: &str, native_roots: bool) -> Result<Option<St
         };
         return Err(anyhow!(
             "PERRY_RS4GC: opt pipeline failed ({}).\n{}\n\
-             reproduce: {} -passes='function(mem2reg),rewrite-statepoints-for-gc' -S {}\n\
+             reproduce: {} -passes='{}' -S {}\n\
              \n\
              stderr:\n{}",
             output.status,
             ir_note,
             opt.display(),
+            STATEPOINT_REWRITE_PASSES,
             ir_path.display(),
             String::from_utf8_lossy(&output.stderr)
         ));