diff --git a/changelog.d/8588-pre-statepoint-inline.md b/changelog.d/8588-pre-statepoint-inline.md
new file mode 100644
index 0000000000..da908747c0
--- /dev/null
+++ b/changelog.d/8588-pre-statepoint-inline.md
@@ -0,0 +1,10 @@
+Added a scoped pre-statepoint inline budget for helpers that must cross Perry's
+RS4GC boundary. The proven nonnegative-index method clone is admitted before
+calls become statepoints, while historical force-inline sites remain ordinary
+LLVM hints under native roots; non-RS4GC behavior is unchanged.
+
+On the `codehz/ecs` 10,000-entity query, an 11-pair contended-host A/B against
+current `main` reduced paired medians by 2.22% for read-only iteration and 1.92%
+for accumulation. The candidate won 10/11 and 11/11 pairs respectively, shrank
+the executable by 148,664 bytes (1.264%), and all 22 processes passed the query
+assertions and exact 50,005,000 accumulation oracle.
diff --git a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs
index 60532b0f85..3761ae175e 100644
--- a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs
+++ b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs
@@ -200,12 +200,25 @@ fn function_body(ir: &str, definition_contains: &str) -> String {
#[test]
fn proven_index_routes_to_live_clone_while_unproven_index_keeps_public_fallback() {
+ let _native = crate::codegen::helpers::NativeRootsPin::native();
let ir = emit();
let clone_symbol = "perry_method_index_method_clone_ts__Reader__read$idx_u31_12";
let clone = function_body(&ir, &format!("@{clone_symbol}("));
let public_symbol = "perry_method_index_method_clone_ts__Reader__read";
let public = function_body(&ir, &format!("@{public_symbol}("));
+ assert!(
+ clone.lines().next().is_some_and(|line| line.contains(" alwaysinline ")),
+ "the proven index clone must be admitted before RS4GC turns its call into a statepoint:\n{clone}"
+ );
+ assert!(
+ public
+ .lines()
+ .next()
+ .is_some_and(|line| !line.contains(" alwaysinline ")),
+ "the public fallback must not consume the scoped pre-statepoint code-size budget:\n{public}"
+ );
+
assert!(
clone.contains("fptosi double %arg12 to i32")
&& clone.contains("arr.guard.deref")
diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs
index c1363c5bba..3041666574 100644
--- a/crates/perry-codegen/src/codegen/method.rs
+++ b/crates/perry-codegen/src/codegen/method.rs
@@ -306,6 +306,9 @@ pub(super) fn compile_method(
{
lf.linkage = "internal".to_string();
}
+ if is_index_clone {
+ lf.pre_statepoint_inline = true;
+ }
// gh #6206 / #6081: methods were compiled WITHOUT a shadow frame — same
// exact-roots liveness hole as closures (see compile_closure). One extra
diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs
index 6759cd63bc..71d56352f1 100644
--- a/crates/perry-codegen/src/function.rs
+++ b/crates/perry-codegen/src/function.rs
@@ -28,6 +28,12 @@ pub struct LlFunction {
/// function at every call site, exposing integer operations to the
/// caller's optimizer context (critical for vectorization of clamp patterns).
pub force_inline: bool,
+ /// Admit this function to the unconditional inliner that runs before
+ /// RewriteStatepointsForGC. RS4GC turns calls into statepoints, after
+ /// which LLVM cannot honor `alwaysinline`; keeping this separate from the
+ /// general small-function policy gives the early pass an explicit code-size
+ /// budget instead of activating every historical hint at once.
+ pub pre_statepoint_inline: bool,
/// When true, keep a small routing wrapper as an optimization boundary.
/// Used when inlining would duplicate guarded fast/fallback call graphs.
pub no_inline: bool,
@@ -237,6 +243,7 @@ impl LlFunction {
params,
linkage: String::new(),
force_inline: false,
+ pre_statepoint_inline: false,
no_inline: false,
inline_hint: false,
hot_loop_callee: false,
@@ -805,11 +812,12 @@ impl LlFunction {
String::new()
};
- let attrs = if self.force_inline {
+ let rs4gc = crate::codegen::helpers::rs4gc_enabled();
+ let attrs = if self.pre_statepoint_inline || (self.force_inline && !rs4gc) {
" alwaysinline"
} else if self.no_inline {
" noinline"
- } else if self.inline_hint {
+ } else if self.inline_hint || self.force_inline {
" inlinehint"
} else {
""
@@ -1127,6 +1135,41 @@ mod define_header_tests {
}
}
+ #[test]
+ fn pre_statepoint_inline_has_an_explicit_native_roots_budget() {
+ use crate::codegen::helpers::NativeRootsPin;
+
+ {
+ let _shadow = NativeRootsPin::shadow();
+ let mut ordinary = probe();
+ ordinary.force_inline = true;
+ assert!(ordinary.define_header(false).contains(" alwaysinline"));
+ }
+
+ {
+ let _native = NativeRootsPin::native();
+ let mut ordinary = probe();
+ ordinary.force_inline = true;
+ let ordinary_header = ordinary.define_header(false);
+ assert!(ordinary_header.contains(" inlinehint"));
+ assert!(
+ !ordinary_header.contains(" alwaysinline"),
+ "the early pass must not activate every historical force-inline hint: \
+ {ordinary_header}"
+ );
+
+ let mut admitted = probe();
+ admitted.pre_statepoint_inline = true;
+ let admitted_header = admitted.define_header(false);
+ assert!(admitted_header.contains(" alwaysinline"));
+ assert!(
+ !admitted_header.contains(" inlinehint"),
+ "an explicitly admitted function needs an unconditional attribute: \
+ {admitted_header}"
+ );
+ }
+ }
+
/// The property that was actually lost, asserted directly (#7982) — in
/// **both** lowerings, neither of them dark.
///
diff --git a/crates/perry-codegen/src/inprocess.rs b/crates/perry-codegen/src/inprocess.rs
index 972a48eb42..fe9556bb9a 100644
--- a/crates/perry-codegen/src/inprocess.rs
+++ b/crates/perry-codegen/src/inprocess.rs
@@ -30,25 +30,7 @@ use inkwell::targets::{
};
use inkwell::OptimizationLevel;
-/// The pass string that inserts every statepoint, relocation and
-/// downstream-use rewrite — i.e. the whole native-roots lowering, after
-/// codegen has retyped its root allocas to `ptr addrspace(1)`.
-///
-/// Named rather than spelled inline because `native_root_coverage` (#7502)
-/// runs it too, and a coverage suite that spelled its own pass list would keep
-/// passing against a pipeline production had stopped using. `mem2reg` is not
-/// incidental company: RS4GC tracks `addrspace(1)` **SSA values**, not memory,
-/// so a root alloca that survives promotion is a root the collector never sees.
-// SCCP—not InstCombine—is before RS4GC deliberately (#8065). Native C-API construction
-// folds constants as instructions are built, while whole-module text parsing
-// retains the equivalent instruction graph. If RS4GC sees those two shapes
-// before canonicalization, their live-root ordering can differ and reach both
-// machine code and the compact GC map. The ordinary optimization pipeline is
-// too late: statepoints and relocations have already been assigned by then.
-// The narrower SCCP preserves dynamic pointer round trips which InstCombine
-// can erase, so the positive live-root witness remains visible to RS4GC.
-pub(crate) const STATEPOINT_REWRITE_PASSES: &str =
- "function(mem2reg,sccp),rewrite-statepoints-for-gc";
+use crate::linker::STATEPOINT_REWRITE_PASSES;
/// Test seam (#7502): parse `ll_text`, run [`STATEPOINT_REWRITE_PASSES`] for
/// `effective_target`, and return the rewritten IR.
@@ -547,6 +529,23 @@ fn optimize_and_emit(
mod tests {
use super::*;
+ fn relocation_results(ir: &str) -> std::collections::HashSet<&str> {
+ ir.lines()
+ .filter(|line| line.contains("@llvm.experimental.gc.relocate"))
+ .filter_map(|line| line.trim().split_once(" = ").map(|(result, _)| result))
+ .collect()
+ }
+
+ fn returned_gc_pointers(ir: &str) -> Vec<&str> {
+ ir.lines()
+ .filter_map(|line| {
+ line.trim()
+ .strip_prefix("ret ptr addrspace(1) ")
+ .and_then(|value| value.split_whitespace().next())
+ })
+ .collect()
+ }
+
fn asm_barrier_fixture(leaf_attr: &str) -> String {
format!(
"declare i64 @may_collect()\n\n\
@@ -773,6 +772,134 @@ mod tests {
);
}
+ #[test]
+ fn rs4gc_honors_alwaysinline_before_rewriting_calls() {
+ let target = crate::codegen::default_target_triple();
+ let ir = r#"
+declare ptr addrspace(1) @alloc()
+
+define internal ptr addrspace(1) @leaf(ptr addrspace(1) %p) alwaysinline gc "statepoint-example" {
+entry:
+ %unused = call ptr addrspace(1) @alloc()
+ ret ptr addrspace(1) %p
+}
+
+define ptr addrspace(1) @caller(ptr addrspace(1) %p) gc "statepoint-example" {
+entry:
+ %result = call ptr addrspace(1) @leaf(ptr addrspace(1) %p)
+ ret ptr addrspace(1) %result
+}
+"#;
+
+ const PRE_FIX_PASSES: &str = "function(mem2reg,sccp),rewrite-statepoints-for-gc";
+ let before =
+ statepoint_rewritten_ir_with_passes(ir, &target, "alwaysinline_before", PRE_FIX_PASSES)
+ .expect("negative control rewrites the fixture");
+ assert!(
+ before.lines().any(|line| {
+ line.contains("@llvm.experimental.gc.statepoint") && line.contains("@leaf")
+ }),
+ "negative control must leave the alwaysinline call as a statepoint:\n{before}"
+ );
+
+ let after = statepoint_rewritten_ir(ir, &target, "alwaysinline_after")
+ .expect("shipped pipeline rewrites the inlined fixture");
+ assert!(
+ !after.contains("@leaf"),
+ "alwaysinline callee and call must disappear before RS4GC:\n{after}"
+ );
+ let live_bundle = after
+ .lines()
+ .find(|line| line.contains("@llvm.experimental.gc.statepoint"))
+ .unwrap_or_else(|| panic!("inlined allocation must remain a statepoint:\n{after}"));
+ assert!(
+ live_bundle.contains("\"gc-live\"") && live_bundle.contains("%p"),
+ "caller root must stay live through the inlined allocation:\n{after}"
+ );
+ let relocation_results = relocation_results(&after);
+ let returned_pointers = returned_gc_pointers(&after);
+ assert_eq!(
+ returned_pointers.len(),
+ 1,
+ "fixture must retain exactly one return edge after inlining:\n{after}"
+ );
+ assert!(
+ relocation_results.contains(returned_pointers[0]),
+ "caller must return the gc.relocate result, not the pre-statepoint root:\n{after}"
+ );
+ }
+
+ #[test]
+ fn rs4gc_rewrites_inlined_invoke_and_preserves_exception_edge() {
+ let target = crate::codegen::default_target_triple();
+ let ir = r#"
+declare ptr addrspace(1) @alloc()
+declare i32 @perry_eh_personality(...)
+
+define internal ptr addrspace(1) @leaf(ptr addrspace(1) %p) alwaysinline gc "statepoint-example" personality ptr @perry_eh_personality {
+entry:
+ %unused = invoke ptr addrspace(1) @alloc()
+ to label %ok unwind label %exception
+ok:
+ ret ptr addrspace(1) %p
+exception:
+ %landing = landingpad token cleanup
+ ret ptr addrspace(1) %p
+}
+
+define ptr addrspace(1) @caller(ptr addrspace(1) %p) gc "statepoint-example" personality ptr @perry_eh_personality {
+entry:
+ %result = call ptr addrspace(1) @leaf(ptr addrspace(1) %p)
+ ret ptr addrspace(1) %result
+}
+"#;
+
+ let after = statepoint_rewritten_ir(ir, &target, "alwaysinline_invoke")
+ .expect("shipped pipeline rewrites an invoke in an inlined callee");
+ assert!(
+ !after.contains("@leaf"),
+ "alwaysinline invoke callee must disappear before RS4GC:\n{after}"
+ );
+ assert!(
+ after.lines().any(|line| {
+ line.contains("invoke token") && line.contains("@llvm.experimental.gc.statepoint")
+ }),
+ "inlined invoke must become a statepoint while retaining its unwind edge:\n{after}"
+ );
+ assert!(
+ after.contains("landingpad token")
+ && after.lines().any(|line| line.trim() == "cleanup"),
+ "statepoint invoke must retain a verifier-valid exceptional pad:\n{after}"
+ );
+ let relocation_results = relocation_results(&after);
+ let returned_pointers = returned_gc_pointers(&after);
+ assert_eq!(
+ returned_pointers.len(),
+ 1,
+ "inlined invoke fixture must retain one merged return edge:\n{after}"
+ );
+ assert_eq!(
+ relocation_results.len(),
+ 2,
+ "normal and exceptional continuations must each relocate the root:\n{after}"
+ );
+ let return_phi = after
+ .lines()
+ .find(|line| {
+ line.trim().starts_with(returned_pointers[0])
+ && line.contains(" = phi ptr addrspace(1) ")
+ })
+ .unwrap_or_else(|| {
+ panic!("invoke continuations must merge through the returned phi:\n{after}")
+ });
+ assert!(
+ relocation_results
+ .iter()
+ .all(|relocated| return_phi.contains(*relocated)),
+ "returned phi must merge both gc.relocate results, not the pre-statepoint root:\n{after}"
+ );
+ }
+
/// Layer-2 readiness (#7174, engine-plan layer 0 -> 2): the in-process
/// pipeline can schedule `RewriteStatepointsForGC` at the pinned LLVM —
/// no `opt` subprocess, no version-skewed toolchain. This is the exact
diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs
index ef0a2bb5a4..a90ba9ff46 100644
--- a/crates/perry-codegen/src/linker.rs
+++ b/crates/perry-codegen/src/linker.rs
@@ -22,6 +22,21 @@ use linker_temp::{
reap_stale_llvm_scratch_once, FailedScratch, FailureRetention, PROCESS_FAILURE_RETENTION,
};
+/// The shared pass string that inserts every statepoint, relocation and
+/// downstream-use rewrite after codegen retypes root allocas to
+/// `ptr addrspace(1)`.
+///
+/// `always-inline` must run first. Once RS4GC rewrites a call to a statepoint,
+/// LLVM's normal optimization pipeline can no longer honor the callee's
+/// `alwaysinline` attribute. The function passes are also load-bearing:
+/// `mem2reg` exposes root allocas as SSA values for RS4GC, while SCCP converges
+/// textual and native-C-API constant folding before root liveness is assigned.
+///
+/// Both the external and in-process backends consume this constant so their
+/// native-roots correctness pipelines cannot drift.
+pub(crate) const STATEPOINT_REWRITE_PASSES: &str =
+ "always-inline,function(mem2reg,sccp),rewrite-statepoints-for-gc";
+
/// Cached result of the pre-flight clang probe — evaluated once per process.
/// `Some(default_triple)` if the probe succeeded, `None` if it failed.
static CLANG_PROBE: OnceLock