perf(codegen): inline monomorphic method shape guards

changelog.d/8573-inline-method-shape-guard.md

@@ -0,0 +1,5 @@
+Scoped direct-method descriptor invalidation to the receiver and relevant
+prototype mutations, then inlined complete monomorphic ShapeId guards. On a
+10,000-entity callback-heavy query with an unrelated descriptor, read-only time
+falls from 3.7599 to 0.3381 ms/op and accumulation from 3.6702 to 0.2829 ms/op;
+inlining contributes a further 7.4% and 8.1% paired reduction respectively.

crates/perry-codegen/src/block.rs

@@ -634,6 +634,19 @@ impl LlBlock {
         r
     }
 
+    /// Acquire atomic load for a sticky runtime gate whose publishing store is
+    /// release-ordered. The explicit alignment is required by LLVM atomics.
+    pub fn load_atomic_acquire(&mut self, ty: LlvmType, ptr: &str, alignment: u32) -> String {
+        let r = self.reg();
+        self.push_inst(crate::inst::LlInst::Load {
+            dst: r.clone(),
+            ty,
+            ptr: ptr.to_string(),
+            flavor: crate::inst::LoadFlavor::AtomicAcquire(alignment),
+        });
+        r
+    }
+
     /// Sequentially-consistent atomic load for globals shared with runtime
     /// atomics. The explicit alignment is required by LLVM atomic loads.
     pub fn load_atomic_seq_cst(&mut self, ty: LlvmType, ptr: &str, alignment: u32) -> String {

crates/perry-codegen/src/collectors/proven_this_routing_tests.rs

@@ -542,7 +542,10 @@ fn guarded_pshape_call_site_is_preceded_by_a_shape_id_guard() {
         });
         let prefix = &probe[..call_pos];
         let guarded = prefix.contains("call i32 @js_typed_feedback_method_direct_call_guard(")
-            || prefix.contains("call i32 @js_method_direct_shape_guard(");
+            || prefix.contains("call i32 @js_method_direct_shape_guard(")
+            || prefix.contains(
+                "load atomic i8, ptr @PERRY_CLASS_PROTOTYPE_FAST_GUARDS_INVALIDATED acquire",
+            );
         assert!(
             guarded,
             "{target}: no ShapeId guard call precedes it in `probe` — a \
@@ -553,6 +556,60 @@ fn guarded_pshape_call_site_is_preceded_by_a_shape_id_guard() {
     }
 }
 
+/// The single-pair shape-only arm is small enough to inline at the call site.
+/// Pin the complete safety gate: acquire the prototype-mutation latch, accept
+/// both the boxed-pointer and internal raw-pointer ABIs, reject addresses
+/// outside the target heap range before dereference, reject own descriptors,
+/// then compare the exact class/ShapeId pair. The out-of-line guard must be
+/// absent from this caller.
+#[test]
+fn single_arm_method_shape_guard_is_inlined_with_the_runtime_contract() {
+    let ir = emit(&guarded_site_module(), false);
+    let probe = function_body(&ir, "__probe(");
+    assert!(
+        probe.contains(
+            "load atomic i8, ptr @PERRY_CLASS_PROTOTYPE_FAST_GUARDS_INVALIDATED acquire, align 1",
+        ),
+        "the inline guard must acquire the runtime's release-published sticky latch:\n{probe}"
+    );
+    assert!(
+        !probe.contains("call i32 @js_method_direct_shape_guard("),
+        "a monomorphic shape-only site must not retain the out-of-line guard call:\n{probe}"
+    );
+    assert!(
+        probe.contains("icmp eq i64")
+            && probe.contains(", 32765")
+            && probe.contains(", 0"),
+        "the dereference gate must accept the 0x7FFD boxed pointer tag and the internal raw-pointer form:\n{probe}"
+    );
+    let target = crate::codegen::default_target_triple();
+    let heap_floor = crate::target_layout::heap_addr_lower_bound_inclusive(&target);
+    let heap_ceiling = crate::target_layout::heap_addr_upper_bound_exclusive(&target);
+    assert!(
+        probe.contains("icmp uge i64")
+            && probe.contains(&format!(", {heap_floor}"))
+            && probe.contains("icmp ult i64")
+            && probe.contains(&format!(", {heap_ceiling}")),
+        "the dereference gate must reject candidates outside the target heap range:\n{probe}"
+    );
+    assert!(
+        probe.contains("method_direct.inline_deref")
+            && probe.contains("getelementptr i8, ptr")
+            && probe.contains("i64 -8")
+            && probe.contains("i64 -7")
+            && probe.contains("i64 -6")
+            && probe.contains("and i16")
+            && probe.contains(", 2048")
+            && probe.contains("icmp ne i32")
+            && probe.contains("i64 4")
+            && probe.contains("add i32")
+            && probe.contains(", -2147483648")
+            && probe.contains("icmp ult i32")
+            && probe.contains(", 1073741824"),
+        "the header block must check the GC type, forwarding flag, own-descriptor bit, nonzero class id, ShapeId domain, and live ShapeId:\n{probe}"
+    );
+}
+
 /// Regression (#7128), Phase 3b guard-free site: a shape-proven LOCAL whose
 /// method also has a typed-receiver clone must still route to the proven-`this`
 /// clone.

crates/perry-codegen/src/dialect/mod.rs

@@ -1379,6 +1379,17 @@ impl<'ctx, 'm> FnReader<'ctx, 'm> {
                             let _ = i.set_alignment(*n);
                         }
                     }
+                    LF::AtomicAcquire(n) => {
+                        if let Some(i) = v.as_instruction_value() {
+                            unsafe {
+                                llvm_sys::core::LLVMSetOrdering(
+                                    i.as_value_ref(),
+                                    llvm_sys::LLVMAtomicOrdering::LLVMAtomicOrderingAcquire,
+                                )
+                            };
+                            let _ = i.set_alignment(*n);
+                        }
+                    }
                     LF::AtomicSeqCst(n) => {
                         if let Some(i) = v.as_instruction_value() {
                             unsafe {

crates/perry-codegen/src/dialect/tests.rs

@@ -221,6 +221,20 @@ fn compiled_module_ir_round_trips_through_the_reader() {
     assert!(n > 0, "round-tripped an empty module");
 }
 
+#[test]
+fn acquire_atomic_load_round_trips_through_the_reader() {
+    let ir = r#"
+@gate = external global i8
+
+define i8 @load_gate() {
+entry:
+  %value = load atomic i8, ptr @gate acquire, align 1
+  ret i8 %value
+}
+"#;
+    assert_eq!(roundtrip_ir(ir, "acquire_load"), 2);
+}
+
 /// The `format!` templates `expr/channel.rs` emits for its `<4 x i32>` SIMD
 /// byte-channel reduction, in emission order. The fixture below is BUILT from
 /// these strings rather than duplicating them, and each is asserted to still

crates/perry-codegen/src/inst.rs

@@ -36,6 +36,7 @@ pub enum LoadFlavor {
     Aligned(u32),
     Volatile,
     AtomicMonotonic(u32),
+    AtomicAcquire(u32),
     AtomicSeqCst(u32),
     /// `!invariant.load !0` tagged (issue #52).
     Invariant,
@@ -219,6 +220,12 @@ impl LlInst {
                         "  {dst} = load atomic {ty}, ptr {ptr} monotonic, align {n}"
                     );
                 }
+                LoadFlavor::AtomicAcquire(n) => {
+                    let _ = write!(
+                        out,
+                        "  {dst} = load atomic {ty}, ptr {ptr} acquire, align {n}"
+                    );
+                }
                 LoadFlavor::AtomicSeqCst(n) => {
                     let _ = write!(
                         out,

crates/perry-codegen/src/lower_call/method_override.rs

@@ -11,7 +11,109 @@ use crate::expr::{
 };
 use crate::nanbox::double_literal;
 use crate::native_value::LoweredValue;
-use crate::types::{DOUBLE, I1, I32, I64};
+use crate::types::{DOUBLE, I1, I16, I32, I64, I8};
+
+const POINTER_TAG_HI16: &str = "32765"; // 0x7FFD
+const GC_TYPE_OBJECT: &str = "2";
+const GC_FLAG_FORWARDED_I8: &str = "-128"; // 0x80 as i8
+const OBJ_FLAG_HAS_DESCRIPTORS_I16: &str = "2048"; // 0x0800
+const SHAPE_ID_BASE_NEG_I32: &str = "-2147483648"; // subtract 0x8000_0000
+const SHAPE_ID_RANGE_LEN: &str = "1073741824"; // 0x4000_0000
+
+/// Emit the single-arm equivalent of `js_method_direct_shape_guard` directly
+/// into the generated module. The guard remains dynamic at every call site:
+/// arbitrary callback code may replace a prototype method or mutate the
+/// receiver between loop iterations.
+///
+/// The first block proves that the value is a tagged heap pointer or the raw
+/// object-address form used by internal method ABIs before any dereference.
+/// The second block reproduces the runtime helper's production contract: the
+/// class-prototype invalidation latch is clear, the receiver is a non-forwarded
+/// ordinary object without own descriptors, and its exact `(class_id, ShapeId)`
+/// pair still matches the compiler-published pair. Any failed proof takes the
+/// unchanged dynamic method fallback.
+fn emit_inline_direct_method_shape_guard(
+    ctx: &mut FnCtx<'_>,
+    recv_box: &str,
+    expected_class_id: &str,
+    expected_shape_id: &str,
+    fast_label: &str,
+    fallback_label: &str,
+) {
+    let deref_idx = ctx.new_block("method_direct.inline_deref");
+    let deref_label = ctx.block_label(deref_idx);
+    let heap_floor =
+        crate::target_layout::heap_addr_lower_bound_inclusive(ctx.target_triple).to_string();
+    let heap_ceiling =
+        crate::target_layout::heap_addr_upper_bound_exclusive(ctx.target_triple).to_string();
+
+    {
+        let blk = ctx.block();
+        let invalidated =
+            blk.load_atomic_acquire(I8, "@PERRY_CLASS_PROTOTYPE_FAST_GUARDS_INVALIDATED", 1);
+        let prototype_ok = blk.icmp_eq(I8, &invalidated, "0");
+        let recv_bits = blk.bitcast_double_to_i64(recv_box);
+        let recv_handle = blk.and(I64, &recv_bits, crate::nanbox::POINTER_MASK_I64);
+        let tag = blk.lshr(I64, &recv_bits, "48");
+        let is_tagged_ptr = blk.icmp_eq(I64, &tag, POINTER_TAG_HI16);
+        // Internal method ABIs also carry an unboxed raw object address in a
+        // double-sized slot. `normalize_raw_object_addr` accepts exactly this
+        // top-word-zero form; all other non-pointer NaN-box tags remain
+        // rejected before dereference.
+        let is_raw_ptr = blk.icmp_eq(I64, &tag, "0");
+        let is_ptr = blk.or(I1, &is_tagged_ptr, &is_raw_ptr);
+        let above_floor = blk.icmp_uge(I64, &recv_handle, &heap_floor);
+        let below_ceiling = blk.icmp_ult(I64, &recv_handle, &heap_ceiling);
+        let in_heap_range = blk.and(I1, &above_floor, &below_ceiling);
+        let ptr_safe = blk.and(I1, &is_ptr, &in_heap_range);
+        let can_deref = blk.and(I1, &prototype_ok, &ptr_safe);
+        blk.cond_br(&can_deref, &deref_label, fallback_label);
+    }
+
+    ctx.current_block = deref_idx;
+    {
+        let blk = ctx.block();
+        let recv_bits = blk.bitcast_double_to_i64(recv_box);
+        let recv_handle = blk.and(I64, &recv_bits, crate::nanbox::POINTER_MASK_I64);
+        let obj_ptr = blk.inttoptr(I64, &recv_handle);
+
+        let gtype_ptr = blk.gep(I8, &obj_ptr, &[(I64, "-8")]);
+        let gtype = blk.load(I8, &gtype_ptr);
+        let gtype_ok = blk.icmp_eq(I8, &gtype, GC_TYPE_OBJECT);
+
+        let gflags_ptr = blk.gep(I8, &obj_ptr, &[(I64, "-7")]);
+        let gflags = blk.load(I8, &gflags_ptr);
+        let forwarded = blk.and(I8, &gflags, GC_FLAG_FORWARDED_I8);
+        let not_forwarded = blk.icmp_eq(I8, &forwarded, "0");
+
+        let reserved_ptr = blk.gep(I8, &obj_ptr, &[(I64, "-6")]);
+        let reserved = blk.load(I16, &reserved_ptr);
+        let descriptor_bits = blk.and(I16, &reserved, OBJ_FLAG_HAS_DESCRIPTORS_I16);
+        let no_own_descriptors = blk.icmp_eq(I16, &descriptor_bits, "0");
+
+        let class_ptr = blk.gep(I8, &obj_ptr, &[(I64, "0")]);
+        let class_id = blk.load(I32, &class_ptr);
+        let class_valid = blk.icmp_ne(I32, &class_id, "0");
+        let class_ok = blk.icmp_eq(I32, &class_id, expected_class_id);
+
+        let shape_ptr = blk.gep(I8, &obj_ptr, &[(I64, "4")]);
+        let shape_id = blk.load(I32, &shape_ptr);
+        // `is_shape_id` is `[0x8000_0000, 0xC000_0000)`. Subtract the base
+        // modulo i32 and compare with the range length, matching the runtime
+        // helper without a call.
+        let shape_id_rel = blk.add(I32, &shape_id, SHAPE_ID_BASE_NEG_I32);
+        let shape_valid = blk.icmp_ult(I32, &shape_id_rel, SHAPE_ID_RANGE_LEN);
+        let shape_ok = blk.icmp_eq(I32, &shape_id, expected_shape_id);
+
+        let mut pass = blk.and(I1, &gtype_ok, &not_forwarded);
+        pass = blk.and(I1, &pass, &no_own_descriptors);
+        pass = blk.and(I1, &pass, &class_valid);
+        pass = blk.and(I1, &pass, &class_ok);
+        pass = blk.and(I1, &pass, &shape_valid);
+        pass = blk.and(I1, &pass, &shape_ok);
+        blk.cond_br(&pass, fast_label, fallback_label);
+    }
+}
 
 fn typed_i1_method_signature_note(reps: &[crate::codegen::TypedParamRep]) -> String {
     let first = reps.first().map(|rep| rep.label()).unwrap_or("void");
@@ -292,9 +394,11 @@ pub(super) fn emit_guarded_direct_method_call(
     ctx.current_block = guard_idx;
     // Multi-arm form: ONE probe resolves the receiver's class id and keys
     // token (every precondition `js_method_direct_shape_guard` checks except
-    // the comparison itself), then an inline compare chain picks the arm. The
-    // single-arm form keeps its original single call.
+    // the comparison itself), then an inline compare chain picks the arm. A
+    // shape-only single-arm site emits the equivalent guard inline; other
+    // single-arm sites retain the runtime helper.
     let multi_arm = !subclass_arms.is_empty();
+    let inline_single_arm = shape_only_guard && !multi_arm;
     if multi_arm {
         let shape_slot = ctx.func.alloca_entry(I32);
         let cid = ctx.block().call(
@@ -328,9 +432,21 @@ pub(super) fn emit_guarded_direct_method_call(
         }
         ctx.current_block = guard_idx;
     }
-    let guard_ok = if multi_arm {
+    if inline_single_arm {
+        emit_inline_direct_method_shape_guard(
+            ctx,
+            recv_box,
+            &expected_class_id_str,
+            &expected_shape_id,
+            &fast_label,
+            &fallback_label,
+        );
+    }
+    let guard_ok = if multi_arm || inline_single_arm {
         // The chain above already terminated the guard block and every test
-        // block; `fast_idx` / `fallback_idx` are entered from it unchanged.
+        // block, or the inline single-arm guard terminated both its pointer
+        // gate and header block; `fast_idx` / `fallback_idx` are entered from
+        // either form unchanged.
         String::new()
     } else if shape_only_guard {
         ctx.block().call(
@@ -360,7 +476,7 @@ pub(super) fn emit_guarded_direct_method_call(
             ],
         )
     };
-    if !multi_arm {
+    if !multi_arm && !inline_single_arm {
         let guard_pass = ctx.block().icmp_ne(I32, &guard_ok, "0");
         ctx.block()
             .cond_br(&guard_pass, &fast_label, &fallback_label);

crates/perry-codegen/src/runtime_decls/objects.rs

@@ -29,6 +29,10 @@ pub fn declare_phase_b_objects(module: &mut LlModule) {
     // when it is non-zero (descriptors / typed-feedback in use). Defined in
     // perry-runtime as `PERRY_CLASS_FIELD_INLINE_GUARD_DISABLED`.
     module.add_external_global("PERRY_CLASS_FIELD_INLINE_GUARD_DISABLED", I8);
+    // Sticky runtime flag (i8, 0 = valid) for class-prototype method guards.
+    // Direct-method lowering reads it with acquire ordering before touching a
+    // receiver header; prototype mutation stores 1 with release ordering.
+    module.add_external_global("PERRY_CLASS_PROTOTYPE_FAST_GUARDS_INVALIDATED", I8);
     // #7834/#7873: process-global count of threads with per-object records.
     // `0` proves both per-object side tables are empty everywhere, so a
     // construction site can skip `js_gc_forget_object_layout` outright.