diff --git a/changelog.d/8702-inline-trusted-box-capture-access.md b/changelog.d/8702-inline-trusted-box-capture-access.md new file mode 100644 index 0000000000..552e4a6594 --- /dev/null +++ b/changelog.d/8702-inline-trusted-box-capture-access.md @@ -0,0 +1,8 @@ +--- +category: Performance +title: Inline trusted boxed capture access +--- + +Compiler-private exact-arrow clones now load validated boxed-capture pointers once at +entry and access their non-moving cells directly. Public closure bodies remain checked, +while the private path retains TDZ behavior and GC write barriers. diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index 3671c91491..10a6456b57 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -10,7 +10,7 @@ use crate::expr::FnCtx; use crate::module::LlModule; use crate::stmt; use crate::strings::StringPool; -use crate::types::{LlvmType, DOUBLE, I1, I32, I64, PTR}; +use crate::types::{LlvmType, DOUBLE, I1, I32, I64, I8, PTR}; use super::opts::CrossModuleCtx; use super::typed_abi::{ @@ -916,6 +916,41 @@ pub(super) fn compile_closure( }) }; + // The private exact-arrow clone is entered only after the runtime has + // verified the public closure identity and its compiler-installed raw-box + // capture mask. Capture slots never change. Load each box pointer once, + // before user code or a safepoint can relocate the closure, and retain the + // non-moving box pointer for the invocation. This removes the repeated + // checked closure-capture helper from hot callback bodies without caching + // the mutable VALUE stored inside the box. + let trusted_box_capture_ptrs = if trusted_box_captures { + let mut trusted = HashMap::new(); + let mut boxed_captures: Vec<_> = closure_captures + .iter() + .filter(|(id, _)| closure_boxed_vars.contains(id)) + .map(|(id, index)| (*id, *index)) + .collect(); + boxed_captures.sort_unstable_by_key(|(_, index)| *index); + if !boxed_captures.is_empty() { + let header_size = + crate::target_layout::closure_header_size_bytes(&cross_module.target_triple) + .to_string(); + let blk = lf.block_mut(0).expect("closure body has an entry block"); + let closure_ptr = blk.inttoptr(I64, "%this_closure"); + let captures_base = blk.gep(I8, &closure_ptr, &[(I64, &header_size)]); + for (id, index) in boxed_captures { + let index = index.to_string(); + let capture_slot = blk.gep(I64, &captures_base, &[(I64, &index)]); + let bits = blk.load(I64, &capture_slot); + let ptr = blk.inttoptr(I64, &bits); + trusted.insert(id, crate::expr::TrustedBoxCapturePtr { bits, ptr }); + } + } + trusted + } else { + HashMap::new() + }; + let mut ctx = FnCtx { func: lf, module_slug: crate::expr::native_region_slug(strings.module_prefix()), @@ -1000,6 +1035,7 @@ pub(super) fn compile_closure( local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), trusted_box_captures, + trusted_box_capture_ptrs, local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), object_literal_locals: HashSet::new(), diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index d478aae49b..120fca6108 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -828,6 +828,7 @@ pub(super) fn compile_module_entry( local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), trusted_box_captures: false, + trusted_box_capture_ptrs: HashMap::new(), local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), object_literal_locals: HashSet::new(), @@ -1522,6 +1523,7 @@ pub(super) fn compile_module_entry( local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), trusted_box_captures: false, + trusted_box_capture_ptrs: HashMap::new(), local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), object_literal_locals: HashSet::new(), diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 7c14e43216..b5550611ec 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -1082,6 +1082,7 @@ pub(super) fn compile_function( local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), trusted_box_captures: false, + trusted_box_capture_ptrs: HashMap::new(), local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), object_literal_locals: HashSet::new(), diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index 10f57df939..cab728750c 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -528,6 +528,7 @@ pub(super) fn compile_method( local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), trusted_box_captures: false, + trusted_box_capture_ptrs: HashMap::new(), local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), object_literal_locals: HashSet::new(), @@ -1780,6 +1781,7 @@ pub(super) fn compile_static_method( local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), trusted_box_captures: false, + trusted_box_capture_ptrs: HashMap::new(), local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), object_literal_locals: HashSet::new(), diff --git a/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs b/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs index ea48c8557f..e2d6c827d6 100644 --- a/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs +++ b/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs @@ -187,8 +187,26 @@ fn direct_arrow_gets_a_private_body_but_keeps_the_public_validation_path() { "{public}" ); - assert!(trusted.contains("@js_box_get_bits_trusted(")); - assert!(trusted.contains("@js_box_set_bits_trusted_no_barrier(")); + // The exact clone reads its immutable raw-box capture pointer once from + // the closure entry layout, then directly accesses the non-moving cell. + // The trusted getter remains only as the cold TDZ/suppression fallback; + // normal writes need no helper at all. + assert!(trusted.contains("getelementptr i8, ptr"), "{trusted}"); + assert!(trusted.contains(", i64 16"), "{trusted}"); + assert!(trusted.contains("inttoptr i64"), "{trusted}"); + assert!(trusted.contains("load i64, ptr"), "{trusted}"); + assert!(trusted.contains("store i64"), "{trusted}"); + assert!(trusted.contains(crate::nanbox::TAG_TDZ_I64), "{trusted}"); + assert!(trusted.contains("trusted_box.tdz"), "{trusted}"); + assert!(trusted.contains("@js_box_get_bits_trusted("), "{trusted}"); + assert!( + !trusted.contains("@js_box_set_bits_trusted_no_barrier("), + "{trusted}" + ); + assert!( + !trusted.contains("@js_closure_get_capture_bits("), + "{trusted}" + ); assert!(!trusted.contains("@js_box_get_bits(")); assert!(!trusted.contains("@js_box_set_bits(")); assert!(trusted.contains("@js_write_barrier(")); diff --git a/crates/perry-codegen/src/expr/literals_vars.rs b/crates/perry-codegen/src/expr/literals_vars.rs index 8ab7ed6964..ef58ba5166 100644 --- a/crates/perry-codegen/src/expr/literals_vars.rs +++ b/crates/perry-codegen/src/expr/literals_vars.rs @@ -22,8 +22,38 @@ use super::{ emit_shadow_slot_clear, emit_shadow_slot_update_for_expr, emit_write_barrier, is_global_this_builtin_function_name, lower_expr, lower_expr_as_i32, lower_pod_local_reassignment, materialize_pod_local, nanbox_string_inline, FnCtx, + TrustedBoxCapturePtr, }; +/// Load the current value from a compiler-proven raw box capture. +/// +/// The exact-arrow resolver has already validated `capture.ptr`, so the hot +/// path is a direct cell load. Preserve lexical TDZ behavior with a cold call +/// to the existing trusted accessor only for the reserved sentinel; that +/// helper owns both ReferenceError construction and Perry's internal TDZ +/// suppression window semantics. +fn load_trusted_box_capture_bits(ctx: &mut FnCtx<'_>, capture: &TrustedBoxCapturePtr) -> String { + let bits = ctx.block().load(I64, &capture.ptr); + let is_tdz = ctx.block().icmp_eq(I64, &bits, crate::nanbox::TAG_TDZ_I64); + let slow_idx = ctx.new_block("trusted_box.tdz"); + let merge_idx = ctx.new_block("trusted_box.read"); + let slow_label = ctx.block_label(slow_idx); + let merge_label = ctx.block_label(merge_idx); + let fast_label = ctx.block().label.clone(); + ctx.block().cond_br(&is_tdz, &slow_label, &merge_label); + + ctx.current_block = slow_idx; + let slow_bits = ctx + .block() + .call(I64, "js_box_get_bits_trusted", &[(I64, &capture.bits)]); + let slow_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + ctx.block() + .phi(I64, &[(&bits, &fast_label), (&slow_bits, &slow_end)]) +} + /// A box, closure cell, or module root is the storage for the source binding, /// not an alias of the string it currently owns. An ordinary read extracts a /// second copy of that value, so demote a heap string before it can outlive the @@ -413,12 +443,19 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { } // Captured by closure (from outer scope): if let Some(&capture_idx) = ctx.closure_captures.get(id) { - let closure_ptr = super::current_closure_ptr_value(ctx, "captured local")?; let idx_str = capture_idx.to_string(); // If the captured id is a boxed var, the capture slot holds a // raw box pointer. Read the capture, extract the box pointer, // and deref via js_box_get_bits. if ctx.boxed_vars.contains(id) { + if let Some(capture) = ctx.trusted_box_capture_ptrs.get(id).cloned() { + let bits = load_trusted_box_capture_bits(ctx, &capture); + let value = ctx.block().bitcast_i64_to_double(&bits); + demote_extracted_string_binding(ctx, *id, &value); + return Ok(value); + } + let closure_ptr = + super::current_closure_ptr_value(ctx, "captured boxed local")?; let getter = if ctx.trusted_box_captures { "js_box_get_bits_trusted" } else { @@ -435,6 +472,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { demote_extracted_string_binding(ctx, *id, &value); return Ok(value); } + let closure_ptr = super::current_closure_ptr_value(ctx, "captured local")?; let bits = ctx.block().call( I64, "js_closure_get_capture_bits", @@ -649,29 +687,38 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // Closure captures first (write through the runtime), then // locals, then module globals. if let Some(&capture_idx) = ctx.closure_captures.get(id) { - let closure_ptr = super::current_closure_ptr_value(ctx, "captured local set")?; let idx_str = capture_idx.to_string(); // Boxed captured var: read the box pointer from the // capture slot, then js_box_set_bits to update the shared // cell. Do NOT overwrite the capture slot — it holds // the box pointer, not the value. if ctx.boxed_vars.contains(id) { - let setter = if ctx.trusted_box_captures { - "js_box_set_bits_trusted_no_barrier" + if let Some(capture) = ctx.trusted_box_capture_ptrs.get(id).cloned() { + let v_bits = ctx.block().bitcast_double_to_i64(&v); + ctx.block().store(I64, &v_bits, &capture.ptr); + // Gen-GC Phase C2: barrier — box is the parent. + emit_write_barrier(ctx, &capture.bits, &v_bits); } else { - "js_box_set_bits" - }; - let blk = ctx.block(); - let box_ptr = blk.call( - I64, - "js_closure_get_capture_bits", - &[(I64, &closure_ptr), (I32, &idx_str)], - ); - let v_bits = blk.bitcast_double_to_i64(&v); - blk.call_void(setter, &[(I64, &box_ptr), (I64, &v_bits)]); - // Gen-GC Phase C2: barrier — box is the parent. - emit_write_barrier(ctx, &box_ptr, &v_bits); + let closure_ptr = + super::current_closure_ptr_value(ctx, "captured boxed local set")?; + let setter = if ctx.trusted_box_captures { + "js_box_set_bits_trusted_no_barrier" + } else { + "js_box_set_bits" + }; + let blk = ctx.block(); + let box_ptr = blk.call( + I64, + "js_closure_get_capture_bits", + &[(I64, &closure_ptr), (I32, &idx_str)], + ); + let v_bits = blk.bitcast_double_to_i64(&v); + blk.call_void(setter, &[(I64, &box_ptr), (I64, &v_bits)]); + // Gen-GC Phase C2: barrier — box is the parent. + emit_write_barrier(ctx, &box_ptr, &v_bits); + } } else { + let closure_ptr = super::current_closure_ptr_value(ctx, "captured local set")?; let v_bits = ctx.block().bitcast_double_to_i64(&v); ctx.block().call_void( "js_closure_set_capture_bits", @@ -782,7 +829,6 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { }; // Closure capture path: runtime get + add/sub + runtime set. if let Some(&capture_idx) = ctx.closure_captures.get(id) { - let closure_ptr = super::current_closure_ptr_value(ctx, "captured local update")?; let idx_str = capture_idx.to_string(); // Boxed captured var: deref box bits, modify, store back. // @@ -807,6 +853,20 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // activation therefore cannot become reusable inside the // nested user frame `coerce_old`/`step_new` may enter. if ctx.boxed_vars.contains(id) { + if let Some(capture) = ctx.trusted_box_capture_ptrs.get(id).cloned() { + let old_bits = load_trusted_box_capture_bits(ctx, &capture); + let old = ctx.block().bitcast_i64_to_double(&old_bits); + let old = coerce_old(ctx.block(), &old); + let new = step_new(ctx.block(), &old); + let new_bits = ctx.block().bitcast_double_to_i64(&new); + ctx.block().store(I64, &new_bits, &capture.ptr); + // Gen-GC Phase C2: `++`/`--` on a BigInt yields a heap + // pointer via js_numeric_step — barrier the box parent. + emit_write_barrier(ctx, &capture.bits, &new_bits); + return Ok(if *prefix { new } else { old }); + } + let closure_ptr = + super::current_closure_ptr_value(ctx, "captured boxed local update")?; let getter = if ctx.trusted_box_captures { "js_box_get_bits_trusted" } else { @@ -834,6 +894,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { emit_write_barrier(ctx, &box_ptr, &new_bits); return Ok(if *prefix { new } else { old }); } + let closure_ptr = super::current_closure_ptr_value(ctx, "captured local update")?; let old_bits = ctx.block().call( I64, "js_closure_get_capture_bits", diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 1b91d93e68..910ebdfa92 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -587,6 +587,12 @@ pub(crate) struct FnCtx<'a> { /// raw helpers. Public and dynamically dispatched closure bodies keep the /// defensive runtime registry validation. pub trusted_box_captures: bool, + /// Raw box capture pointers loaded once in the entry block of a + /// compiler-private exact-arrow clone. The capture slots are immutable, + /// and a live exact capture edge keeps each box cell alive and non-moving + /// for the invocation, so these SSA values remain valid across safepoints + /// even though the closure object itself may relocate. + pub trusted_box_capture_ptrs: std::collections::HashMap, /// Immutable local aliases of same-module function declarations. /// Calling one is semantically the same as calling its `FuncRef` directly; /// retain the runtime function object in the local for identity/property @@ -1484,6 +1490,14 @@ pub(crate) struct FnCtx<'a> { pub buffer_alias_base: u32, } +#[derive(Clone)] +pub(crate) struct TrustedBoxCapturePtr { + /// Integer form used as the write-barrier parent. + pub bits: String, + /// Opaque LLVM pointer used by direct box-cell loads and stores. + pub ptr: String, +} + /// (Issue #50) Info about a flat-folded const 2D int array. #[derive(Debug, Clone)] pub struct FlatConstInfo { diff --git a/crates/perry-codegen/src/target_layout.rs b/crates/perry-codegen/src/target_layout.rs index 7efbdd6a3a..897c3b13ae 100644 --- a/crates/perry-codegen/src/target_layout.rs +++ b/crates/perry-codegen/src/target_layout.rs @@ -88,6 +88,23 @@ pub fn object_header_size_bytes(_target_triple: &str) -> u64 { 16 } +/// `std::mem::size_of::()` for the +/// target. +/// +/// `ClosureHeader` is `repr(C)` and contains a pointer followed by two `u32` +/// fields. It is therefore 16 bytes on LP64 and 12 bytes on ILP32. Trusted +/// exact-arrow bodies use this offset to read compiler-installed raw box +/// capture pointers directly from their immutable capture slots. Keep the +/// target derivation here: using the compiler host's pointer width would make +/// cross-compiled arm64_32 watchOS closures read four bytes past the slot. +pub fn closure_header_size_bytes(target_triple: &str) -> u64 { + if target_is_ilp32(target_triple) { + 12 + } else { + 16 + } +} + /// Minimum number of inline field slots `perry-runtime` allocates for EVERY /// object, mirroring `perry_runtime::object::INLINE_SLOT_FLOOR`. /// @@ -230,6 +247,14 @@ mod tests { } } + #[test] + fn closure_header_size_tracks_target_pointer_width() { + assert_eq!(closure_header_size_bytes("aarch64-apple-darwin"), 16); + assert_eq!(closure_header_size_bytes("x86_64-unknown-linux-gnu"), 16); + assert_eq!(closure_header_size_bytes("arm64_32-apple-watchos"), 12); + assert_eq!(closure_header_size_bytes("wasm32-unknown-unknown"), 12); + } + #[test] fn object_header_size_matches_pointer_width() { // #8047 — 64-bit targets: 2×u32 + one pointer = 16.