From bce22f96fa10a8bf1378cbe1692b54d8a47379d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 24 Aug 2026 23:45:47 +0200 Subject: [PATCH 1/5] perf(codegen): deopt exact callbacks from versioned loops --- crates/perry-codegen/src/block.rs | 92 ++++++- .../src/codegen/artifact_context.rs | 1 + crates/perry-codegen/src/codegen/artifacts.rs | 31 +++ crates/perry-codegen/src/codegen/closure.rs | 49 +++- .../src/codegen/closure_collect.rs | 101 ++++++++ crates/perry-codegen/src/codegen/entry.rs | 4 + crates/perry-codegen/src/codegen/function.rs | 2 + .../src/codegen/index_method_clone_tests.rs | 14 +- crates/perry-codegen/src/codegen/method.rs | 18 ++ crates/perry-codegen/src/codegen/mod.rs | 7 + .../perry-codegen/src/codegen/string_pool.rs | 13 + .../src/codegen/trusted_box_callback_tests.rs | 142 +++++++++- crates/perry-codegen/src/dialect/mod.rs | 14 + crates/perry-codegen/src/expr/binary.rs | 1 + crates/perry-codegen/src/expr/mod.rs | 55 ++++ .../src/expr/property_get/generic_dispatch.rs | 1 + crates/perry-codegen/src/gc_call_effects.rs | 18 +- crates/perry-codegen/src/inst.rs | 17 ++ .../src/lower_call/early_branches.rs | 35 +++ .../property_get/dynamic_dispatch.rs | 15 +- crates/perry-codegen/src/root_reload.rs | 16 +- crates/perry-codegen/src/root_reload_tests.rs | 50 ++++ .../src/runtime_decls/strings.rs | 10 + crates/perry-codegen/src/stmt/loops.rs | 6 + .../src/stmt/versioned_indexed_loop.rs | 122 ++++++++- .../src/closure/dispatch/direct.rs | 70 +++++ crates/perry-runtime/src/closure/registry.rs | 47 ++++ .../versioned_indexed_loop_callback_deopt.rs | 245 ++++++++++++++++++ 28 files changed, 1169 insertions(+), 27 deletions(-) create mode 100644 crates/perry/tests/versioned_indexed_loop_callback_deopt.rs diff --git a/crates/perry-codegen/src/block.rs b/crates/perry-codegen/src/block.rs index a048f641c6..be7ad0c5f7 100644 --- a/crates/perry-codegen/src/block.rs +++ b/crates/perry-codegen/src/block.rs @@ -879,6 +879,7 @@ impl LlBlock { callee: "llvm.aarch64.fjcvtzs".to_string(), args: vec![("double", val.to_string())], cconv: None, + gc_leaf: false, }); return r; } @@ -1224,6 +1225,26 @@ impl LlBlock { } pub fn call(&mut self, ret_ty: LlvmType, func_name: &str, args: &[(LlvmType, &str)]) -> String { + self.call_with_gc_leaf(ret_ty, func_name, args, false) + } + + /// Direct-call counterpart of [`Self::call_indirect_gc_leaf`]. + pub fn call_gc_leaf( + &mut self, + ret_ty: LlvmType, + func_name: &str, + args: &[(LlvmType, &str)], + ) -> String { + self.call_with_gc_leaf(ret_ty, func_name, args, true) + } + + fn call_with_gc_leaf( + &mut self, + ret_ty: LlvmType, + func_name: &str, + args: &[(LlvmType, &str)], + gc_leaf: bool, + ) -> String { // #835 + #846: record this emission against the FFI provenance // registry. The driver consults the registry after all per-module // codegen finishes to auto-link the providing crate. @@ -1244,9 +1265,10 @@ impl LlBlock { if let Some((cont, lpad)) = self.eh_invoke_suffix(func_name) { let arg_str = format_args(args); let cc = cconv.map(|c| format!("{c} ")).unwrap_or_default(); + let leaf_attr = if gc_leaf { " \"gc-leaf-function\"" } else { "" }; self.emit(format!( - "{} = invoke {}{} @{}({}) to label %{} unwind label %{}", - r, cc, ret_ty, func_name, arg_str, cont, lpad + "{} = invoke {}{} @{}({}){} to label %{} unwind label %{}", + r, cc, ret_ty, func_name, arg_str, leaf_attr, cont, lpad )); self.emit_inline_label(&cont); } else { @@ -1256,6 +1278,7 @@ impl LlBlock { callee: func_name.to_string(), args: args.iter().map(|(t, v)| (*t, v.to_string())).collect(), cconv, + gc_leaf, }); } r @@ -1285,6 +1308,7 @@ impl LlBlock { callee: func_name.to_string(), args: args.iter().map(|(t, v)| (*t, v.to_string())).collect(), cconv, + gc_leaf: false, }); } } @@ -1306,15 +1330,38 @@ impl LlBlock { ret_ty: LlvmType, fn_ptr: &str, args: &[(LlvmType, &str)], + ) -> String { + self.call_indirect_with_gc_leaf(ret_ty, fn_ptr, args, false) + } + + /// Emit an indirect call whose caller-side native GC values need not be + /// relocated across the call. The target may still collect, so this must + /// only be used when every collecting return path makes those values dead. + pub fn call_indirect_gc_leaf( + &mut self, + ret_ty: LlvmType, + fn_ptr: &str, + args: &[(LlvmType, &str)], + ) -> String { + self.call_indirect_with_gc_leaf(ret_ty, fn_ptr, args, true) + } + + fn call_indirect_with_gc_leaf( + &mut self, + ret_ty: LlvmType, + fn_ptr: &str, + args: &[(LlvmType, &str)], + gc_leaf: bool, ) -> String { let r = self.reg(); // Indirect targets (closures, method pointers) can always throw. if let Some(lpad) = self.counter.current_eh_unwind_label() { let arg_str = format_args(args); let cont = format!("eh.cont{}", self.counter.next()); + let leaf_attr = if gc_leaf { " \"gc-leaf-function\"" } else { "" }; self.emit(format!( - "{} = invoke {} {}({}) to label %{} unwind label %{}", - r, ret_ty, fn_ptr, arg_str, cont, lpad + "{} = invoke {} {}({}){} to label %{} unwind label %{}", + r, ret_ty, fn_ptr, arg_str, leaf_attr, cont, lpad )); self.emit_inline_label(&cont); } else { @@ -1323,6 +1370,7 @@ impl LlBlock { ret: ret_ty, fptr: fn_ptr.to_string(), args: args.iter().map(|(t, v)| (*t, v.to_string())).collect(), + gc_leaf, }); } r @@ -1563,6 +1611,17 @@ mod tests { .contains("call double @js_nanbox_string(i64 %handle)")); } + #[test] + fn direct_gc_leaf_call_places_the_callsite_attribute_after_arguments() { + let mut b = fresh(); + let r = b.call_gc_leaf(DOUBLE, "guarded_reader", &[(I64, "%handle")]); + assert_eq!(r, "%r1"); + assert_eq!( + b.to_ir(), + "entry.0:\n %r1 = call double @guarded_reader(i64 %handle) \"gc-leaf-function\"" + ); + } + #[test] fn indirect_call_uses_opaque_pointer_syntax() { let mut b = fresh(); @@ -1574,6 +1633,18 @@ mod tests { ); } + #[test] + fn indirect_gc_leaf_call_places_the_callsite_attribute_after_arguments() { + let mut b = fresh(); + let r = + b.call_indirect_gc_leaf(DOUBLE, "%callback", &[(I64, "%closure"), (DOUBLE, "%arg")]); + assert_eq!(r, "%r1"); + assert_eq!( + b.to_ir(), + "entry.0:\n %r1 = call double %callback(i64 %closure, double %arg) \"gc-leaf-function\"" + ); + } + #[test] fn indirect_invoke_uses_opaque_pointer_syntax() { let mut b = fresh(); @@ -1586,6 +1657,19 @@ mod tests { ); } + #[test] + fn indirect_gc_leaf_invoke_places_the_attribute_before_the_successor() { + let mut b = fresh(); + b.counter.push_eh_scope("catch.0".to_string()); + let r = + b.call_indirect_gc_leaf(DOUBLE, "%callback", &[(I64, "%closure"), (DOUBLE, "%arg")]); + assert_eq!(r, "%r1"); + assert_eq!( + b.to_ir(), + "entry.0:\n %r1 = invoke double %callback(i64 %closure, double %arg) \"gc-leaf-function\" to label %eh.cont2 unwind label %catch.0\neh.cont2:" + ); + } + #[test] fn terminator_blocks_further_emits() { let mut b = fresh(); diff --git a/crates/perry-codegen/src/codegen/artifact_context.rs b/crates/perry-codegen/src/codegen/artifact_context.rs index cf65023593..bdc278dea7 100644 --- a/crates/perry-codegen/src/codegen/artifact_context.rs +++ b/crates/perry-codegen/src/codegen/artifact_context.rs @@ -55,6 +55,7 @@ pub(super) struct ModuleArtifactsCtx<'a> { pub closure_lengths: &'a HashMap, pub closure_arrow_functions: &'a HashSet, pub trusted_box_closures: &'a HashMap, + pub versioned_loop_callbacks: &'a HashSet, pub closures: &'a [(perry_hir::types::FuncId, perry_hir::Expr)], pub class_keys_init_data: &'a [(String, String, u32, Vec, Vec)], /// Keys global to `(class id, packed GcHeader word)` for inline `new`. diff --git a/crates/perry-codegen/src/codegen/artifacts.rs b/crates/perry-codegen/src/codegen/artifacts.rs index 11e9f4f7d7..0e2c24dc63 100644 --- a/crates/perry-codegen/src/codegen/artifacts.rs +++ b/crates/perry-codegen/src/codegen/artifacts.rs @@ -71,6 +71,7 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> { closure_lengths, closure_arrow_functions, trusted_box_closures, + versioned_loop_callbacks, closures, class_keys_init_data, class_header_image_inits, @@ -159,6 +160,7 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> { closure_rest_params, cross_module, false, + false, ) .with_context(|| format!("lowering closure func_id={}", func_id))?; if trusted_box_closures.contains_key(func_id) { @@ -184,9 +186,37 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> { closure_rest_params, cross_module, true, + false, ) .with_context(|| format!("lowering trusted-box closure func_id={}", func_id))?; } + if versioned_loop_callbacks.contains(func_id) { + compile_closure( + llmod, + *func_id, + closure_expr, + func_names, + strings, + class_table, + method_names, + module_globals, + opts.import_function_prefixes, + enum_table, + static_field_globals, + class_ids, + func_signatures, + func_synthetic_arguments, + module_prefix, + module_boxed_vars, + module_receiver_types, + &module_reassigned_locals, + closure_rest_params, + cross_module, + true, + true, + ) + .with_context(|| format!("lowering versioned-loop closure func_id={}", func_id))?; + } let done = closure_index + 1; if done == closures.len() || done % closure_progress_step == 0 { progress.items("closure bodies", done, closures.len(), closure_started); @@ -1977,6 +2007,7 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> { closure_lengths, closure_arrow_functions, trusted_box_closures, + versioned_loop_callbacks, &user_fn_wrapper_rest, closure_synthetic_arguments, &user_fn_wrapper_synthetic_arguments, diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index 0cd51ad02f..a2e6a22115 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -500,6 +500,7 @@ pub(super) fn compile_closure( closure_rest_params: &HashMap, cross_module: &CrossModuleCtx, trusted_box_captures: bool, + versioned_loop_callback: bool, ) -> Result<()> { // Destructure the closure expression. We trust that the caller // passes only `Expr::Closure` here (from `collect_closures_*`). @@ -568,13 +569,18 @@ pub(super) fn compile_closure( } else { public_llvm_name.clone() }; - let llvm_name = if trusted_box_captures { + let llvm_name = if versioned_loop_callback { + format!("{ordinary_body_name}$trusted_boxes$versioned_loop") + } else if trusted_box_captures { format!("{ordinary_body_name}$trusted_boxes") } else { ordinary_body_name }; - // Param list: i64 this_closure, then each param as double. + // Param list: i64 this_closure, then each param as double. The private + // versioned-loop clone reuses its proven-unused first callback parameter + // for the caller's stack context, so its ABI and register footprint stay + // identical to the ordinary trusted clone. let mut llvm_params: Vec<(LlvmType, String)> = Vec::with_capacity(params.len() + 1); llvm_params.push((I64, "%this_closure".to_string())); for p in params { @@ -634,6 +640,16 @@ pub(super) fn compile_closure( let _ = lf.create_block("entry"); + let versioned_loop_deopt_context = versioned_loop_callback.then(|| { + let scratch_param = params + .first() + .expect("versioned-loop callback selection requires a scratch parameter"); + let scratch_arg = format!("%arg{}", scratch_param.id); + let blk = lf.block_mut(0).expect("closure body has an entry block"); + let context_bits = blk.bitcast_double_to_i64(&scratch_arg); + blk.inttoptr(I64, &context_bits) + }); + let mut closure_boxed_vars: HashSet = closure_relevant_ids .iter() .filter(|id| module_boxed_vars.contains(id)) @@ -665,8 +681,19 @@ pub(super) fn compile_closure( // their types available inside the body. Without this, closures // that capture an array `items` and do `items.length` miss the // typed fast path and return undefined. - let mut local_types: HashMap = - params.iter().map(|p| (p.id, p.ty.clone())).collect(); + let mut local_types: HashMap = params + .iter() + .map(|p| { + ( + p.id, + if versioned_loop_callback { + perry_hir::types::Type::Any + } else { + p.ty.clone() + }, + ) + }) + .collect(); for id in &closure_relevant_ids { if let Some(ty) = module_receiver_types.get(id) { local_types.entry(*id).or_insert_with(|| ty.clone()); @@ -832,11 +859,13 @@ pub(super) fn compile_closure( &cross_module.compile_time_constants, &cross_module.module_dispatch, ); - if let Some(callback_shapes) = cross_module.array_callback_shapes.get(&func_id) { - native_facts - .shape_stability - .shape_proven_ptr_locals - .extend(callback_shapes.clone()); + if !versioned_loop_callback { + if let Some(callback_shapes) = cross_module.array_callback_shapes.get(&func_id) { + native_facts + .shape_stability + .shape_proven_ptr_locals + .extend(callback_shapes.clone()); + } } // Representation-selection context gates (see codegen/function.rs). @@ -1034,7 +1063,9 @@ pub(super) fn compile_closure( local_closure_func_ids: HashMap::new(), local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), + resolved_versioned_loop_callback_targets: HashMap::new(), trusted_box_captures, + versioned_loop_deopt_context, trusted_box_capture_ptrs, local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/closure_collect.rs b/crates/perry-codegen/src/codegen/closure_collect.rs index ab56a53de1..dc7645b0da 100644 --- a/crates/perry-codegen/src/codegen/closure_collect.rs +++ b/crates/perry-codegen/src/codegen/closure_collect.rs @@ -39,6 +39,107 @@ pub(crate) struct TrustedBoxClosure { pub boxed_capture_mask: u64, } +fn versioned_loop_value_expr( + expr: &perry_hir::Expr, + param_ids: &std::collections::HashSet, + capture_ids: &std::collections::HashSet, +) -> bool { + match expr { + perry_hir::Expr::Integer(_) | perry_hir::Expr::Number(_) => true, + perry_hir::Expr::LocalGet(id) => param_ids.contains(id) || capture_ids.contains(id), + perry_hir::Expr::PropertyGet { object, .. } => { + matches!(object.as_ref(), perry_hir::Expr::LocalGet(id) if param_ids.contains(id)) + } + perry_hir::Expr::Binary { + op: perry_hir::BinaryOp::Add, + left, + right, + } => { + versioned_loop_value_expr(left, param_ids, capture_ids) + && versioned_loop_value_expr(right, param_ids, capture_ids) + } + _ => false, + } +} + +fn versioned_loop_expr_uses_local(expr: &perry_hir::Expr, local_id: u32) -> bool { + match expr { + perry_hir::Expr::LocalGet(id) => *id == local_id, + perry_hir::Expr::PropertyGet { object, .. } => { + matches!(object.as_ref(), perry_hir::Expr::LocalGet(id) if *id == local_id) + } + perry_hir::Expr::Binary { left, right, .. } => { + versioned_loop_expr_uses_local(left, local_id) + || versioned_loop_expr_uses_local(right, local_id) + } + _ => false, + } +} + +/// Select exact arrow callbacks whose only source-level effect is replacing a +/// captured box with an additive expression over callback parameters. The +/// private clone forces parameter property reads through the descriptor-aware +/// generic PIC and poisons its caller's fast loop before any PIC or dynamic-+ +/// fallback can run user code. Everything outside this deliberately small +/// grammar keeps the ordinary guarded loop. +pub(crate) fn select_versioned_loop_callbacks( + closures: &[(perry_hir::types::FuncId, perry_hir::Expr)], + trusted_box_closures: &std::collections::HashMap, + module_boxed_vars: &std::collections::HashSet, + module_globals: &std::collections::HashMap, +) -> std::collections::HashSet { + closures + .iter() + .filter_map(|(func_id, expr)| { + if !trusted_box_closures.contains_key(func_id) { + return None; + } + let perry_hir::Expr::Closure { + params, + body, + captures, + captures_this: false, + captures_new_target: false, + is_arrow: true, + is_async: false, + is_generator: false, + .. + } = expr + else { + return None; + }; + let [perry_hir::Stmt::Expr(perry_hir::Expr::LocalSet(target, value))] = body.as_slice() + else { + return None; + }; + if !module_boxed_vars.contains(target) { + return None; + } + // The private body carries the caller's stack context in the first + // otherwise-unused callback argument. Reusing the public ABI keeps + // the context out of an extra live argument on every iteration. + let scratch_param = params.first()?; + if versioned_loop_expr_uses_local(value, scratch_param.id) { + return None; + } + let param_ids: std::collections::HashSet = + params.iter().map(|param| param.id).collect(); + let capture_ids: std::collections::HashSet = + crate::type_analysis::compute_auto_captures_with_globals( + params, + body, + captures, + module_globals, + ) + .into_iter() + .collect(); + (capture_ids.contains(target) + && versioned_loop_value_expr(value, ¶m_ids, &capture_ids)) + .then_some(*func_id) + }) + .collect() +} + fn collect_direct_call_closures_in_stmts( stmts: &[perry_hir::Stmt], out: &mut std::collections::HashSet, diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index 0012ed02c7..b175c445d1 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -827,7 +827,9 @@ pub(super) fn compile_module_entry( local_closure_func_ids: HashMap::new(), local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), + resolved_versioned_loop_callback_targets: HashMap::new(), trusted_box_captures: false, + versioned_loop_deopt_context: None, trusted_box_capture_ptrs: HashMap::new(), local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), @@ -1530,7 +1532,9 @@ pub(super) fn compile_module_entry( local_closure_func_ids: HashMap::new(), local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), + resolved_versioned_loop_callback_targets: HashMap::new(), trusted_box_captures: false, + versioned_loop_deopt_context: None, trusted_box_capture_ptrs: HashMap::new(), local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index a265be8f14..c6f5d3aff1 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -1081,7 +1081,9 @@ pub(super) fn compile_function( local_closure_func_ids: HashMap::new(), local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), + resolved_versioned_loop_callback_targets: HashMap::new(), trusted_box_captures: false, + versioned_loop_deopt_context: None, trusted_box_capture_ptrs: HashMap::new(), local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs index 452a3da5fa..d0c1812609 100644 --- a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs +++ b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs @@ -271,7 +271,19 @@ fn emit_versioned_checked_reader_loop() -> String { param(ENTITIES, "entities", Type::Array(Box::new(Type::Any))), param(COLUMN, "column", Type::Array(Box::new(Type::Any))), param(BOUND, "bound", Type::Number), - param(CALLBACK, "callback", Type::Any), + param( + CALLBACK, + "callback", + Type::Function(perry_hir::types::FunctionType { + params: vec![ + ("entity".to_string(), Type::Any, false), + ("value".to_string(), Type::Any, false), + ], + return_type: Box::new(Type::Void), + is_async: false, + is_generator: false, + }), + ), param(FILTER, "filter", Type::Any), ], Type::Void, diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index cc9d116252..a46beae2e7 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -387,7 +387,9 @@ pub(super) fn compile_method( local_closure_func_ids: HashMap::new(), local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), + resolved_versioned_loop_callback_targets: HashMap::new(), trusted_box_captures: false, + versioned_loop_deopt_context: None, trusted_box_capture_ptrs: HashMap::new(), local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), @@ -563,6 +565,7 @@ pub(super) fn compile_method( .map(|call| (call.source_param, call.arity)) .collect(); let mut resolved_callback_ptrs = HashMap::new(); + let mut resolved_versioned_callback_ptrs = HashMap::new(); for (source_param, arity) in callback_keys { let Some(source_slot) = ctx.locals.get(&source_param).cloned() else { continue; @@ -575,6 +578,12 @@ pub(super) fn compile_method( &[(I64, &source_handle), (I32, &arity.to_string())], ); resolved_callback_ptrs.insert((source_param, arity), fn_ptr); + let versioned_fn_ptr = ctx.block().call( + PTR, + "js_closure_resolve_versioned_loop_direct_call", + &[(I64, &source_handle), (I32, &arity.to_string())], + ); + resolved_versioned_callback_ptrs.insert((source_param, arity), versioned_fn_ptr); } for call in hoisted_callback_calls { let Some(fn_ptr) = resolved_callback_ptrs @@ -585,6 +594,13 @@ pub(super) fn compile_method( }; ctx.resolved_arrow_callback_targets .insert((call.callee_local, call.arity), fn_ptr); + if let Some(versioned_fn_ptr) = resolved_versioned_callback_ptrs + .get(&(call.source_param, call.arity)) + .cloned() + { + ctx.resolved_versioned_loop_callback_targets + .insert((call.callee_local, call.arity), versioned_fn_ptr); + } } super::arguments::materialize_arguments_object( @@ -1650,7 +1666,9 @@ pub(super) fn compile_static_method( local_closure_func_ids: HashMap::new(), local_closure_param_counts: HashMap::new(), resolved_arrow_callback_targets: HashMap::new(), + resolved_versioned_loop_callback_targets: HashMap::new(), trusted_box_captures: false, + versioned_loop_deopt_context: None, trusted_box_capture_ptrs: HashMap::new(), local_func_ref_ids: HashMap::new(), option_object_locals: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/mod.rs b/crates/perry-codegen/src/codegen/mod.rs index 7257ad8967..c79eab123c 100644 --- a/crates/perry-codegen/src/codegen/mod.rs +++ b/crates/perry-codegen/src/codegen/mod.rs @@ -2780,6 +2780,12 @@ pub fn compile_module(hir: &HirModule, opts: CompileOptions) -> Result> &module_globals, &trusted_box_exclusions, ); + let versioned_loop_callbacks = closure_collect::select_versioned_loop_callbacks( + &closures, + &trusted_box_closures, + &module_boxed_vars, + &module_globals, + ); // ---- Representation-selection Phase 2: specialized-ABI plan selection. // Runs AFTER the typed_abi clone sets so mutual exclusion is decidable; @@ -3378,6 +3384,7 @@ pub fn compile_module(hir: &HirModule, opts: CompileOptions) -> Result> closure_lengths: &closure_lengths, closure_arrow_functions: &closure_arrow_functions, trusted_box_closures: &trusted_box_closures, + versioned_loop_callbacks: &versioned_loop_callbacks, closures: &closures, class_keys_init_data: &class_keys_init_data, class_header_image_inits: &class_header_image_inits, diff --git a/crates/perry-codegen/src/codegen/string_pool.rs b/crates/perry-codegen/src/codegen/string_pool.rs index dfb89b5823..f424245bf3 100644 --- a/crates/perry-codegen/src/codegen/string_pool.rs +++ b/crates/perry-codegen/src/codegen/string_pool.rs @@ -132,6 +132,7 @@ pub(super) fn emit_string_pool( u32, super::closure_collect::TrustedBoxClosure, >, + versioned_loop_callbacks: &std::collections::HashSet, // Issue #653: wrappers (`__perry_wrap_`) for top-level user functions // that declare a rest param. Each entry is `(wrapper_symbol, fixed_arity)` // — the runtime side-table is keyed on the wrapper's func_ptr, NOT the @@ -1390,6 +1391,18 @@ pub(super) fn emit_string_pool( (I64, &plan.boxed_capture_mask.to_string()), ], ); + if versioned_loop_callbacks.contains(&fid) { + let versioned_ref = format!("{}$trusted_boxes$versioned_loop", public_ref); + blk.call_void( + "js_register_closure_versioned_loop_direct", + &[ + (PTR, &public_ref), + (PTR, &versioned_ref), + (I32, &plan.capture_count.to_string()), + (I64, &plan.boxed_capture_mask.to_string()), + ], + ); + } } // Issue #653: register `__perry_wrap_` wrappers for top-level user diff --git a/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs b/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs index e2d6c827d6..7a1e3fb74c 100644 --- a/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs +++ b/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs @@ -3,10 +3,10 @@ use crate::{compile_module, CompileOptions}; use perry_hir::types::{FunctionType, Type}; -use perry_hir::{Expr, Function, Module, ModuleInitKind, Param, Stmt, UpdateOp}; +use perry_hir::{BinaryOp, Expr, Function, Module, ModuleInitKind, Param, Stmt, UpdateOp}; use std::collections::{HashMap, HashSet}; -use super::closure_collect::select_trusted_box_closures; +use super::closure_collect::{select_trusted_box_closures, select_versioned_loop_callbacks}; const COUNT: u32 = 10; const CALLBACK: u32 = 20; @@ -126,6 +126,37 @@ fn callback_with(func_id: u32, params: Vec, body: Vec) -> Expr { } } +fn versioned_callback(func_id: u32) -> Expr { + const ENTITY: u32 = 29; + const POS: u32 = 30; + const VEL: u32 = 31; + let property = |id, name: &str| Expr::PropertyGet { + object: Box::new(Expr::LocalGet(id)), + property: name.to_string(), + byte_offset: 0, + }; + callback_with( + func_id, + vec![ + param(ENTITY, "entity", Type::Any), + param(POS, "pos", Type::Any), + param(VEL, "vel", Type::Any), + ], + vec![Stmt::Expr(Expr::LocalSet( + COUNT, + Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(COUNT)), + right: Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(property(POS, "x")), + right: Box::new(property(VEL, "vx")), + }), + }), + ))], + ) +} + fn select(closures: Vec<(u32, Expr)>, direct: impl IntoIterator) -> HashSet { select_trusted_box_closures( &closures, @@ -158,6 +189,33 @@ fn emit(direct_literal: bool) -> String { .expect("LLVM IR is UTF-8") } +fn emit_versioned_callback() -> String { + const VERSIONED_FUNC: u32 = 100; + let mut outer = outer_function(true); + let call = outer.body.last_mut().expect("outer call exists"); + let Stmt::Expr(Expr::Call { args, .. }) = call else { + panic!("outer tail is a call"); + }; + args[0] = versioned_callback(VERSIONED_FUNC); + + let mut module = Module::new("versioned_loop_callback.ts"); + module.init_kind = ModuleInitKind::Eager; + module.functions = vec![consume_function(), outer]; + module.init.push(Stmt::Expr(Expr::Call { + callee: Box::new(Expr::FuncRef(3)), + args: Vec::new(), + type_args: Vec::new(), + byte_offset: 0, + })); + let opts = CompileOptions { + emit_ir_only: true, + output_type: "executable".to_string(), + ..Default::default() + }; + String::from_utf8(compile_module(&module, opts).expect("fixture compiles")) + .expect("LLVM IR is UTF-8") +} + fn function_body(ir: &str, symbol: &str) -> String { let start = ir .lines() @@ -216,6 +274,86 @@ fn direct_arrow_gets_a_private_body_but_keeps_the_public_validation_path() { )); } +#[test] +fn additive_property_callback_gets_a_cold_deopting_private_body() { + let ir = emit_versioned_callback(); + let special = function_body( + &ir, + "perry_closure_versioned_loop_callback_ts__100$trusted_boxes$versioned_loop", + ); + assert!( + special.lines().next().is_some_and(|line| { + !line.contains("ptr %versioned_loop_deopt") && line.contains(" internal ") + }), + "the private ABI must reuse the unused first callback argument:\n{special}" + ); + assert!( + special.contains("pic.miss.call") + && special.contains("js_object_get_field_ic_miss") + && special.contains("guarded_add.dynamic") + && special.contains("versioned_callback.deopt.mark"), + "both observable cold arms must poison the loop before fallback:\n{special}" + ); + assert!(ir.contains( + "@js_register_closure_versioned_loop_direct(ptr @perry_closure_versioned_loop_callback_ts__100, ptr @perry_closure_versioned_loop_callback_ts__100$trusted_boxes$versioned_loop, i32 1, i64 1)" + )); +} + +#[test] +fn versioned_callback_selector_rejects_calls_and_heap_writes() { + const VERSIONED_FUNC: u32 = 100; + let eligible = versioned_callback(VERSIONED_FUNC); + let closures = vec![(VERSIONED_FUNC, eligible.clone())]; + let direct = HashSet::from([VERSIONED_FUNC]); + let boxed = HashSet::from([COUNT]); + let globals = HashMap::new(); + let trusted = + select_trusted_box_closures(&closures, &direct, &boxed, &globals, &HashSet::new()); + assert!( + select_versioned_loop_callbacks(&closures, &trusted, &boxed, &globals) + .contains(&VERSIONED_FUNC) + ); + + for rejected_body in [ + vec![Stmt::Expr(Expr::Call { + callee: Box::new(Expr::LocalGet(30)), + args: Vec::new(), + type_args: Vec::new(), + byte_offset: 0, + })], + vec![Stmt::Expr(Expr::PropertySet { + object: Box::new(Expr::LocalGet(30)), + property: "x".to_string(), + value: Box::new(Expr::Integer(1)), + })], + ] { + let rejected = callback_with( + VERSIONED_FUNC, + vec![param(30, "value", Type::Any)], + rejected_body, + ); + let closures = vec![(VERSIONED_FUNC, rejected)]; + assert!(select_versioned_loop_callbacks(&closures, &trusted, &boxed, &globals).is_empty()); + } + + let used_first_param = callback_with( + VERSIONED_FUNC, + vec![param(30, "value", Type::Any)], + vec![Stmt::Expr(Expr::LocalSet( + COUNT, + Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(COUNT)), + right: Box::new(Expr::LocalGet(30)), + }), + ))], + ); + let closures = vec![(VERSIONED_FUNC, used_first_param)]; + let trusted = + select_trusted_box_closures(&closures, &direct, &boxed, &globals, &HashSet::new()); + assert!(select_versioned_loop_callbacks(&closures, &trusted, &boxed, &globals).is_empty()); +} + #[test] fn closure_first_stored_as_a_value_does_not_get_a_trusted_body() { let ir = emit(false); diff --git a/crates/perry-codegen/src/dialect/mod.rs b/crates/perry-codegen/src/dialect/mod.rs index 87c3f3ad82..fab4c81b2e 100644 --- a/crates/perry-codegen/src/dialect/mod.rs +++ b/crates/perry-codegen/src/dialect/mod.rs @@ -1535,6 +1535,7 @@ impl<'ctx, 'm> FnReader<'ctx, 'm> { callee, args, cconv, + gc_leaf, } => { let mut argv: Vec = Vec::with_capacity(args.len()); let mut argtys: Vec = @@ -1572,6 +1573,12 @@ impl<'ctx, 'm> FnReader<'ctx, 'm> { // not something to normalize away silently. Some(cc) => bail!("unknown calling-convention token `{cc}`"), } + if *gc_leaf { + site.add_attribute( + inkwell::attributes::AttributeLoc::Function, + self.ctx.create_string_attribute("gc-leaf-function", ""), + ); + } if let Some(d) = dst { match site.try_as_basic_value() { inkwell::values::ValueKind::Basic(v) => self.def(d, v)?, @@ -1585,6 +1592,7 @@ impl<'ctx, 'm> FnReader<'ctx, 'm> { ret, fptr, args, + gc_leaf, } => { let mut argv: Vec = Vec::with_capacity(args.len()); let mut argtys: Vec = @@ -1602,6 +1610,12 @@ impl<'ctx, 'm> FnReader<'ctx, 'm> { .builder .build_indirect_call(fn_ty, fp, &argv, dst.trim_start_matches('%')) .map_err(be)?; + if *gc_leaf { + site.add_attribute( + inkwell::attributes::AttributeLoc::Function, + self.ctx.create_string_attribute("gc-leaf-function", ""), + ); + } match site.try_as_basic_value() { inkwell::values::ValueKind::Basic(v) => self.def(dst, v), other => bail!("typed indirect call expected a value, got {other:?}"), diff --git a/crates/perry-codegen/src/expr/binary.rs b/crates/perry-codegen/src/expr/binary.rs index f03b45ebf1..f799aa4088 100644 --- a/crates/perry-codegen/src/expr/binary.rs +++ b/crates/perry-codegen/src/expr/binary.rs @@ -227,6 +227,7 @@ fn lower_guarded_numeric_add(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result ctx.block().br(&merge_label); ctx.current_block = slow_idx; + crate::expr::emit_versioned_loop_callback_deopt(ctx); let slow_val = rebuild_add_tree(ctx, expr, values, &mut 0, false); let slow_end = ctx.block().label.clone(); ctx.block().br(&merge_label); diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index e9f48237eb..ede94ed330 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -582,12 +582,19 @@ pub(crate) struct FnCtx<'a> { /// parameters, indexed by callback local (including exact const aliases) /// and call arity. pub resolved_arrow_callback_targets: std::collections::HashMap<(u32, usize), String>, + /// Nullable compiler-private callback targets whose guarded cold arms + /// poison a versioned loop before they can run user code. + pub resolved_versioned_loop_callback_targets: std::collections::HashMap<(u32, usize), String>, /// This is an internal clone of a compiler-proven direct arrow body. Its /// boxed capture slots were installed through /// `js_closure_set_box_capture_ptr`, so captured-box accesses may use the /// raw helpers. Public and dynamically dispatched closure bodies keep the /// defensive runtime registry validation. pub trusted_box_captures: bool, + /// Stack context supplied only to a compiler-private versioned-loop + /// callback clone. Its cold arms record the exact resume index and poison + /// the caller's private counter before executing observable fallback code. + pub versioned_loop_deopt_context: Option, /// Raw box capture pointers loaded once in the entry block of a /// compiler-private exact-arrow clone. The capture slots are immutable, /// and a live exact capture edge keeps each box cell alive and non-moving @@ -1577,6 +1584,17 @@ pub(crate) struct VersionedIndexedMethodFact { pub method_guard_slot: String, } +#[derive(Clone, Debug)] +pub(crate) enum VersionedIndexedGuardMode { + Fingerprints, + CallbackDeopt { + callback_local_id: u32, + callback_arity: usize, + target: String, + context: String, + }, +} + #[derive(Clone, Debug)] pub(crate) struct VersionedIndexedLoopFact { pub counter_local_id: u32, @@ -1584,6 +1602,7 @@ pub(crate) struct VersionedIndexedLoopFact { pub side_exit_label: String, pub arrays: Vec, pub method: VersionedIndexedMethodFact, + pub guard_mode: VersionedIndexedGuardMode, /// Populated by the iteration-entry revalidation block. These SSA handles /// dominate the complete fast body and are never retained across the loop /// callback/back edge. @@ -1966,6 +1985,42 @@ pub(crate) fn inline_cache_global_name(ctx: &FnCtx<'_>, site_id: u32) -> String inline_cache_global_name_for_prefix(ctx.strings.module_prefix(), site_id) } +/// Record a cold-arm bailout for a compiler-private versioned-loop callback. +/// The stack context is `[counter_slot_ptr, original_bound, resume_index]` as +/// three i64 words. The first cold arm stores `counter + 1` and poisons the +/// private i32 counter to `bound - 1`; the caller's ordinary update advances +/// it to `bound`, so the existing loop condition exits without a hot-path +/// check. Later cold arms in the same callback are idempotent. +pub(crate) fn emit_versioned_loop_callback_deopt(ctx: &mut FnCtx<'_>) { + let Some(context) = ctx.versioned_loop_deopt_context.clone() else { + return; + }; + let resume_ptr = ctx.block().gep(I64, &context, &[(I64, "2")]); + let resume = ctx.block().load(I64, &resume_ptr); + let unmarked = ctx.block().icmp_eq(I64, &resume, "-1"); + let mark_idx = ctx.new_block("versioned_callback.deopt.mark"); + let continue_idx = ctx.new_block("versioned_callback.deopt.continue"); + let mark_label = ctx.block_label(mark_idx); + let continue_label = ctx.block_label(continue_idx); + ctx.block().cond_br(&unmarked, &mark_label, &continue_label); + + ctx.current_block = mark_idx; + let counter_slot_ptr = ctx.block().gep(I64, &context, &[(I64, "0")]); + let counter_slot_bits = ctx.block().load(I64, &counter_slot_ptr); + let counter_slot = ctx.block().inttoptr(I64, &counter_slot_bits); + let counter = ctx.block().load(I32, &counter_slot); + let next = ctx.block().add(I32, &counter, "1"); + let next_i64 = ctx.block().zext(I32, &next, I64); + ctx.block().store(I64, &next_i64, &resume_ptr); + let bound_ptr = ctx.block().gep(I64, &context, &[(I64, "1")]); + let bound_i64 = ctx.block().load(I64, &bound_ptr); + let bound = ctx.block().trunc(I64, &bound_i64, I32); + let poison = ctx.block().sub(I32, &bound, "1"); + ctx.block().store(I32, &poison, &counter_slot); + ctx.block().br(&continue_label); + ctx.current_block = continue_idx; +} + fn inline_cache_global_name_for_prefix(module_prefix: &str, site_id: u32) -> String { if module_prefix.is_empty() { format!("perry_ic_{site_id}") diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index bd1dbf2057..28bdc9c7f4 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -651,6 +651,7 @@ pub(crate) fn lower_generic_property_get( // PIC miss: slow path with cache population. ctx.current_block = call_idx; + crate::expr::emit_versioned_loop_callback_deopt(ctx); let miss_key_handle = emit_key_handle(ctx, &key_handle_global); let val_miss = ctx.block().call( DOUBLE, diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index ac65c3f397..7eeeb089ba 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -889,7 +889,10 @@ mod tests { let wrapper = void_function("wrapper", &["leaf"]); let recursive_a = void_function("recursive_a", &["recursive_b"]); let recursive_b = void_function("recursive_b", &["recursive_a"]); - let allocating = void_function("allocating", &["js_array_alloc"]); + let mut allocating = LlFunction::new("allocating", crate::types::I64, vec![]); + let entry = allocating.create_block("entry"); + entry.call_void("js_array_alloc", &[]); + entry.ret(crate::types::I64, "0"); let reaches_allocating = void_function("reaches_allocating", &["allocating"]); let functions = [ &leaf, @@ -919,12 +922,21 @@ mod tests { let entry = indirect.create_block("entry"); entry.call_indirect(crate::types::I64, "%callback", &[]); entry.ret_void(); - let functions = [&unknown, &indirect]; + let mut guarded = LlFunction::new("guarded", crate::types::VOID, vec![]); + let entry = guarded.create_block("entry"); + entry.call_indirect_gc_leaf(crate::types::I64, "%callback", &[]); + entry.ret_void(); + let allocating = void_function("allocating", &["js_array_alloc"]); + let mut guarded_direct = LlFunction::new("guarded_direct", crate::types::VOID, vec![]); + let entry = guarded_direct.create_block("entry"); + entry.call_gc_leaf(crate::types::I64, "allocating", &[]); + entry.ret_void(); + let functions = [&unknown, &indirect, &guarded, &allocating, &guarded_direct]; let safe = transitive_leaf_functions(&functions); assert!( safe.is_empty(), - "unknown and indirect calls must fail closed" + "unknown, indirect, and collecting direct calls must fail closed; a guarded call-site marker must not turn its containing function transitively leaf" ); } diff --git a/crates/perry-codegen/src/inst.rs b/crates/perry-codegen/src/inst.rs index fbbee1c003..6988ac2906 100644 --- a/crates/perry-codegen/src/inst.rs +++ b/crates/perry-codegen/src/inst.rs @@ -123,6 +123,10 @@ pub enum LlInst { /// for the default C convention. Must match the callee's define /// header — a mismatch is UB, not a verifier error (#8175). cconv: Option<&'static str>, + /// Call-site relocation exemption. The callee still participates in + /// transitive effect analysis; this flag only controls caller-side + /// root relocation and the emitted RS4GC attribute. + gc_leaf: bool, }, /// Opaque-pointer indirect call. The callee operand itself is a `ptr`; /// argument types remain explicit at the call site. @@ -131,6 +135,11 @@ pub enum LlInst { ret: LlvmType, fptr: String, args: Vec<(LlvmType, String)>, + /// Call-site promise to RS4GC that no relocated value is observed + /// after a collecting arm returns. This is narrower than a function + /// effect: transitive leaf analysis must still treat the opaque target + /// as collecting. + gc_leaf: bool, }, /// `call void asm sideeffect "", ""()` — the loop-preservation barrier. AsmBarrier, @@ -277,6 +286,7 @@ impl LlInst { callee, args, cconv, + gc_leaf, } => { out.push_str(" "); if let Some(d) = dst { @@ -292,16 +302,23 @@ impl LlInst { } push_args(out, args); out.push(')'); + if *gc_leaf { + out.push_str(" \"gc-leaf-function\""); + } } LlInst::CallIndirect { dst, ret, fptr, args, + gc_leaf, } => { let _ = write!(out, " {dst} = call {ret} {fptr}("); push_args(out, args); out.push(')'); + if *gc_leaf { + out.push_str(" \"gc-leaf-function\""); + } } LlInst::AsmBarrier => { // `"gc-leaf-function"` exempts the barrier from diff --git a/crates/perry-codegen/src/lower_call/early_branches.rs b/crates/perry-codegen/src/lower_call/early_branches.rs index 51fa4ac894..479a86678d 100644 --- a/crates/perry-codegen/src/lower_call/early_branches.rs +++ b/crates/perry-codegen/src/lower_call/early_branches.rs @@ -459,6 +459,41 @@ pub fn try_lower_closure_typed_local_call( // Exact immutable aliases (`const cb = callback`) have the same // identity whenever their read succeeds. A TDZ read throws while // lowering `callee` above, before this dispatch arm is reached. + if let Some(crate::expr::VersionedIndexedGuardMode::CallbackDeopt { + callback_local_id, + callback_arity, + target, + context, + .. + }) = ctx + .versioned_indexed_loop_facts + .last() + .map(|fact| fact.guard_mode.clone()) + { + if callback_local_id == *id && callback_arity == lowered_args.len() { + let context_bits = ctx.block().ptrtoint(&context, I64); + let context_box = ctx.block().bitcast_i64_to_double(&context_bits); + let mut direct_args: Vec<(crate::types::LlvmType, &str)> = + Vec::with_capacity(lowered_args.len() + 1); + direct_args.push((I64, &closure_handle)); + direct_args.extend(lowered_args.iter().enumerate().map(|(index, value)| { + if index == 0 { + (DOUBLE, context_box.as_str()) + } else { + (DOUBLE, value.as_str()) + } + })); + // The exact clone marks the caller's counter for an + // immediate side exit before any cold arm that can run + // user code or collect. Hot returns cannot collect; cold + // returns never observe caller-side cached heap handles. + let value = ctx + .block() + .call_indirect_gc_leaf(DOUBLE, &target, &direct_args); + callee_group.release(ctx); + return Ok(Some(value)); + } + } if let Some(target) = ctx .resolved_arrow_callback_targets .get(&(*id, lowered_args.len())) diff --git a/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs b/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs index ed6316c519..87da9ed45d 100644 --- a/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs +++ b/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs @@ -1376,7 +1376,20 @@ pub(crate) fn try_lower_instance_method_call( &fallback_fn, params, ); - return Ok(Some(ctx.block().call(DOUBLE, &target, &fast_args))); + let value = if matches!( + fact.guard_mode, + crate::expr::VersionedIndexedGuardMode::CallbackDeopt { .. } + ) { + // Admission proved every private handle argument + // is a live in-bounds array. The checked-reader + // clone's only collecting arms are therefore + // unreachable in this loop version; a callback + // cold arm side-exits before the next use. + ctx.block().call_gc_leaf(DOUBLE, &target, &fast_args) + } else { + ctx.block().call(DOUBLE, &target, &fast_args) + }; + return Ok(Some(value)); } } let typed_receiver_direct = match ( diff --git a/crates/perry-codegen/src/root_reload.rs b/crates/perry-codegen/src/root_reload.rs index e117c762e7..dd01493de8 100644 --- a/crates/perry-codegen/src/root_reload.rs +++ b/crates/perry-codegen/src/root_reload.rs @@ -926,13 +926,17 @@ fn facts_of(inst: &LlInst, slots: &HashSet) -> Facts { use_op(&mut uses, b); } LlInst::Call { - dst, callee, args, .. + dst, + callee, + args, + gc_leaf, + .. } => { result = dst.as_deref().and_then(reg); for (_, v) in args { use_op(&mut uses, v); } - collecting = is_collecting(callee); + collecting = !gc_leaf && is_collecting(callee); // #7725: the two capture-bits halves. GET extends a derivation like a transparent // bit op (see CAPTURE_GET_CALLEE); SET's side effect on the capture slot has to // invalidate that derivation the way a store does, via the shared `stores_to` @@ -949,14 +953,18 @@ fn facts_of(inst: &LlInst, slots: &HashSet) -> Facts { } } LlInst::CallIndirect { - dst, fptr, args, .. + dst, + fptr, + args, + gc_leaf, + .. } => { result = reg(dst); use_op(&mut uses, fptr); for (_, v) in args { use_op(&mut uses, v); } - collecting = true; + collecting = !gc_leaf; } LlInst::AsmBarrier => {} LlInst::Br { label } => succs.push(label.clone()), diff --git a/crates/perry-codegen/src/root_reload_tests.rs b/crates/perry-codegen/src/root_reload_tests.rs index b9ec74e65d..86bfcff312 100644 --- a/crates/perry-codegen/src/root_reload_tests.rs +++ b/crates/perry-codegen/src/root_reload_tests.rs @@ -83,6 +83,56 @@ fn a_non_collecting_window_is_left_byte_for_byte_alone() { assert_eq!(body(&f), before); } +#[test] +fn guarded_indirect_leaf_call_does_not_reload_a_dead_on_collecting_return_handle() { + let mut f = LlFunction::new("t", DOUBLE, vec![(DOUBLE, "%arg".into())]); + let b = f.create_block("entry"); + let slot = b.alloca(DOUBLE); + b.store(DOUBLE, "%arg", &slot); + b.call_void( + "js_shadow_slot_bind", + &[(crate::types::I32, "0"), (PTR, &slot)], + ); + let value = b.load(DOUBLE, &slot); + b.call_indirect_gc_leaf(DOUBLE, "%callback", &[]); + let result = b.call( + DOUBLE, + "js_object_assign_one", + &[(DOUBLE, &value), (DOUBLE, "0.0")], + ); + b.ret(DOUBLE, &result); + + let before = body(&f); + assert_eq!(apply_to_function(&mut f), 0); + assert_eq!(body(&f), before); + assert!(before.contains("\"gc-leaf-function\"")); +} + +#[test] +fn guarded_direct_leaf_call_does_not_reload_a_proven_live_handle() { + let mut f = LlFunction::new("t", DOUBLE, vec![(DOUBLE, "%arg".into())]); + let b = f.create_block("entry"); + let slot = b.alloca(DOUBLE); + b.store(DOUBLE, "%arg", &slot); + b.call_void( + "js_shadow_slot_bind", + &[(crate::types::I32, "0"), (PTR, &slot)], + ); + let value = b.load(DOUBLE, &slot); + b.call_gc_leaf(DOUBLE, "guarded_reader", &[]); + let result = b.call( + DOUBLE, + "js_object_assign_one", + &[(DOUBLE, &value), (DOUBLE, "0.0")], + ); + b.ret(DOUBLE, &result); + + let before = body(&f); + assert_eq!(apply_to_function(&mut f), 0); + assert_eq!(body(&f), before); + assert!(before.contains("call double @guarded_reader() \"gc-leaf-function\"")); +} + #[test] fn a_slot_the_program_reassigns_in_the_window_is_not_reloaded() { // ★ The soundness half. `f(x, (x = other, 1))` must pass the ORIGINAL diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index d0b622dab4..e67b00515d 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -227,6 +227,16 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { VOID, &[PTR, PTR, I32, I64], ); + module.declare_function( + "js_register_closure_versioned_loop_direct", + VOID, + &[PTR, PTR, I32, I64], + ); + module.declare_function( + "js_closure_resolve_versioned_loop_direct_call", + PTR, + &[I64, I32], + ); module.declare_function("js_register_closure_strict_function", VOID, &[PTR]); module.declare_function("js_register_closure_async_function", VOID, &[PTR]); module.declare_function("js_register_closure_generator_function", VOID, &[PTR]); diff --git a/crates/perry-codegen/src/stmt/loops.rs b/crates/perry-codegen/src/stmt/loops.rs index a952e0a96e..cc2f04ad2b 100644 --- a/crates/perry-codegen/src/stmt/loops.rs +++ b/crates/perry-codegen/src/stmt/loops.rs @@ -5638,6 +5638,12 @@ pub(crate) fn emit_gc_loop_safepoint( if !ctx.element_shape_loop_facts.is_empty() || !ctx.class_field_loop_facts.is_empty() || !ctx.stable_packed_loop_facts.is_empty() + || ctx.versioned_indexed_loop_facts.last().is_some_and(|fact| { + matches!( + fact.guard_mode, + crate::expr::VersionedIndexedGuardMode::CallbackDeopt { .. } + ) + }) { return; } diff --git a/crates/perry-codegen/src/stmt/versioned_indexed_loop.rs b/crates/perry-codegen/src/stmt/versioned_indexed_loop.rs index f8079d7cf0..81b84db586 100644 --- a/crates/perry-codegen/src/stmt/versioned_indexed_loop.rs +++ b/crates/perry-codegen/src/stmt/versioned_indexed_loop.rs @@ -14,7 +14,8 @@ use anyhow::Result; use perry_hir::{CompareOp, Expr, LogicalOp, Stmt, UpdateOp}; use crate::expr::{ - FnCtx, VersionedIndexedArrayFact, VersionedIndexedLoopFact, VersionedIndexedMethodFact, + FnCtx, VersionedIndexedArrayFact, VersionedIndexedGuardMode, VersionedIndexedLoopFact, + VersionedIndexedMethodFact, }; use crate::types::{DOUBLE, I1, I128, I16, I32, I64, I8}; @@ -26,6 +27,8 @@ struct Candidate { arrays: Vec, class_name: String, method_name: String, + callback_id: u32, + callback_arity: usize, } fn checked_reader_call( @@ -233,6 +236,8 @@ fn match_candidate( arrays: arrays.into_iter().collect(), class_name, method_name, + callback_id, + callback_arity: callback_args.len(), }) } @@ -352,6 +357,17 @@ pub(super) fn emit_iteration_guard(ctx: &mut FnCtx<'_>) -> bool { let Some(fact) = ctx.versioned_indexed_loop_facts.last().cloned() else { return false; }; + if matches!( + fact.guard_mode, + VersionedIndexedGuardMode::CallbackDeopt { .. } + ) { + // The callback clone's hot path cannot collect. Every cold arm first + // poisons the private counter so the loop exits without another body + // iteration. The preheader handles therefore remain live exactly on + // the paths which can use them; reloading shadow roots here would add + // two loads and masks to every ECS entity. + return true; + } let continue_idx = ctx.new_block("versioned_index.iteration.fast"); let continue_label = ctx.block_label(continue_idx); let array_invalidated = ctx @@ -431,11 +447,32 @@ pub(super) fn lower( return Ok(false); }; - let fast_pre_idx = ctx.new_block("versioned_index.loop.fast.preheader"); + // A cold callback arm may collect and then throw. Keep the specialized + // call out of an active EH scope: a local catch/finally could otherwise + // observe caller roots across the call's unwind edge. Ordinary guarded + // loop versioning remains available there. + let versioned_callback_target = (ctx.try_depth == 0 + && ctx.i32_counter_slots.contains_key(&candidate.counter_id)) + .then(|| { + ctx.resolved_versioned_loop_callback_targets + .get(&(candidate.callback_id, candidate.callback_arity)) + .cloned() + }) + .flatten(); + let guarded_pre_idx = ctx.new_block("versioned_index.loop.fast.preheader"); + let callback_pre_idx = versioned_callback_target + .as_ref() + .map(|_| ctx.new_block("versioned_index.loop.callback.preheader")); + let fast_pre_idx = if callback_pre_idx.is_some() { + ctx.new_block("versioned_index.loop.fast.dispatch") + } else { + guarded_pre_idx + }; let slow_pre_idx = ctx.new_block("versioned_index.loop.slow.preheader"); let merge_idx = ctx.new_block("versioned_index.loop.merge"); let convert_idx = ctx.new_block("versioned_index.bound.convert"); let fast_pre_label = ctx.block_label(fast_pre_idx); + let guarded_pre_label = ctx.block_label(guarded_pre_idx); let slow_pre_label = ctx.block_label(slow_pre_idx); let merge_label = ctx.block_label(merge_idx); let convert_label = ctx.block_label(convert_idx); @@ -552,7 +589,85 @@ pub(super) fn lower( expected_shape_id, method_guard_slot, }; - ctx.current_block = fast_pre_idx; + if let (Some(target), Some(callback_pre_idx)) = (versioned_callback_target, callback_pre_idx) { + let callback_pre_label = ctx.block_label(callback_pre_idx); + ctx.current_block = fast_pre_idx; + let target_is_exact = ctx.block().icmp_ne(crate::types::PTR, &target, "null"); + ctx.block() + .cond_br(&target_is_exact, &callback_pre_label, &guarded_pre_label); + + ctx.current_block = callback_pre_idx; + let counter_i32_slot = ctx + .i32_counter_slots + .get(&candidate.counter_id) + .expect("matched integer counter has i32 storage") + .clone(); + let deopt_context = ctx.func.alloca_entry_array(I64, 3); + let counter_ptr_bits = ctx.block().ptrtoint(&counter_i32_slot, I64); + let context_counter_ptr = ctx.block().gep(I64, &deopt_context, &[(I64, "0")]); + ctx.block() + .store(I64, &counter_ptr_bits, &context_counter_ptr); + let context_bound_ptr = ctx.block().gep(I64, &deopt_context, &[(I64, "1")]); + let bound_i64 = ctx.block().zext(I32, &bound_i32, I64); + ctx.block().store(I64, &bound_i64, &context_bound_ptr); + let context_resume_ptr = ctx.block().gep(I64, &deopt_context, &[(I64, "2")]); + ctx.block().store(I64, "-1", &context_resume_ptr); + + let mut callback_live_handles = HashMap::new(); + for array in &array_facts { + let array_box = ctx.block().load(DOUBLE, &array.local_slot); + let array_bits = ctx.block().bitcast_double_to_i64(&array_box); + let array_handle = ctx + .block() + .and(I64, &array_bits, crate::nanbox::POINTER_MASK_I64); + callback_live_handles.insert(array.local_id, array_handle); + } + + ctx.versioned_indexed_loop_facts + .push(VersionedIndexedLoopFact { + counter_local_id: candidate.counter_id, + falsy_local_id: candidate.filter_id, + side_exit_label: slow_pre_label.clone(), + arrays: array_facts.clone(), + method: method_fact.clone(), + guard_mode: VersionedIndexedGuardMode::CallbackDeopt { + callback_local_id: candidate.callback_id, + callback_arity: candidate.callback_arity, + target, + context: deopt_context, + }, + live_array_handles: callback_live_handles, + }); + super::loops::lower_for_after_init_with_i32_bound( + ctx, + init, + condition, + update, + body, + "for.versioned_index_callback", + Some((candidate.counter_id, bound_i32.clone())), + )?; + ctx.versioned_indexed_loop_facts.pop(); + if !ctx.block().is_terminated() { + let resume = ctx.block().load(I64, &context_resume_ptr); + let completed_without_deopt = ctx.block().icmp_eq(I64, &resume, "-1"); + let resume_idx = ctx.new_block("versioned_index.loop.callback.resume"); + let resume_label = ctx.block_label(resume_idx); + ctx.block() + .cond_br(&completed_without_deopt, &merge_label, &resume_label); + + ctx.current_block = resume_idx; + let resume_i32 = ctx.block().trunc(I64, &resume, I32); + ctx.block().store(I32, &resume_i32, &counter_i32_slot); + if let Some(counter_slot) = ctx.locals.get(&candidate.counter_id).cloned() { + let resume_f64 = ctx.block().sitofp(I32, &resume_i32, DOUBLE); + ctx.block().store(DOUBLE, &resume_f64, &counter_slot); + } + ctx.block().br(&slow_pre_label); + } + } + + ctx.current_block = guarded_pre_idx; ctx.versioned_indexed_loop_facts .push(VersionedIndexedLoopFact { counter_local_id: candidate.counter_id, @@ -560,6 +675,7 @@ pub(super) fn lower( side_exit_label: slow_pre_label.clone(), arrays: array_facts, method: method_fact, + guard_mode: VersionedIndexedGuardMode::Fingerprints, live_array_handles: HashMap::new(), }); super::loops::lower_for_after_init_with_i32_bound( diff --git a/crates/perry-runtime/src/closure/dispatch/direct.rs b/crates/perry-runtime/src/closure/dispatch/direct.rs index fe066eac08..d3e42cc6ab 100644 --- a/crates/perry-runtime/src/closure/dispatch/direct.rs +++ b/crates/perry-runtime/src/closure/dispatch/direct.rs @@ -116,6 +116,40 @@ pub extern "C" fn js_closure_resolve_arrow_direct_call( trusted.func_ptr } +/// Resolve only a compiler-private versioned-loop callback clone. A runtime +/// closure must match the registered capture layout exactly; any other arrow, +/// ordinary function, rest/padded call, or forged capture falls back. +#[no_mangle] +pub extern "C" fn js_closure_resolve_versioned_loop_direct_call( + closure: *const ClosureHeader, + arity: u32, +) -> *const u8 { + let Some(func_ptr) = resolve_direct_func_ptr(closure, arity) else { + return std::ptr::null(); + }; + if !resolve_strategy(func_ptr).is_arrow() { + return std::ptr::null(); + } + let Some(target) = super::super::registry::lookup_closure_versioned_loop_direct(func_ptr) + else { + return std::ptr::null(); + }; + let actual_capture_count = unsafe { real_capture_count((*closure).capture_count) }; + if actual_capture_count != target.capture_count { + return std::ptr::null(); + } + let mut mask = target.boxed_capture_mask; + while mask != 0 { + let index = mask.trailing_zeros(); + let box_ptr = crate::closure::js_closure_get_capture_bits(closure, index); + if crate::r#box::box_slot_contents_bits(box_ptr).is_none() { + return std::ptr::null(); + } + mask &= mask - 1; + } + target.func_ptr +} + macro_rules! define_direct_call_site { ( $(#[$meta:meta])* @@ -236,6 +270,14 @@ mod tests { value } + extern "C" fn versioned_source(_c: *const ClosureHeader, value: f64) -> f64 { + value + } + + extern "C" fn versioned_boxed1(_c: *const ClosureHeader, value: f64, _deopt: *mut u64) -> f64 { + value + } + fn closure_for(body: *const u8) -> *const ClosureHeader { crate::closure::js_closure_alloc(body, 0) } @@ -318,6 +360,34 @@ mod tests { ); } + #[test] + fn versioned_target_is_exact_and_fails_closed() { + crate::closure::js_register_closure_arity(versioned_source as *const u8, 1); + crate::closure::js_register_closure_arrow_function(versioned_source as *const u8); + super::super::registry::js_register_closure_versioned_loop_direct( + versioned_source as *const u8, + versioned_boxed1 as *const u8, + 1, + 1, + ); + + let wrong_count = closure_for(versioned_source as *const u8); + assert!(js_closure_resolve_versioned_loop_direct_call(wrong_count, 1).is_null()); + + let non_box = crate::closure::js_closure_alloc(versioned_source as *const u8, 1); + crate::closure::js_closure_set_capture_bits(non_box, 0, crate::value::TAG_UNDEFINED); + assert!(js_closure_resolve_versioned_loop_direct_call(non_box, 1).is_null()); + + let valid = crate::closure::js_closure_alloc(versioned_source as *const u8, 1); + let cell = crate::r#box::js_box_alloc_bits(crate::value::TAG_UNDEFINED as i64); + crate::closure::js_closure_set_box_capture_ptr(valid, 0, cell as i64); + assert_eq!( + js_closure_resolve_versioned_loop_direct_call(valid, 1), + versioned_boxed1 as *const u8 + ); + assert!(js_closure_resolve_versioned_loop_direct_call(valid, 0).is_null()); + } + #[test] fn a_plain_callback_resolves_and_answers_identically_to_the_slow_path() { let c = closure_for(add3 as *const u8); diff --git a/crates/perry-runtime/src/closure/registry.rs b/crates/perry-runtime/src/closure/registry.rs index 56bf18e62c..fb977d5340 100644 --- a/crates/perry-runtime/src/closure/registry.rs +++ b/crates/perry-runtime/src/closure/registry.rs @@ -71,6 +71,12 @@ crate::perry_thread_local! { RefCell>> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); + /// Exact arrows with a compiler-private callback body whose cold arms + /// poison the caller's versioned loop before observable fallback code. + static CLOSURE_VERSIONED_LOOP_REGISTRY: + RefCell> = + RefCell::new(crate::fast_hash::new_ptr_hash_map()); + /// Side-table marking closure body `func_ptr`s whose body is strict-mode /// code (file-level `"use strict"` or a body directive). Drives /// OrdinaryCallBindThis in `call`/`apply`/`bind`: a strict callee @@ -572,6 +578,38 @@ pub(crate) fn lookup_closure_trusted_direct(func_ptr: *const u8) -> Option Option { + CLOSURE_VERSIONED_LOOP_REGISTRY.with(|r| r.borrow().get(&(func_ptr as usize)).copied()) +} + /// Keepalive anchor for the auto-optimize whole-program build — registration /// is referenced only by generated module-init code. #[cfg(feature = "keepalive-anchors")] @@ -579,6 +617,15 @@ pub(crate) fn lookup_closure_trusted_direct(func_ptr: *const u8) -> Option PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_perry")) +} + +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .expect("canonicalize workspace root") +} + +fn runtime_dir() -> PathBuf { + static BUILD_RUNTIME: Once = Once::new(); + BUILD_RUNTIME.call_once(|| { + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + let mut command = Command::new(cargo); + command.current_dir(workspace_root()).arg("build"); + if !cfg!(debug_assertions) { + command.arg("--release"); + } + let build = command + .args(["-p", "perry-runtime-static"]) + .output() + .expect("build static runtime archive"); + assert!( + build.status.success(), + "static runtime build failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&build.stdout), + String::from_utf8_lossy(&build.stderr) + ); + }); + + perry_bin() + .parent() + .expect("Perry binary directory") + .to_path_buf() +} + +fn run_fixture(binary: &Path, force_evacuation: bool) -> Output { + let mut command = Command::new(binary); + if force_evacuation { + command + .env("PERRY_GC_FORCE_EVACUATE", "1") + .env("PERRY_GC_VERIFY_EVACUATION", "1"); + } else { + command + .env_remove("PERRY_GC_FORCE_EVACUATE") + .env_remove("PERRY_GC_VERIFY_EVACUATION"); + } + command.output().expect("run callback-deopt fixture") +} + +fn llvm_function_body(ir: &str, symbol: &str) -> String { + let start = ir + .lines() + .position(|line| line.starts_with("define") && line.contains(symbol)) + .unwrap_or_else(|| panic!("no LLVM definition containing {symbol:?}")); + ir.lines() + .skip(start) + .take_while(|line| *line != "}") + .collect::>() + .join("\n") +} + +#[test] +fn cold_callback_arms_resume_once_at_the_next_index() { + let dir = tempfile::tempdir().expect("tempdir"); + let entry = dir.path().join("main.ts"); + let binary = dir.path().join("main_bin"); + std::fs::write( + &entry, + r#" +class Reader { + entities: number[] = []; + + private checkedRead(column: any[], index: number, type: number): any { + if (column === undefined) throw new Error("missing column " + type); + const value = column[index]; + if (value === undefined) throw new Error("missing value " + type); + return value; + } + + iterate( + column: any[], + callback: (entity: number, value: any) => void, + entityFilter?: (entity: number) => boolean, + ): void { + const entities = this.entities; + const entityCount = entities.length; + const cb = callback; + for (let i = 0; i < entityCount; i++) { + const entity = entities[i]!; + if (entityFilter && !entityFilter(entity)) continue; + cb(entity, this.checkedRead(column, i, 1)); + } + } + + iterateCaught( + column: any[], + callback: (entity: number, value: any) => void, + entityFilter?: (entity: number) => boolean, + ): string { + const entities = this.entities; + const entityCount = entities.length; + const cb = callback; + try { + for (let i = 0; i < entityCount; i++) { + const entity = entities[i]!; + if (entityFilter && !entityFilter(entity)) continue; + cb(entity, this.checkedRead(column, i, 1)); + } + } catch (_error) { + return entities.length + ":" + column.length; + } + return "none"; + } +} + +function makeReader(count: number): Reader { + const reader = new Reader(); + for (let i = 0; i < count; i++) reader.entities.push(i); + return reader; +} + +const plainReader = makeReader(4); +let plainSum: any = 0; +plainReader.iterate( + [{ n: 1 }, { n: 2 }, { n: 3 }, { n: 4 }], + (_entity, value) => { plainSum += value.n; }, + undefined, +); + +const mutatingReader = makeReader(4); +const accessor: any = {}; +const mutatingColumn: any[] = [{ n: 10 }, accessor, { n: 30 }, { n: 40 }]; +let getterCalls = 0; +Object.defineProperty(accessor, "n", { + get() { + getterCalls++; + mutatingColumn.push({ n: 50 }); + (mutatingReader as any).checkedRead = ( + _column: any[], + index: number, + _type: number, + ) => ({ n: 100 + index }); + return 20; + }, +}); +let mutatingSum: any = 0; +mutatingReader.iterate( + mutatingColumn, + (_entity, value) => { mutatingSum += value.n; }, + undefined, +); + +const stringReader = makeReader(3); +let stringSum: any = 0; +stringReader.iterate( + [{ n: 1 }, { n: "x" }, { n: 3 }], + (_entity, value) => { stringSum += value.n; }, + undefined, +); + +const caughtReader = makeReader(2); +let caughtSum: any = 0; +const caughtCallback = (_entity: number, value: any) => { caughtSum += value.n; }; +caughtReader.iterate([{ n: 1 }, { n: 2 }], caughtCallback, undefined); +caughtSum = 0; +const throwingValue: any = {}; +Object.defineProperty(throwingValue, "n", { + get() { + const churn: any[] = []; + for (let i = 0; i < 2048; i++) churn.push({ i }); + throw new Error("cold getter"); + }, +}); +const caught = caughtReader.iterateCaught( + [{ n: 1 }, throwingValue], + caughtCallback, + undefined, +); + +console.log( + plainSum + ":" + mutatingSum + ":" + mutatingColumn.length + ":" + + getterCalls + ":" + stringSum + ":" + caught + ":" + caughtSum, +); +"#, + ) + .expect("write callback-deopt fixture"); + + let compile = Command::new(perry_bin()) + .current_dir(dir.path()) + .arg("compile") + .arg(&entry) + .arg("-o") + .arg(&binary) + .arg("--no-cache") + .arg("--no-auto-optimize") + .arg("--trace") + .arg("llvm") + .env("PERRY_RUNTIME_DIR", runtime_dir()) + .output() + .expect("compile callback-deopt fixture"); + assert!( + compile.status.success(), + "compile failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&compile.stdout), + String::from_utf8_lossy(&compile.stderr) + ); + + let ir = std::fs::read_to_string(dir.path().join(".perry-trace/llvm/main_ts.ll")) + .expect("read traced main module LLVM IR"); + let ordinary = llvm_function_body(&ir, "__Reader__iterate$undef2("); + let caught = llvm_function_body(&ir, "__Reader__iterateCaught$undef2("); + assert!( + ordinary.contains("versioned_index.loop.callback.preheader"), + "ordinary loop should select the exact callback version:\n{ordinary}" + ); + assert!( + !caught.contains("versioned_index.loop.callback.preheader"), + "an active local EH scope must keep the collecting callback clone out:\n{caught}" + ); + + for force_evacuation in [false, true] { + let run = run_fixture(&binary, force_evacuation); + assert!( + run.status.success(), + "fixture failed (force_evacuation={force_evacuation})\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&run.stdout), + String::from_utf8_lossy(&run.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&run.stdout), + "10:235:5:1:1x3:2:2:1\n" + ); + } +} From 2ef8f9a67ac34e8de00dd56b1b1fdb92ce8e610b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 24 Aug 2026 23:57:24 +0200 Subject: [PATCH 2/5] docs(changelog): note exact callback loop deopt --- changelog.d/8783-exact-callback-versioned-loops.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/8783-exact-callback-versioned-loops.md diff --git a/changelog.d/8783-exact-callback-versioned-loops.md b/changelog.d/8783-exact-callback-versioned-loops.md new file mode 100644 index 0000000000..9dcf62ab40 --- /dev/null +++ b/changelog.d/8783-exact-callback-versioned-loops.md @@ -0,0 +1 @@ +Versioned checked-reader loops now recognize a narrow family of exact additive arrow callbacks and keep admitted array handles in the loop preheader while the callback stays on its non-collecting path. Property-cache misses, dynamic addition, identity or capture-layout mismatches, and local exception scopes fail closed to an exact next-index resume through the ordinary guarded loop, eliminating steady-state callback guards without changing observable fallback behavior. From 8f149ae611c8b04c6e90a44fb362c628af020edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 25 Aug 2026 07:22:58 +0200 Subject: [PATCH 3/5] perf(codegen): specialize captured numeric updates --- .../8783-exact-callback-versioned-loops.md | 2 +- .../src/codegen/closure_collect.rs | 38 +++++---- .../src/codegen/trusted_box_callback_tests.rs | 78 ++++++++++++++++++- .../perry-codegen/src/expr/literals_vars.rs | 49 ++++++++++++ .../versioned_indexed_loop_callback_deopt.rs | 76 ++++++++++++++++-- 5 files changed, 221 insertions(+), 22 deletions(-) diff --git a/changelog.d/8783-exact-callback-versioned-loops.md b/changelog.d/8783-exact-callback-versioned-loops.md index 9dcf62ab40..6a2e39ca83 100644 --- a/changelog.d/8783-exact-callback-versioned-loops.md +++ b/changelog.d/8783-exact-callback-versioned-loops.md @@ -1 +1 @@ -Versioned checked-reader loops now recognize a narrow family of exact additive arrow callbacks and keep admitted array handles in the loop preheader while the callback stays on its non-collecting path. Property-cache misses, dynamic addition, identity or capture-layout mismatches, and local exception scopes fail closed to an exact next-index resume through the ordinary guarded loop, eliminating steady-state callback guards without changing observable fallback behavior. +Versioned checked-reader loops now recognize a narrow family of exact additive arrow callbacks, including unused-result `++`/`--` updates to compiler-proven captured boxes, and keep admitted array handles in the loop preheader while the callback stays on its non-collecting path. Property-cache misses, dynamic addition, ToNumeric coercion, TDZ reads, identity or capture-layout mismatches, and local exception scopes fail closed to an exact next-index resume through the ordinary guarded loop, eliminating steady-state callback guards without changing observable fallback behavior. diff --git a/crates/perry-codegen/src/codegen/closure_collect.rs b/crates/perry-codegen/src/codegen/closure_collect.rs index dc7645b0da..4a68d8bf9f 100644 --- a/crates/perry-codegen/src/codegen/closure_collect.rs +++ b/crates/perry-codegen/src/codegen/closure_collect.rs @@ -77,11 +77,12 @@ fn versioned_loop_expr_uses_local(expr: &perry_hir::Expr, local_id: u32) -> bool } /// Select exact arrow callbacks whose only source-level effect is replacing a -/// captured box with an additive expression over callback parameters. The -/// private clone forces parameter property reads through the descriptor-aware -/// generic PIC and poisons its caller's fast loop before any PIC or dynamic-+ -/// fallback can run user code. Everything outside this deliberately small -/// grammar keeps the ordinary guarded loop. +/// captured box with an additive expression over callback parameters, or an +/// unused-result `++`/`--` on that box. The private clone forces parameter +/// property reads through the descriptor-aware generic PIC and poisons its +/// caller's fast loop before any PIC, dynamic-+, or ToNumeric fallback can run +/// user code. Everything outside this deliberately small grammar keeps the +/// ordinary guarded loop. pub(crate) fn select_versioned_loop_callbacks( closures: &[(perry_hir::types::FuncId, perry_hir::Expr)], trusted_box_closures: &std::collections::HashMap, @@ -108,19 +109,29 @@ pub(crate) fn select_versioned_loop_callbacks( else { return None; }; - let [perry_hir::Stmt::Expr(perry_hir::Expr::LocalSet(target, value))] = body.as_slice() - else { - return None; + let (target, value) = match body.as_slice() { + [perry_hir::Stmt::Expr(perry_hir::Expr::LocalSet(target, value))] => { + (*target, Some(value.as_ref())) + } + // The result of a prefix/postfix update is unobservable when + // the update is the callback's complete expression statement. + // A private clone can therefore guard the captured value as a + // Number, perform the step inline, and deopt before ToNumeric + // for strings, objects, BigInts, TDZ, or any other cold case. + [perry_hir::Stmt::Expr(perry_hir::Expr::Update { id, .. })] => (*id, None), + _ => return None, }; - if !module_boxed_vars.contains(target) { + if !module_boxed_vars.contains(&target) { return None; } // The private body carries the caller's stack context in the first // otherwise-unused callback argument. Reusing the public ABI keeps // the context out of an extra live argument on every iteration. let scratch_param = params.first()?; - if versioned_loop_expr_uses_local(value, scratch_param.id) { - return None; + if let Some(value) = value { + if versioned_loop_expr_uses_local(value, scratch_param.id) { + return None; + } } let param_ids: std::collections::HashSet = params.iter().map(|param| param.id).collect(); @@ -133,8 +144,9 @@ pub(crate) fn select_versioned_loop_callbacks( ) .into_iter() .collect(); - (capture_ids.contains(target) - && versioned_loop_value_expr(value, ¶m_ids, &capture_ids)) + (capture_ids.contains(&target) + && value + .is_none_or(|value| versioned_loop_value_expr(value, ¶m_ids, &capture_ids))) .then_some(*func_id) }) .collect() diff --git a/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs b/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs index 7a1e3fb74c..06700bf2e6 100644 --- a/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs +++ b/crates/perry-codegen/src/codegen/trusted_box_callback_tests.rs @@ -157,6 +157,22 @@ fn versioned_callback(func_id: u32) -> Expr { ) } +fn versioned_update_callback(func_id: u32, op: UpdateOp, prefix: bool) -> Expr { + callback_with( + func_id, + vec![ + param(29, "entity", Type::Any), + param(30, "pos", Type::Any), + param(31, "vel", Type::Any), + ], + vec![Stmt::Expr(Expr::Update { + id: COUNT, + op, + prefix, + })], + ) +} + fn select(closures: Vec<(u32, Expr)>, direct: impl IntoIterator) -> HashSet { select_trusted_box_closures( &closures, @@ -189,14 +205,13 @@ fn emit(direct_literal: bool) -> String { .expect("LLVM IR is UTF-8") } -fn emit_versioned_callback() -> String { - const VERSIONED_FUNC: u32 = 100; +fn emit_versioned_callback_with(callback: Expr) -> String { let mut outer = outer_function(true); let call = outer.body.last_mut().expect("outer call exists"); let Stmt::Expr(Expr::Call { args, .. }) = call else { panic!("outer tail is a call"); }; - args[0] = versioned_callback(VERSIONED_FUNC); + args[0] = callback; let mut module = Module::new("versioned_loop_callback.ts"); module.init_kind = ModuleInitKind::Eager; @@ -216,6 +231,10 @@ fn emit_versioned_callback() -> String { .expect("LLVM IR is UTF-8") } +fn emit_versioned_callback() -> String { + emit_versioned_callback_with(versioned_callback(100)) +} + fn function_body(ir: &str, symbol: &str) -> String { let start = ir .lines() @@ -228,6 +247,19 @@ fn function_body(ir: &str, symbol: &str) -> String { .join("\n") } +fn named_block_body<'a>(function: &'a str, prefix: &str) -> String { + let start = function + .lines() + .position(|line| line.starts_with(prefix) && line.ends_with(':')) + .unwrap_or_else(|| panic!("missing block {prefix}:\n{function}")); + function + .lines() + .skip(start + 1) + .take_while(|line| !line.ends_with(':')) + .collect::>() + .join("\n") +} + #[test] fn direct_arrow_gets_a_private_body_but_keeps_the_public_validation_path() { let ir = emit(true); @@ -299,6 +331,29 @@ fn additive_property_callback_gets_a_cold_deopting_private_body() { )); } +#[test] +fn captured_update_callback_guards_number_and_deopts_before_tonumeric() { + let ir = + emit_versioned_callback_with(versioned_update_callback(100, UpdateOp::Increment, false)); + let special = function_body( + &ir, + "perry_closure_versioned_loop_callback_ts__100$trusted_boxes$versioned_loop", + ); + assert!(special.contains("versioned_update.number"), "{special}"); + assert!(special.contains("versioned_update.tonumeric"), "{special}"); + assert!( + special.contains("versioned_callback.deopt.mark") + && special.contains("@js_to_numeric(") + && special.contains("@js_numeric_step("), + "the cold arm must poison the loop before preserving full update semantics:\n{special}" + ); + let fast = named_block_body(&special, "versioned_update.number"); + assert!(fast.contains("fadd double"), "{fast}"); + assert!(!fast.contains("@js_to_numeric("), "{fast}"); + assert!(!fast.contains("@js_numeric_step("), "{fast}"); + assert!(!fast.contains("@js_write_barrier("), "{fast}"); +} + #[test] fn versioned_callback_selector_rejects_calls_and_heap_writes() { const VERSIONED_FUNC: u32 = 100; @@ -314,6 +369,23 @@ fn versioned_callback_selector_rejects_calls_and_heap_writes() { .contains(&VERSIONED_FUNC) ); + for (op, prefix) in [ + (UpdateOp::Increment, false), + (UpdateOp::Increment, true), + (UpdateOp::Decrement, false), + (UpdateOp::Decrement, true), + ] { + let update = versioned_update_callback(VERSIONED_FUNC, op, prefix); + let closures = vec![(VERSIONED_FUNC, update)]; + let trusted = + select_trusted_box_closures(&closures, &direct, &boxed, &globals, &HashSet::new()); + assert!( + select_versioned_loop_callbacks(&closures, &trusted, &boxed, &globals) + .contains(&VERSIONED_FUNC), + "unused-result {op:?} prefix={prefix} must be eligible" + ); + } + for rejected_body in [ vec![Stmt::Expr(Expr::Call { callee: Box::new(Expr::LocalGet(30)), diff --git a/crates/perry-codegen/src/expr/literals_vars.rs b/crates/perry-codegen/src/expr/literals_vars.rs index 3e5071094d..023b526505 100644 --- a/crates/perry-codegen/src/expr/literals_vars.rs +++ b/crates/perry-codegen/src/expr/literals_vars.rs @@ -42,6 +42,11 @@ fn load_trusted_box_capture_bits(ctx: &mut FnCtx<'_>, capture: &TrustedBoxCaptur ctx.block().cond_br(&is_tdz, &slow_label, &merge_label); ctx.current_block = slow_idx; + // The trusted accessor throws for a real TDZ read (and can allocate while + // constructing the error). A versioned-loop clone must poison its caller + // before entering that observable cold arm, just like a PIC miss or + // dynamic `+` fallback. + crate::expr::emit_versioned_loop_callback_deopt(ctx); let slow_bits = ctx .block() .call(I64, "js_box_get_bits_trusted", &[(I64, &capture.bits)]); @@ -904,6 +909,50 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { if let Some(capture) = ctx.trusted_box_capture_ptrs.get(id).cloned() { let old_bits = load_trusted_box_capture_bits(ctx, &capture); let old = ctx.block().bitcast_i64_to_double(&old_bits); + if needs_numeric_coerce && ctx.versioned_loop_deopt_context.is_some() { + let is_number = crate::stmt::emit_js_value_is_number(ctx, &old); + let fast_idx = ctx.new_block("versioned_update.number"); + let slow_idx = ctx.new_block("versioned_update.tonumeric"); + let merge_idx = ctx.new_block("versioned_update.merge"); + let fast_label = ctx.block_label(fast_idx); + let slow_label = ctx.block_label(slow_idx); + let merge_label = ctx.block_label(merge_idx); + ctx.block().cond_br(&is_number, &fast_label, &slow_label); + + ctx.current_block = fast_idx; + let fast_new = match op { + UpdateOp::Increment => ctx.block().fadd(&old, "1.0"), + UpdateOp::Decrement => ctx.block().fsub(&old, "1.0"), + }; + let fast_new_bits = ctx.block().bitcast_double_to_i64(&fast_new); + ctx.block().store(I64, &fast_new_bits, &capture.ptr); + let fast_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = slow_idx; + // ToNumeric can invoke user code and collect. Mark + // the exact resume index before it becomes + // observable; the caller exits after this update + // and resumes the guarded loop at the next entity. + crate::expr::emit_versioned_loop_callback_deopt(ctx); + let slow_old = coerce_old(ctx.block(), &old); + let slow_new = step_new(ctx.block(), &slow_old); + let slow_new_bits = ctx.block().bitcast_double_to_i64(&slow_new); + ctx.block().store(I64, &slow_new_bits, &capture.ptr); + // Only the cold arm can produce a BigInt pointer. + emit_write_barrier(ctx, &capture.bits, &slow_new_bits); + let slow_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + return Ok(ctx.block().phi( + DOUBLE, + &[ + (if *prefix { &fast_new } else { &old }, &fast_end), + (if *prefix { &slow_new } else { &slow_old }, &slow_end), + ], + )); + } let old = coerce_old(ctx.block(), &old); let new = step_new(ctx.block(), &old); let new_bits = ctx.block().bitcast_double_to_i64(&new); diff --git a/crates/perry/tests/versioned_indexed_loop_callback_deopt.rs b/crates/perry/tests/versioned_indexed_loop_callback_deopt.rs index ed031c2e4e..5f2e5df469 100644 --- a/crates/perry/tests/versioned_indexed_loop_callback_deopt.rs +++ b/crates/perry/tests/versioned_indexed_loop_callback_deopt.rs @@ -1,6 +1,7 @@ //! Runtime regression for the zero-steady-state-check callback specialization -//! in versioned checked-reader loops. Cold property/addition arms must mark the -//! current loop for an exact once-only resume before any observable fallback. +//! in versioned checked-reader loops. Cold property/addition/ToNumeric arms +//! must mark the current loop for an exact once-only resume before any +//! observable fallback. use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -130,6 +131,7 @@ function makeReader(count: number): Reader { return reader; } +function runFixture(): void { const plainReader = makeReader(4); let plainSum: any = 0; plainReader.iterate( @@ -169,6 +171,47 @@ stringReader.iterate( undefined, ); +const updateReader = makeReader(4); +let incrementCount: any = 0; +updateReader.iterate([0, 0, 0, 0], (_entity, _value) => { incrementCount++; }, undefined); +let decrementCount: any = 4; +updateReader.iterate([0, 0, 0, 0], (_entity, _value) => { --decrementCount; }, undefined); + +const coercionReader = makeReader(3); +let stringCount: any = "1"; +coercionReader.iterate([0, 0, 0], (_entity, _value) => { stringCount++; }, undefined); +let valueOfCalls = 0; +let objectCount: any = { + valueOf() { + valueOfCalls++; + const churn: any[] = []; + for (let i = 0; i < 2048; i++) churn.push({ i }); + return 5; + }, +}; +coercionReader.iterate([0, 0, 0], (_entity, _value) => { ++objectCount; }, undefined); +let bigintCount: any = 10n; +coercionReader.iterate([0, 0, 0], (_entity, _value) => { bigintCount++; }, undefined); + +const throwingUpdateReader = makeReader(2); +let throwingCount: any = { + valueOf() { + const churn: any[] = []; + for (let i = 0; i < 2048; i++) churn.push({ i }); + throw new Error("cold update"); + }, +}; +let updateError = "none"; +try { + throwingUpdateReader.iterate( + [0, 0], + (_entity, _value) => { throwingCount++; }, + undefined, + ); +} catch (error: any) { + updateError = error.message; +} + const caughtReader = makeReader(2); let caughtSum: any = 0; const caughtCallback = (_entity: number, value: any) => { caughtSum += value.n; }; @@ -190,8 +233,13 @@ const caught = caughtReader.iterateCaught( console.log( plainSum + ":" + mutatingSum + ":" + mutatingColumn.length + ":" + - getterCalls + ":" + stringSum + ":" + caught + ":" + caughtSum, + getterCalls + ":" + stringSum + ":" + incrementCount + ":" + decrementCount + ":" + + stringCount + ":" + objectCount + ":" + valueOfCalls + ":" + String(bigintCount) + ":" + + updateError + ":" + caught + ":" + caughtSum, ); +} + +runFixture(); "#, ) .expect("write callback-deopt fixture"); @@ -216,7 +264,8 @@ console.log( String::from_utf8_lossy(&compile.stderr) ); - let ir = std::fs::read_to_string(dir.path().join(".perry-trace/llvm/main_ts.ll")) + let trace_dir = dir.path().join(".perry-trace/llvm"); + let ir = std::fs::read_to_string(trace_dir.join("main_ts.ll")) .expect("read traced main module LLVM IR"); let ordinary = llvm_function_body(&ir, "__Reader__iterate$undef2("); let caught = llvm_function_body(&ir, "__Reader__iterateCaught$undef2("); @@ -228,6 +277,23 @@ console.log( !caught.contains("versioned_index.loop.callback.preheader"), "an active local EH scope must keep the collecting callback clone out:\n{caught}" ); + let callback_ir = std::fs::read_dir(&trace_dir) + .expect("read LLVM trace directory") + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "ll")) + .map(|path| std::fs::read_to_string(path).expect("read traced LLVM module")) + .collect::>() + .join("\n"); + let update_number_blocks = callback_ir.matches("versioned_update.number").count(); + let update_tonumeric_blocks = callback_ir.matches("versioned_update.tonumeric").count(); + let callback_deopt_blocks = callback_ir.matches("versioned_callback.deopt.mark").count(); + assert!( + update_number_blocks != 0 && update_tonumeric_blocks != 0 && callback_deopt_blocks != 0, + "captured updates must keep numeric stepping hot and ToNumeric behind exact deopt \ + (number={update_number_blocks}, tonumeric={update_tonumeric_blocks}, \ + deopt={callback_deopt_blocks})" + ); for force_evacuation in [false, true] { let run = run_fixture(&binary, force_evacuation); @@ -239,7 +305,7 @@ console.log( ); assert_eq!( String::from_utf8_lossy(&run.stdout), - "10:235:5:1:1x3:2:2:1\n" + "10:235:5:1:1x3:4:0:4:8:1:13:cold update:2:2:1\n" ); } } From a6c610f57ae604c328573243c9000a7925ff40d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 25 Aug 2026 07:39:19 +0200 Subject: [PATCH 4/5] test: normalize callback deopt GC arms --- .../versioned_indexed_loop_callback_deopt.rs | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/crates/perry/tests/versioned_indexed_loop_callback_deopt.rs b/crates/perry/tests/versioned_indexed_loop_callback_deopt.rs index 5f2e5df469..22a25902d7 100644 --- a/crates/perry/tests/versioned_indexed_loop_callback_deopt.rs +++ b/crates/perry/tests/versioned_indexed_loop_callback_deopt.rs @@ -7,6 +7,29 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use std::sync::Once; +/// Normalize every collector input that can make a nominal evacuation arm +/// non-moving. Several of these are compile-time Perry inputs, so apply the +/// same baseline to the runtime build, fixture compile, and child process. +const GC_ENV_OVERRIDES: &[&str] = &[ + "PERRY_GEN_GC", + "PERRY_GC_SCAVENGE", + "PERRY_GC_SCAVENGE_NURSERY_MB", + "PERRY_GC_MOVING_SAFEPOINT", + "PERRY_GC_MOVING_LOOP_POLLS", + "PERRY_GC_FORCE_EVACUATE", + "PERRY_GC_VERIFY_EVACUATION", + "PERRY_CONSERVATIVE_STACK_SCAN", + "PERRY_WRITE_BARRIERS", + "PERRY_GC_INCREMENTAL", + "PERRY_GC_HEAP_LIMIT", +]; + +fn remove_gc_env_overrides(command: &mut Command) { + for key in GC_ENV_OVERRIDES { + command.env_remove(key); + } +} + fn perry_bin() -> PathBuf { PathBuf::from(env!("CARGO_BIN_EXE_perry")) } @@ -24,6 +47,7 @@ fn runtime_dir() -> PathBuf { let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); let mut command = Command::new(cargo); command.current_dir(workspace_root()).arg("build"); + remove_gc_env_overrides(&mut command); if !cfg!(debug_assertions) { command.arg("--release"); } @@ -47,14 +71,11 @@ fn runtime_dir() -> PathBuf { fn run_fixture(binary: &Path, force_evacuation: bool) -> Output { let mut command = Command::new(binary); + remove_gc_env_overrides(&mut command); if force_evacuation { command .env("PERRY_GC_FORCE_EVACUATE", "1") .env("PERRY_GC_VERIFY_EVACUATION", "1"); - } else { - command - .env_remove("PERRY_GC_FORCE_EVACUATE") - .env_remove("PERRY_GC_VERIFY_EVACUATION"); } command.output().expect("run callback-deopt fixture") } @@ -244,7 +265,8 @@ runFixture(); ) .expect("write callback-deopt fixture"); - let compile = Command::new(perry_bin()) + let mut compile_command = Command::new(perry_bin()); + compile_command .current_dir(dir.path()) .arg("compile") .arg(&entry) @@ -254,7 +276,9 @@ runFixture(); .arg("--no-auto-optimize") .arg("--trace") .arg("llvm") - .env("PERRY_RUNTIME_DIR", runtime_dir()) + .env("PERRY_RUNTIME_DIR", runtime_dir()); + remove_gc_env_overrides(&mut compile_command); + let compile = compile_command .output() .expect("compile callback-deopt fixture"); assert!( From f5e1698310a0f8fb6c56b41e359e55345d9c234d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 25 Aug 2026 10:51:27 +0200 Subject: [PATCH 5/5] chore: root-holder verdict for CLOSURE_VERSIONED_LOOP_REGISTRY (#8783) --- scripts/gc_runtime_root_holders.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index e39071bd95..fb94ea3754 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -155,6 +155,12 @@ "verdict": "not_a_gc_pointer", "why": "Cache miss counter (telemetry). Holds no address." }, + { + "file": "crates/perry-runtime/src/closure/registry.rs", + "name": "CLOSURE_VERSIONED_LOOP_REGISTRY", + "verdict": "not_a_gc_pointer", + "why": "Maps a closure body `func_ptr` (a code pointer, never moved) to TrustedDirectTarget{func_ptr:*const u8, capture_count:u32, boxed_capture_mask:u64}. Both key and value are code pointers and plain integers; no field can hold a heap pointer, so the table cannot hold a GC root. Same shape as its sibling CLOSURE_ARROW_FUNCTION_REGISTRY in the same thread_local block." + }, { "file": "crates/perry-runtime/src/fs/filehandle.rs", "name": "NEXT_READ_LINES_ID",