From e969427bb52e5f999710235f3e9790986422f117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 26 Aug 2026 06:51:12 +0200 Subject: [PATCH 01/15] perf: cache owning Uint32Array admissions --- crates/perry-runtime/src/typedarray/mod.rs | 71 +++++++++++++++++++++- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/crates/perry-runtime/src/typedarray/mod.rs b/crates/perry-runtime/src/typedarray/mod.rs index d2af56b85a..eeadc0a0f4 100644 --- a/crates/perry-runtime/src/typedarray/mod.rs +++ b/crates/perry-runtime/src/typedarray/mod.rs @@ -233,6 +233,18 @@ pub const TA_CACHE_NEGATIVE: u64 = 0xFF; pub static PERRY_TA_KIND_CACHE: [AtomicU64; TA_KIND_CACHE_SLOTS] = [const { AtomicU64::new(0) }; TA_KIND_CACHE_SLOTS]; +// The generic kind cache deliberately uses the exact slot formula duplicated +// by codegen. Large, equal-sized ECS columns can therefore share the same low +// address bits and continually evict one another. Whole-loop admission needs +// the stronger, persistent fact "this exact address is an owning Uint32Array", +// so keep a separate direct cache whose index folds higher address bits too. +// A hit is safe until unregister: a TypedArray header's kind and owning/view +// storage class never change during its lifetime, and unregister clears both +// caches before an address can be reused. +const INLINE_OWNING_U32_CACHE_SLOTS: usize = 64; +static INLINE_OWNING_U32_CACHE: [AtomicU64; INLINE_OWNING_U32_CACHE_SLOTS] = + [const { AtomicU64::new(0) }; INLINE_OWNING_U32_CACHE_SLOTS]; + /// #5525 follow-up: process-global "any exotic typed-array views exist" guard, /// exported under a stable link name for the codegen inline element path. A /// non-owning typed array (an `ArrayBuffer`-aliasing view, or a native-arena @@ -287,6 +299,33 @@ fn ta_kind_cache_invalidate(addr: usize) { } } +#[inline] +fn inline_owning_u32_cache_slot(addr: usize) -> usize { + let word = addr >> 3; + let mixed = word ^ (word >> 6) ^ (word >> 12); + mixed & (INLINE_OWNING_U32_CACHE_SLOTS - 1) +} + +#[inline] +fn inline_owning_u32_cache_get(addr: usize) -> bool { + INLINE_OWNING_U32_CACHE[inline_owning_u32_cache_slot(addr)].load(Ordering::Relaxed) + == addr as u64 +} + +#[inline] +fn inline_owning_u32_cache_store(addr: usize) { + INLINE_OWNING_U32_CACHE[inline_owning_u32_cache_slot(addr)] + .store(addr as u64, Ordering::Relaxed); +} + +#[inline] +fn inline_owning_u32_cache_invalidate(addr: usize) { + let slot = inline_owning_u32_cache_slot(addr); + if INLINE_OWNING_U32_CACHE[slot].load(Ordering::Relaxed) == addr as u64 { + INLINE_OWNING_U32_CACHE[slot].store(0, Ordering::Relaxed); + } +} + /// Cache probe: `None` = miss (consult the registry), `Some(None)` = cached /// negative ("not a typed array"), `Some(Some(kind))` = cached typed array. #[inline] @@ -339,6 +378,7 @@ pub(crate) fn typed_array_registry_ever_used() -> bool { pub fn unregister_typed_array(ptr: *const TypedArrayHeader) { let owner = ptr as usize; ta_kind_cache_invalidate(owner); + inline_owning_u32_cache_invalidate(owner); TYPED_ARRAY_REGISTRY.with(|r| { r.borrow_mut().remove(&owner); }); @@ -587,9 +627,11 @@ pub extern "C" fn js_typed_array_masked_window_data_ptr(receiver: f64) -> i64 { /// One-time loop admission primitive for erased ECS component columns. Return /// the stable owning-header address only for an exact inline `Uint32Array`. -/// Consult the authoritative registry instead of the tiny direct-mapped kind -/// cache: sibling columns can collide there, which is harmless for individual -/// accesses but must not make a whole-loop proof spuriously fail forever. +/// Use the admission-specific address cache first, then consult the +/// authoritative registry on a miss. The generic direct-mapped kind cache is +/// intentionally not authority here: sibling columns can collide there, +/// which is harmless for individual accesses but must not make a whole-loop +/// proof spuriously fail forever. #[inline] pub(crate) fn inline_u32_addr(receiver: f64) -> usize { let value = crate::value::JSValue::from_bits(receiver.to_bits()); @@ -597,12 +639,16 @@ pub(crate) fn inline_u32_addr(receiver: f64) -> usize { return 0; } let addr = value.as_pointer::() as usize; + if inline_owning_u32_cache_get(addr) { + return addr; + } if lookup_typed_array_kind(addr) != Some(KIND_UINT32) || crate::native_arena::is_native_typed_view(addr as *const TypedArrayHeader) || crate::typedarray_view::view_meta_of(addr).is_some() { return 0; } + inline_owning_u32_cache_store(addr); addr } @@ -1253,6 +1299,25 @@ pub extern "C" fn js_native_memory_copy(dst_raw: u64, src_raw: u64) { mod tests { use super::*; + #[test] + fn owning_u32_admission_cache_skips_registry_and_invalidates() { + let ta = typed_array_alloc(KIND_UINT32, 16); + let boxed = crate::value::js_nanbox_pointer(ta as i64); + + let before = test_typed_array_registry_probe_count(); + assert_eq!(inline_u32_addr(boxed), ta as usize); + let primed = test_typed_array_registry_probe_count(); + assert_eq!(primed, before + 1); + assert_eq!(inline_u32_addr(boxed), ta as usize); + assert_eq!(test_typed_array_registry_probe_count(), primed); + + unregister_typed_array(ta); + assert_eq!(inline_u32_addr(boxed), 0); + assert_eq!(test_typed_array_registry_probe_count(), primed + 1); + // Leave the live allocation registered for its eventual finalizer. + register_typed_array(ta, KIND_UINT32); + } + #[test] fn large_object_typed_array_alloc_uses_old_gc_header_and_stays_usable() { let ta = typed_array_alloc(KIND_UINT8, crate::gc::LARGE_OBJECT_THRESHOLD_BYTES as u32); From 15d7673b294d03a53d8227c8935e3b561f8cad05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 26 Aug 2026 06:51:12 +0200 Subject: [PATCH 02/15] perf: fast-path Array subclass length misses --- .../src/object/field_get_set/ic_miss.rs | 78 +++++++++++++++---- 1 file changed, 65 insertions(+), 13 deletions(-) diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index f6f1d9949d..a0aa5f574f 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -508,19 +508,35 @@ pub extern "C" fn js_object_get_field_ic_miss( // run time — more than the entire polymorphic-dispatch fix above saved. // // `GC_TYPE_ARRAY` is a genuine dense array: buffers, typed arrays, lazy - // arrays, Sets and Maps all carry their own distinct `obj_type`, and an - // `class X extends Array` instance is an `ObjectHeader` - // (`GC_TYPE_OBJECT`). `js_array_length` still resolves growth-forwarding - // stubs, proxies and subclass receivers, so this only skips probes that - // cannot match — the expression returned is exactly the one - // `get_field_by_name_object_tail`'s array arm computes for this key, - // which is what makes it a pure short-circuit rather than a second - // implementation. - if unsafe { gc_type_of(obj) } == Some(crate::gc::GC_TYPE_ARRAY) - && unsafe { key_bytes_are(key, b"length") } - { - let arr = obj as *const crate::array::ArrayHeader; - return crate::array::js_array_length(arr) as f64; + // arrays, Sets and Maps all carry their own distinct `obj_type`. A + // `class X extends Array` instance instead uses `GC_TYPE_OBJECT`, but + // the exact-ShapeId dense-layout proof can read its live own `length` + // slot without repeating generic object dispatch. Both arms retain + // their established helpers, making this a dispatch short-circuit + // rather than a second implementation of either representation. + if unsafe { key_bytes_are(key, b"length") } { + match unsafe { gc_type_of(obj) } { + Some(crate::gc::GC_TYPE_ARRAY) => { + let arr = obj as *const crate::array::ArrayHeader; + return crate::array::js_array_length(arr) as f64; + } + Some(crate::gc::GC_TYPE_OBJECT) => { + // Wolf ECS's Query and Archetype are `class ... extends + // Array` instances. They use ObjectHeader storage, so the + // Array arm above cannot recognize them and a megamorphic + // `.length` site otherwise repeats the full object lookup + // on every loop entry. Reuse the exact ShapeId-backed + // subclass layout proof already used by packed numeric + // reads. It declines accessor, prototype-override, sparse, + // and non-Array-subclass receivers, preserving the generic + // lookup below for every case it cannot prove. + let receiver = crate::value::js_nanbox_pointer(obj as i64); + if let Some(length) = crate::array::array_subclass_fast_length(receiver) { + return length; + } + } + _ => {} + } } unsafe { if let Some(val) = closure_dynamic_prop_by_key(obj as usize, key) { @@ -1955,4 +1971,40 @@ mod array_length_fast_path_tests { ); } } + + /// Array subclasses are ObjectHeader-backed, so a polymorphic loop over + /// differently shaped instances cannot use the real-Array short circuit + /// above or reliably stay in one property PIC. The dense subclass proof + /// must return the live own `length`, while unrelated object-backed values + /// continue through ordinary property lookup. + #[test] + fn array_subclass_length_short_circuit_preserves_object_semantics() { + const CLASS_ID_ARRAY: u32 = 0xFFFF_0024; + const SUBCLASS_ID: u32 = 0x0077_8655; + let _lock = crate::gc::global_side_table_test_lock(); + crate::object::js_register_class_parent(SUBCLASS_ID, CLASS_ID_ARRAY); + + let obj = crate::object::js_object_alloc(SUBCLASS_ID, 2); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + for (index, value) in [11.0, 22.0, 33.0].into_iter().enumerate() { + crate::object::js_object_set_index_polymorphic(obj as i64, index as f64, value); + } + + let len_key = crate::string::js_string_from_bytes(b"length".as_ptr(), 6); + let mut cache = [0i64; super::PIC_CACHE_WORDS]; + let via_ic = super::js_object_get_field_ic_miss(obj, len_key, &mut cache); + let via_ladder = super::js_object_get_field_by_name_f64(obj, len_key); + assert_eq!(via_ic.to_bits(), via_ladder.to_bits()); + assert_eq!(via_ic, 3.0, "the fast path must observe the live length"); + + let plain = crate::object::js_object_alloc(0, 1); + crate::object::js_object_set_field_by_name(plain, len_key, 123.0); + let mut plain_cache = [0i64; super::PIC_CACHE_WORDS]; + assert_eq!( + super::js_object_get_field_ic_miss(plain, len_key, &mut plain_cache), + 123.0, + "ordinary objects must retain their own `length` property semantics" + ); + } } From bbe2b4c25e5b5b7f8f6a0b7525ead547b46bce7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 10:51:52 +0200 Subject: [PATCH 03/15] perf(array): accumulated ECS optimization work through v74 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accumulated codex ECS campaign work (v40–v74) on top of the two prior commits on this branch: Array-subclass dense-tail fast paths and validated-object prototype-override reads (v72), pre-statepoint inlining of compact exact-receiver ($pshape) guarded specializations using the lowered LLVM IR size (v74), plus the supporting collectors/tests. Details, rejected experiments and measurements are in secret-tests/ECS_PERFORMANCE_HANDOFF_2026-08-27.md. Mac mini (taskpolicy -t 0 -l 0, 11 alternating pairs) at v74: wolf-ecs add/remove 0.5562 ms/op, entity-cycle 0.4988 ms/op (Node 26.5.1: 0.1337 / 0.1492). Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- crates/perry-codegen/src/codegen/closure.rs | 3 + crates/perry-codegen/src/codegen/entry.rs | 6 + crates/perry-codegen/src/codegen/function.rs | 3 + .../guarded_falsy_default_method_tests.rs | 176 ++++ crates/perry-codegen/src/codegen/helpers.rs | 31 + .../src/codegen/index_method_clone_tests.rs | 532 +++++++++- .../src/codegen/indexed_method_artifacts.rs | 84 ++ crates/perry-codegen/src/codegen/method.rs | 160 ++- .../src/codegen/method_trampolines.rs | 207 ++++ crates/perry-codegen/src/codegen/mod.rs | 36 +- crates/perry-codegen/src/codegen/opts.rs | 8 + .../src/codegen/ordinary_method_artifacts.rs | 13 +- .../perry-codegen/src/codegen/param_guard.rs | 208 ++++ crates/perry-codegen/src/codegen/typed_abi.rs | 101 +- .../src/collectors/index_uses.rs | 17 +- crates/perry-codegen/src/collectors/mod.rs | 2 +- .../src/collectors/proven_this.rs | 25 +- .../collectors/proven_this_routing_tests.rs | 361 +++++++ .../perry-codegen/src/collectors/ptr_shape.rs | 29 +- .../src/collectors/scalar_method_dispatch.rs | 277 ++++- crates/perry-codegen/src/expr/arrays_finds.rs | 6 +- crates/perry-codegen/src/expr/binary.rs | 8 + crates/perry-codegen/src/expr/bitset_test.rs | 235 +++++ .../src/expr/call_return_array_index_tests.rs | 19 +- crates/perry-codegen/src/expr/compare.rs | 237 ++++- .../perry-codegen/src/expr/compare_tests.rs | 122 +++ crates/perry-codegen/src/expr/index_get.rs | 97 +- .../src/expr/index_get/guarded_array.rs | 98 +- .../expr/index_get/inline_dyn_typed_array.rs | 271 ++++- .../src/expr/index_get_claim_tests.rs | 207 +++- crates/perry-codegen/src/expr/index_set.rs | 28 +- .../src/expr/index_set_barrier_tests.rs | 33 + .../src/expr/logical_collections.rs | 4 +- crates/perry-codegen/src/expr/mod.rs | 48 +- crates/perry-codegen/src/expr/property_get.rs | 395 ++++++- .../src/expr/property_get/generic_dispatch.rs | 224 +++- .../src/expr/property_get/tests.rs | 102 +- crates/perry-codegen/src/expr/unary.rs | 41 +- .../src/expr/unary_bitnot_tests.rs | 98 ++ .../perry-codegen/src/expr/write_barrier.rs | 30 +- crates/perry-codegen/src/gc_call_effects.rs | 3 + .../src/lower_call/method_override.rs | 221 +++- .../src/lower_call/native/mod.rs | 3 +- .../native/native_instance_branch.rs | 61 +- .../property_get/dynamic_dispatch.rs | 71 +- crates/perry-codegen/src/lower_conditional.rs | 81 +- crates/perry-codegen/src/module.rs | 13 + .../perry-codegen/src/runtime_decls/arrays.rs | 2 + .../src/runtime_decls/strings.rs | 3 + .../src/runtime_decls/strings_part2.rs | 11 + .../src/stmt/cached_field_index_return.rs | 321 ++++++ crates/perry-codegen/src/stmt/if_stmt.rs | 15 +- crates/perry-codegen/src/stmt/mod.rs | 9 + .../perry-runtime/src/array/element_shape.rs | 220 +++- .../src/array/element_shape_tests.rs | 76 ++ crates/perry-runtime/src/array/header.rs | 29 + crates/perry-runtime/src/array/indexing.rs | 185 ++++ crates/perry-runtime/src/array/mod.rs | 17 +- crates/perry-runtime/src/array/push_pop.rs | 162 ++- crates/perry-runtime/src/array/subclass.rs | 987 +++++++++++++++++- .../perry-runtime/src/array/subclass_tests.rs | 705 ++++++++++++- crates/perry-runtime/src/array/tests.rs | 62 ++ .../perry-runtime/src/builtins/arithmetic.rs | 183 +++- crates/perry-runtime/src/gc/layout.rs | 79 +- crates/perry-runtime/src/gc/telemetry.rs | 3 + .../src/object/array_tail_transition.rs | 489 +++++++++ .../perry-runtime/src/object/field_get_set.rs | 5 +- .../src/object/field_get_set/ic_miss.rs | 36 +- .../object/field_set_by_name/fast_paths.rs | 9 +- .../src/object/field_set_by_name/tail.rs | 26 + crates/perry-runtime/src/object/mod.rs | 100 +- crates/perry-runtime/src/object/shapes.rs | 199 +++- .../perry-runtime/src/object/shapes_tests.rs | 21 + crates/perry-runtime/src/object/tests.rs | 19 +- crates/perry-runtime/src/symbol.rs | 24 +- crates/perry-runtime/src/symbol/accessors.rs | 9 +- crates/perry-runtime/src/symbol/get.rs | 278 +++++ crates/perry-runtime/src/symbol/properties.rs | 2 + .../perry-runtime/src/value/dynamic_object.rs | 26 +- crates/perry-runtime/src/value/mod.rs | 1 + 80 files changed, 8865 insertions(+), 483 deletions(-) create mode 100644 crates/perry-codegen/src/codegen/guarded_falsy_default_method_tests.rs create mode 100644 crates/perry-codegen/src/expr/bitset_test.rs create mode 100644 crates/perry-codegen/src/expr/unary_bitnot_tests.rs create mode 100644 crates/perry-codegen/src/stmt/cached_field_index_return.rs create mode 100644 crates/perry-runtime/src/object/array_tail_transition.rs diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index 4212abeaf2..e3d9ccbc12 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -998,6 +998,8 @@ pub(super) fn compile_closure( current_block: 0, discard_expr_value: false, discard_this_expr: false, + truthy_call_result_requested: false, + pending_truthy_call_result: None, func_names, strings, loop_targets: Vec::new(), @@ -1210,6 +1212,7 @@ pub(super) fn compile_closure( was_unrolled: false, ic_site_counter: ic_base, ic_globals: Vec::new(), + property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), buffer_view_slots: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index 943f7f6bb4..0a1cf69cc5 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -778,6 +778,8 @@ pub(super) fn compile_module_entry( current_block: 0, discard_expr_value: false, discard_this_expr: false, + truthy_call_result_requested: false, + pending_truthy_call_result: None, func_names, strings, loop_targets: Vec::new(), @@ -981,6 +983,7 @@ pub(super) fn compile_module_entry( was_unrolled: hir.init_was_unrolled, ic_site_counter: ic_base, ic_globals: Vec::new(), + property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), buffer_view_slots: HashMap::new(), @@ -1489,6 +1492,8 @@ pub(super) fn compile_module_entry( current_block: 0, discard_expr_value: false, discard_this_expr: false, + truthy_call_result_requested: false, + pending_truthy_call_result: None, func_names, strings, loop_targets: Vec::new(), @@ -1692,6 +1697,7 @@ pub(super) fn compile_module_entry( was_unrolled: hir.init_was_unrolled, ic_site_counter: ic_base, ic_globals: Vec::new(), + property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), buffer_view_slots: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 4c01eda022..5f5dce0f52 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -1032,6 +1032,8 @@ pub(super) fn compile_function( current_block: 0, discard_expr_value: false, discard_this_expr: false, + truthy_call_result_requested: false, + pending_truthy_call_result: None, func_names, strings, loop_targets: Vec::new(), @@ -1231,6 +1233,7 @@ pub(super) fn compile_function( was_unrolled: f.was_unrolled, ic_site_counter: ic_base, ic_globals: Vec::new(), + property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), buffer_view_slots: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/guarded_falsy_default_method_tests.rs b/crates/perry-codegen/src/codegen/guarded_falsy_default_method_tests.rs new file mode 100644 index 0000000000..895bb412b3 --- /dev/null +++ b/crates/perry-codegen/src/codegen/guarded_falsy_default_method_tests.rs @@ -0,0 +1,176 @@ +//! Guarded omitted-argument/false-field indexed method versioning. + +use crate::{compile_module, CompileOptions}; +use perry_hir::types::Type; +use perry_hir::{Class, ClassField, CompareOp, Expr, Function, Module, Param, Stmt}; + +const ROWS: u32 = 30; +const INDEX: u32 = 31; +const DEFER: u32 = 32; + +fn param(id: u32, name: &str) -> Param { + Param { + id, + name: name.to_string(), + ty: Type::Any, + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + } +} + +fn default_get() -> Expr { + Expr::PropertyGet { + object: Box::new(Expr::This), + property: "DEFAULT_DEFER".to_string(), + byte_offset: 0, + } +} + +fn candidate_method() -> Function { + let mut defer = param(DEFER, "defer"); + defer.default = Some(default_get()); + Function { + id: 40, + name: "update".to_string(), + type_params: Vec::new(), + params: vec![param(ROWS, "rows"), param(INDEX, "index"), defer], + return_type: Type::Any, + body: vec![ + Stmt::If { + condition: Expr::Compare { + op: CompareOp::Eq, + left: Box::new(Expr::LocalGet(DEFER)), + right: Box::new(Expr::Undefined), + }, + then_branch: vec![Stmt::Expr(Expr::LocalSet(DEFER, Box::new(default_get())))], + else_branch: None, + }, + // Nominates the existing nonnegative-index family. + Stmt::Expr(Expr::IndexGet { + object: Box::new(Expr::LocalGet(ROWS)), + index: Box::new(Expr::LocalGet(INDEX)), + }), + Stmt::If { + condition: Expr::LocalGet(DEFER), + then_branch: vec![Stmt::Return(Some(Expr::Integer(1)))], + else_branch: Some(vec![Stmt::Return(Some(Expr::Integer(2)))]), + }, + ], + is_async: false, + is_generator: false, + is_strict: true, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + } +} + +fn fixture(method: Function) -> Module { + let class = Class { + id: 41, + name: "Store".to_string(), + type_params: Vec::new(), + extends: None, + extends_name: None, + native_extends: None, + extends_expr: None, + heritage_lexically_shadowed: false, + fields: vec![ClassField { + name: "DEFAULT_DEFER".to_string(), + key_expr: None, + ty: Type::Any, + init: Some(Expr::Bool(false)), + is_private: false, + is_readonly: false, + decorators: Vec::new(), + }], + constructor: None, + methods: vec![method], + getters: Vec::new(), + setters: Vec::new(), + static_accessor_names: Vec::new(), + static_accessor_fn_ids: Vec::new(), + computed_members: Vec::new(), + static_fields: Vec::new(), + static_methods: Vec::new(), + decorators: Vec::new(), + is_exported: false, + aliases: Vec::new(), + is_nested: false, + alloc_width_hint: 0, + specialized_from: None, + }; + let mut module = Module::new("guarded_falsy_default_method.ts"); + module.classes = vec![class]; + module +} + +fn emit(method: Function) -> String { + let opts = CompileOptions { + emit_ir_only: true, + output_type: "executable".to_string(), + ..Default::default() + }; + String::from_utf8(compile_module(&fixture(method), opts).expect("fixture compiles")) + .expect("LLVM IR is UTF-8") +} + +fn function_body<'a>(ir: &'a str, marker: &str) -> &'a str { + let start = ir + .match_indices("define ") + .find(|(index, _)| { + let end = ir[*index..] + .find('\n') + .map(|offset| index + offset) + .unwrap_or(ir.len()); + ir[*index..end].contains(marker) + }) + .map(|(index, _)| index) + .unwrap_or_else(|| panic!("missing function containing {marker}:\n{ir}")); + let end = ir[start..] + .find("\n}") + .map(|offset| start + offset) + .expect("function terminator"); + &ir[start..end] +} + +#[test] +fn wrapper_proves_live_false_field_and_clone_erases_default_and_branch() { + let ir = emit(candidate_method()); + let base = "perry_method_guarded_falsy_default_method_ts__Store__update"; + let index = format!("{base}$idx_u31_{INDEX}"); + let specialized = format!("{index}$default_false2"); + let wrapper = function_body(&ir, &format!("@{base}(")); + let ordinary = function_body(&ir, &format!("@{index}(")); + let false_default = function_body(&ir, &format!("@{specialized}(")); + + assert!(wrapper.contains(&crate::nanbox::TAG_UNDEFINED_I64.to_string())); + assert!(wrapper.contains(&crate::nanbox::TAG_FALSE_I64.to_string())); + assert!(wrapper.contains("load i32, ptr @perry_class_shape_id_")); + assert!(wrapper.contains(&format!("@{specialized}("))); + assert!(wrapper.contains(&format!("@{index}("))); + assert!(ordinary.contains("@js_is_truthy("), "{ordinary}"); + assert!( + !false_default.contains("@js_is_truthy("), + "the guarded clone retained the known-false condition:\n{false_default}" + ); + assert!( + !false_default.contains("class_field_get"), + "the guarded clone reevaluated its already-proved default field:\n{false_default}" + ); +} + +#[test] +fn arbitrary_parameter_use_rejects_the_clone() { + let mut method = candidate_method(); + method.body.push(Stmt::Return(Some(Expr::LocalGet(DEFER)))); + let ir = emit(method); + assert!( + !ir.contains("$default_false"), + "a parameter whose actual false value remains observable was specialized" + ); +} diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index 26a690ed44..26370c3425 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -358,6 +358,20 @@ pub(super) fn apply_pshape_inline_policy( } } +/// Maximum pre-optimization LLVM IR body size admitted to the native-roots +/// pre-statepoint inliner for a guarded specialization. +/// +/// HIR statement count is deliberately not used here: one source statement +/// can lower to a large property/index dispatch lattice. Sixteen KiB admits +/// compact exact-receiver and nonnegative-index leaves while rejecting bodies +/// such as mutation-heavy ECS transitions by nearly an order of magnitude. +pub(super) const GUARDED_SPECIALIZATION_PREINLINE_MAX_IR_BYTES: usize = 16 * 1024; + +#[inline] +pub(super) fn guarded_specialization_fits_preinline_budget(ir_bytes: usize) -> bool { + ir_bytes <= GUARDED_SPECIALIZATION_PREINLINE_MAX_IR_BYTES +} + /// Maximum total (module-wide) direct call sites a function may have and still /// be hinted. This is the anti-bloat backstop: the raised `-inlinehint-threshold` /// lifts LLVM's ceiling for a hinted callee at *every* one of its call sites, so @@ -1470,6 +1484,23 @@ mod sanitize_tests { } } +#[cfg(test)] +mod guarded_specialization_preinline_tests { + use super::{ + guarded_specialization_fits_preinline_budget, GUARDED_SPECIALIZATION_PREINLINE_MAX_IR_BYTES, + }; + + #[test] + fn generated_ir_budget_is_inclusive_and_bounded() { + assert!(guarded_specialization_fits_preinline_budget( + GUARDED_SPECIALIZATION_PREINLINE_MAX_IR_BYTES + )); + assert!(!guarded_specialization_fits_preinline_budget( + GUARDED_SPECIALIZATION_PREINLINE_MAX_IR_BYTES + 1 + )); + } +} + #[cfg(test)] mod resolve_target_triple_tests { use super::resolve_target_triple; diff --git a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs index d0c1812609..489d615b90 100644 --- a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs +++ b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs @@ -10,7 +10,7 @@ use crate::{compile_module, CompileOptions}; use perry_hir::types::Type; -use perry_hir::{Class, Expr, Function, Module, ModuleInitKind, Param, Stmt}; +use perry_hir::{Class, ClassField, Expr, Function, Module, ModuleInitKind, Param, Stmt}; const COLUMN_ID: u32 = 11; const INDEX_ID: u32 = 12; @@ -141,6 +141,117 @@ fn reader_class() -> Class { } } +fn shaped_reader_class() -> Class { + let mut defer = param(13, "defer", Type::Any); + defer.default = Some(Expr::PropertyGet { + object: Box::new(Expr::This), + property: "defaultFlag".to_string(), + byte_offset: 0, + }); + let read = function( + 93, + "read", + vec![param(INDEX_ID, "index", Type::Any), defer], + Type::Any, + vec![ + Stmt::Let { + id: 14, + name: "value".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::IndexGet { + object: Box::new(Expr::PropertyGet { + object: Box::new(Expr::This), + property: "column".to_string(), + byte_offset: 0, + }), + index: Box::new(Expr::LocalGet(INDEX_ID)), + }), + }, + Stmt::Return(Some(Expr::This)), + ], + ); + let mut class = reader_class(); + class.id = 101; + class.name = "ShapedReader".to_string(); + class.fields = vec![ + ClassField { + name: "column".to_string(), + key_expr: None, + ty: Type::Array(Box::new(Type::Any)), + init: None, + is_private: false, + is_readonly: false, + decorators: Vec::new(), + }, + ClassField { + name: "defaultFlag".to_string(), + key_expr: None, + ty: Type::Boolean, + init: None, + is_private: false, + is_readonly: false, + decorators: Vec::new(), + }, + ]; + class.methods = vec![read]; + class +} + +fn shaped_push_class() -> Class { + const OBSERVED_ID: u32 = 31; + let append = function( + 94, + "append", + vec![param(INDEX_ID, "entity", Type::Any)], + Type::Any, + vec![ + // Give the method selector a constructive nonnegative-index use; + // the real SparseSet method has the same proof through sparse[x]. + Stmt::Let { + id: OBSERVED_ID, + name: "observed".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::IndexGet { + object: Box::new(Expr::PropertyGet { + object: Box::new(Expr::This), + property: "packed".to_string(), + byte_offset: 0, + }), + index: Box::new(Expr::LocalGet(INDEX_ID)), + }), + }, + Stmt::Expr(Expr::NativeMethodCall { + module: "array".to_string(), + class_name: None, + object: Some(Box::new(Expr::PropertyGet { + object: Box::new(Expr::This), + property: "packed".to_string(), + byte_offset: 0, + })), + method: "push_single".to_string(), + args: vec![Expr::LocalGet(INDEX_ID)], + }), + Stmt::Return(Some(Expr::This)), + ], + ); + let mut class = reader_class(); + class.id = 102; + class.name = "PackedOwner".to_string(); + class.fields = vec![ClassField { + name: "packed".to_string(), + key_expr: None, + ty: Type::Array(Box::new(Type::Any)), + init: None, + is_private: false, + is_readonly: false, + decorators: Vec::new(), + }]; + class.methods = vec![append]; + class +} + fn method_call(receiver: Expr, column: Expr, index: Expr) -> Expr { Expr::Call { callee: Box::new(Expr::PropertyGet { @@ -230,6 +341,32 @@ fn emit() -> String { .expect("LLVM IR is UTF-8") } +fn emit_shaped_reader() -> String { + let mut module = Module::new("shaped_index_method_clone.ts"); + module.classes = vec![shaped_reader_class()]; + module.init_kind = ModuleInitKind::Eager; + let opts = CompileOptions { + emit_ir_only: true, + output_type: "executable".to_string(), + ..Default::default() + }; + String::from_utf8(compile_module(&module, opts).expect("shaped reader compiles")) + .expect("LLVM IR is UTF-8") +} + +fn emit_shaped_push() -> String { + let mut module = Module::new("shaped_push_method_clone.ts"); + module.classes = vec![shaped_push_class()]; + module.init_kind = ModuleInitKind::Eager; + let opts = CompileOptions { + emit_ir_only: true, + output_type: "executable".to_string(), + ..Default::default() + }; + String::from_utf8(compile_module(&module, opts).expect("shaped push compiles")) + .expect("LLVM IR is UTF-8") +} + fn emit_checked_reader() -> String { let mut class = reader_class(); class.methods = vec![checked_read_method()]; @@ -245,6 +382,128 @@ fn emit_checked_reader() -> String { .expect("LLVM IR is UTF-8") } +fn bitset_class() -> Class { + const MASK_ID: u32 = 61; + const BIT_INDEX_ID: u32 = 62; + let has = function( + 97, + "has", + vec![ + param(MASK_ID, "mask", Type::Any), + param(BIT_INDEX_ID, "index", Type::Any), + ], + Type::Any, + vec![Stmt::Return(Some(Expr::Binary { + op: perry_hir::BinaryOp::BitAnd, + left: Box::new(Expr::IndexGet { + object: Box::new(Expr::LocalGet(MASK_ID)), + index: Box::new(Expr::Unary { + op: perry_hir::UnaryOp::BitNot, + operand: Box::new(Expr::Unary { + op: perry_hir::UnaryOp::BitNot, + operand: Box::new(Expr::Binary { + op: perry_hir::BinaryOp::Div, + left: Box::new(Expr::LocalGet(BIT_INDEX_ID)), + right: Box::new(Expr::Integer(32)), + }), + }), + }), + }), + right: Box::new(Expr::Binary { + op: perry_hir::BinaryOp::Shl, + left: Box::new(Expr::Integer(1)), + right: Box::new(Expr::Binary { + op: perry_hir::BinaryOp::Mod, + left: Box::new(Expr::LocalGet(BIT_INDEX_ID)), + right: Box::new(Expr::Integer(32)), + }), + }), + }))], + ); + let mut class = reader_class(); + class.id = 103; + class.name = "Bitset".to_string(); + class.fields.clear(); + class.methods = vec![has]; + class +} + +fn emit_bitset() -> String { + let mut module = Module::new("u32_bitset_method_clone.ts"); + module.classes = vec![bitset_class()]; + module.init_kind = ModuleInitKind::Eager; + let opts = CompileOptions { + emit_ir_only: true, + output_type: "executable".to_string(), + ..Default::default() + }; + String::from_utf8(compile_module(&module, opts).expect("bitset method compiles")) + .expect("LLVM IR is UTF-8") +} + +fn transition_class() -> Class { + const OWNER_ID: u32 = 70; + const TRANSITION_INDEX_ID: u32 = 71; + let access = || Expr::IndexGet { + object: Box::new(Expr::PropertyGet { + object: Box::new(Expr::LocalGet(OWNER_ID)), + property: "change".to_string(), + byte_offset: 0, + }), + index: Box::new(Expr::LocalGet(TRANSITION_INDEX_ID)), + }; + let receiver = || Expr::PropertyGet { + object: Box::new(Expr::LocalGet(OWNER_ID)), + property: "change".to_string(), + byte_offset: 0, + }; + let transition = function( + 98, + "transition", + vec![ + param(OWNER_ID, "owner", Type::Any), + param(TRANSITION_INDEX_ID, "index", Type::Any), + ], + Type::Any, + vec![ + Stmt::If { + condition: Expr::Unary { + op: perry_hir::UnaryOp::Not, + operand: Box::new(access()), + }, + then_branch: vec![Stmt::Expr(Expr::PutValueSet { + target: Box::new(receiver()), + key: Box::new(Expr::LocalGet(TRANSITION_INDEX_ID)), + value: Box::new(Expr::Integer(7)), + receiver: Box::new(receiver()), + strict: true, + })], + else_branch: None, + }, + Stmt::Return(Some(access())), + ], + ); + let mut class = reader_class(); + class.id = 104; + class.name = "TransitionCache".to_string(); + class.fields.clear(); + class.methods = vec![transition]; + class +} + +fn emit_transition() -> String { + let mut module = Module::new("cached_transition_method_clone.ts"); + module.classes = vec![transition_class()]; + module.init_kind = ModuleInitKind::Eager; + let opts = CompileOptions { + emit_ir_only: true, + output_type: "executable".to_string(), + ..Default::default() + }; + String::from_utf8(compile_module(&module, opts).expect("transition method compiles")) + .expect("LLVM IR is UTF-8") +} + fn emit_versioned_checked_reader_loop() -> String { const ENTITIES: u32 = 20; const COLUMN: u32 = 21; @@ -376,6 +635,7 @@ fn proven_index_routes_to_live_clone_while_unproven_index_keeps_public_fallback( let clone = function_body(&ir, &format!("@{clone_symbol}(")); let public_symbol = "perry_method_index_method_clone_ts__Reader__read"; let public = function_body(&ir, &format!("@{public_symbol}(")); + let generic = function_body(&ir, &format!("@{public_symbol}$generic(")); assert!( clone.lines().next().is_some_and(|line| line.contains(" alwaysinline ")), @@ -385,15 +645,16 @@ fn proven_index_routes_to_live_clone_while_unproven_index_keeps_public_fallback( public .lines() .next() - .is_some_and(|line| !line.contains(" alwaysinline ")), - "the public fallback must not consume the scoped pre-statepoint code-size budget:\n{public}" + .is_some_and(|line| line.contains(" alwaysinline ")), + "a compact guarded public entry must flatten before RS4GC so its admitted clone does not leave a second native call boundary:\n{public}" ); assert!( - clone.contains("fptosi double %arg12 to i32") + !clone.contains("js_typed_i32_arg_to_raw") + && clone.contains("fptosi double") && clone.contains("arr.guard.deref") && clone.contains("call double @js_typed_feedback_array_index_get_fallback_boxed("), - "the clone must consume the integer proof through a guarded direct-slot tier:\n{clone}" + "the clone must decode the established integer proof inline and use a guarded direct-slot tier:\n{clone}" ); assert!( !clone.contains("js_array_get_index_or_string"), @@ -405,11 +666,22 @@ fn proven_index_routes_to_live_clone_while_unproven_index_keeps_public_fallback( .filter(|line| line.contains(&format!("call double @{clone_symbol}("))) .collect(); assert!( - !clone_calls.is_empty() - && clone_calls - .iter() - .all(|line| line.trim_end().ends_with("double 0.0)")), - "every emitted body clone must route only the proven zero-index call:\n{clone_calls:#?}\n{ir}" + clone_calls + .iter() + .any(|line| line.trim_end().ends_with("double 0.0)")), + "the statically proven zero-index call must route directly to the clone:\n{clone_calls:#?}\n{ir}" + ); + assert!( + !public.contains("js_typed_i32_arg_guard") + && !public.contains("js_typed_i32_arg_to_raw") + && public.contains("fptosi double") + && public.contains("-9223372036854775808") + && public.contains("icmp sge i32") + && public.contains("nonnegative_index_method.fast") + && public.contains("nonnegative_index_method.generic") + && public.contains(&format!("call double @{clone_symbol}(")) + && public.contains(&format!("call double @{public_symbol}$generic(")), + "the stable public entry must guard erased live values once and preserve a generic miss:\n{public}" ); let public_calls = ir .lines() @@ -420,8 +692,164 @@ fn proven_index_routes_to_live_clone_while_unproven_index_keeps_public_fallback( "negative, fractional, and unproven Number calls must retain the public fallback:\n{ir}" ); assert!( - public.contains("aidxkey.sso") && public.contains("js_array_get_index_or_string"), - "the public body must preserve arbitrary JavaScript property-key semantics:\n{public}" + generic.contains("aidxkey.sso") && generic.contains("js_array_get_index_or_string"), + "the generic miss body must preserve arbitrary JavaScript property-key semantics:\n{generic}" + ); +} + +#[test] +fn receiver_shape_and_live_index_proofs_compose_without_losing_either_fallback() { + let _native = crate::codegen::helpers::NativeRootsPin::native(); + let ir = emit_shaped_reader(); + let public = "perry_method_shaped_index_method_clone_ts__ShapedReader__read"; + let pshape = crate::collectors::pshape_method_name(public); + let pshape_generic = format!("{pshape}$generic"); + let combined = format!("{pshape}$idx_u31_{INDEX_ID}"); + let wrapper = function_body(&ir, &format!("@{pshape}(")); + let generic = function_body(&ir, &format!("@{pshape_generic}(")); + let fast = function_body(&ir, &format!("@{combined}(")); + + assert!( + wrapper + .lines() + .next() + .is_some_and(|line| line.starts_with("define double ")) + && !wrapper.contains("js_typed_i32_arg_guard") + && !wrapper.contains("js_typed_i32_arg_to_raw") + && wrapper.contains("fptosi double") + && wrapper.contains("-9223372036854775808") + && wrapper.contains(&format!("call double @{combined}(")) + && wrapper.contains(&format!("call double @{pshape_generic}(")), + "the published receiver-shape capability must guard the live index and retain its receiver-safe miss:\n{wrapper}" + ); + assert!( + generic.contains("js_array_get_index_or_string") + && !generic.contains("js_object_get_field_by_name"), + "the pshape generic arm must preserve arbitrary keys without re-looking up this.column:\n{generic}" + ); + assert!( + !fast.contains("js_array_get_index_or_string") + && !fast.contains("js_object_get_field_by_name") + && !fast.contains("js_typed_i32_arg_to_raw") + && fast.contains("fptosi double"), + "the composed clone must consume both proofs in the same body:\n{fast}" + ); +} + +#[test] +fn combined_receiver_and_u31_clone_fuses_property_array_push() { + let _native = crate::codegen::helpers::NativeRootsPin::native(); + let ir = emit_shaped_push(); + let public = "perry_method_shaped_push_method_clone_ts__PackedOwner__append"; + let pshape = crate::collectors::pshape_method_name(public); + let combined = format!("{pshape}$idx_u31_{INDEX_ID}"); + let fast = function_body(&ir, &format!("@{combined}(")); + let generic = function_body(&ir, &format!("@{pshape}$generic(")); + + assert!( + fast.contains("call i64 @js_array_push_u31_with_length") + && !fast.contains("call void @js_array_push_guard") + && !fast.contains("call i64 @js_array_push_f64") + && !fast.contains("call i32 @js_array_length"), + "the composed clone must consume the u31 proof in one push/length runtime entry:\n{fast}" + ); + assert!( + generic.contains("call void @js_array_push_guard") + && generic.contains("call i64 @js_array_push_f64") + && generic.contains("call i32 @js_array_length") + && !generic.contains("js_array_push_u31_with_length"), + "the receiver-safe generic miss must retain arbitrary value and receiver semantics:\n{generic}" + ); +} + +#[test] +fn u31_bitset_clone_uses_one_guarded_uint32_load_and_keeps_dynamic_miss() { + const BIT_INDEX_ID: u32 = 62; + let _native = crate::codegen::helpers::NativeRootsPin::native(); + let class = bitset_class(); + assert_eq!( + super::typed_abi::nonnegative_index_method_params(&class.methods[0]), + vec![BIT_INDEX_ID], + "the receiver mask is not itself a numeric index" + ); + + let ir = emit_bitset(); + let public = "perry_method_u32_bitset_method_clone_ts__Bitset__has"; + let clone = function_body(&ir, &format!("@{public}$idx_u31_{BIT_INDEX_ID}(")); + let generic = function_body(&ir, &format!("@{public}$generic(")); + + assert!( + clone.contains("u32bitset.header") + && clone.contains("u32bitset.fast") + && clone.contains("lshr i32") + && clone.contains("and i32") + && clone.contains(", 31") + && clone.contains("shl i32 1") + && clone.contains("icmp eq i64") + && clone.contains(", 5") + && clone.contains("load i32") + && clone.contains("call double @js_dyn_index_get(") + && clone.contains("call double @js_dynamic_bitand(") + && !clone.contains("tav.k.i8") + && !clone.contains("tav.k.f64"), + "the u31 clone must use the monomorphic Uint32 bitset tier and a canonical miss:\n{clone}" + ); + assert!( + generic.contains("tav.k.i8") + && generic.contains("tav.k.f64") + && generic.contains("call double @js_dynamic_bitand("), + "the unproven body must retain the full dynamic typed-array and BigInt behavior:\n{generic}" + ); +} + +#[test] +fn u31_transition_clone_returns_a_proved_cached_array_hit_without_second_get() { + const TRANSITION_INDEX_ID: u32 = 71; + let _native = crate::codegen::helpers::NativeRootsPin::native(); + let class = transition_class(); + assert_eq!( + super::typed_abi::nonnegative_index_method_params(&class.methods[0]), + vec![TRANSITION_INDEX_ID] + ); + + let ir = emit_transition(); + let public = "perry_method_cached_transition_method_clone_ts__TransitionCache__transition"; + let clone = function_body(&ir, &format!("@{public}$idx_u31_{TRANSITION_INDEX_ID}(")); + let generic = function_body(&ir, &format!("@{public}$generic(")); + + assert!( + clone.contains("cached_field_index.object_header") + && clone.contains("cached_field_index.prefix_token") + && clone.contains("cached_field_index.array_header") + && clone.contains("cached_field_index.array_load") + && clone.contains("cached_field_index.return") + && clone + .split("\ncached_field_index.return.") + .nth(1) + .and_then(|tail| tail.split("\ncached_field_index.normal.").next()) + .is_some_and(|fast_return| fast_return.contains("ret double")) + && clone.contains("cached_field_index.normal") + && clone.contains("tav.k.i8") + && clone.contains("if.then"), + "the proved truthy hit must return directly while the complete original body remains as fallback:\n{clone}" + ); + let first_cache = clone + .lines() + .find(|line| line.contains("cached_field_index") && line.contains("@perry_ic_")) + .or_else(|| clone.lines().find(|line| line.contains("@perry_ic_"))) + .and_then(|line| line.split('@').nth(1)) + .and_then(|tail| { + tail.split(|c: char| !(c.is_ascii_alphanumeric() || c == '_')) + .next() + }) + .unwrap_or_else(|| panic!("guard has no property cache reference:\n{clone}")); + assert!( + clone.matches(&format!("@{first_cache}")).count() >= 6, + "the speculative guard and original property reads must share one primed cache ({first_cache}):\n{clone}" + ); + assert!( + !generic.contains("cached_field_index"), + "an unproved/negative index must retain only the original generic behavior:\n{generic}" ); } @@ -433,6 +861,29 @@ fn selector_rejects_mutated_defaulted_and_closure_captured_indices() { vec![INDEX_ID] ); + let mut erased = candidate.clone(); + erased.params[1].ty = Type::Any; + assert_eq!( + super::typed_abi::nonnegative_index_method_params(&erased), + vec![INDEX_ID], + "plain JavaScript lowers entity-id parameters to Any" + ); + + let mut unknown = candidate.clone(); + unknown.params[1].ty = Type::Unknown; + assert_eq!( + super::typed_abi::nonnegative_index_method_params(&unknown), + vec![INDEX_ID] + ); + + let mut unrelated_default = candidate.clone(); + unrelated_default.params[0].default = Some(Expr::Undefined); + assert_eq!( + super::typed_abi::nonnegative_index_method_params(&unrelated_default), + vec![INDEX_ID], + "a default on an unrelated parameter does not alter the index proof" + ); + let mut mutated = candidate.clone(); mutated.body.insert( 0, @@ -466,6 +917,63 @@ fn selector_rejects_mutated_defaulted_and_closure_captured_indices() { assert!(super::typed_abi::nonnegative_index_method_params(&captured).is_empty()); } +#[test] +fn selector_does_not_guard_an_object_whose_field_produces_the_index() { + const SOURCE_ID: u32 = 51; + const ARRAY_ID: u32 = 52; + const DERIVED_ID: u32 = 53; + let from_object = function( + 96, + "fromObject", + vec![ + param(SOURCE_ID, "source", Type::Any), + param(ARRAY_ID, "array", Type::Array(Box::new(Type::Any))), + ], + Type::Any, + vec![ + Stmt::Let { + id: DERIVED_ID, + name: "derived".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::PropertyGet { + object: Box::new(Expr::LocalGet(SOURCE_ID)), + property: "id".to_string(), + byte_offset: 0, + }), + }, + Stmt::Return(Some(Expr::IndexGet { + object: Box::new(Expr::LocalGet(ARRAY_ID)), + index: Box::new(Expr::LocalGet(DERIVED_ID)), + })), + ], + ); + assert!( + super::typed_abi::nonnegative_index_method_params(&from_object).is_empty(), + "the component-like object is a base used to obtain an index, not a numeric index argument" + ); + + let mut numeric_flow = from_object; + numeric_flow.name = "fromNumber".to_string(); + numeric_flow.params[0].ty = Type::Number; + numeric_flow.body[0] = Stmt::Let { + id: DERIVED_ID, + name: "derived".to_string(), + ty: Type::Number, + mutable: false, + init: Some(Expr::Binary { + op: perry_hir::BinaryOp::Add, + left: Box::new(Expr::LocalGet(SOURCE_ID)), + right: Box::new(Expr::Integer(0)), + }), + }; + assert_eq!( + super::typed_abi::nonnegative_index_method_params(&numeric_flow), + vec![SOURCE_ID], + "an annotated numeric parameter still propagates through arithmetic into an index" + ); +} + #[test] fn checked_reader_gets_a_handle_abi_clone_with_no_array_fallback() { let method = checked_read_method(); diff --git a/crates/perry-codegen/src/codegen/indexed_method_artifacts.rs b/crates/perry-codegen/src/codegen/indexed_method_artifacts.rs index 2f30812736..84eba7bab5 100644 --- a/crates/perry-codegen/src/codegen/indexed_method_artifacts.rs +++ b/crates/perry-codegen/src/codegen/indexed_method_artifacts.rs @@ -92,6 +92,90 @@ pub(super) fn compile_indexed_method_clones( ) })?; + if cross_module + .guarded_falsy_field_default_methods + .contains_key(&(class.name.clone(), method.name.clone())) + { + compile_method( + llmod, + class, + method, + func_names, + strings, + classes, + methods, + module_globals, + module_global_types, + import_function_prefixes, + enums, + static_field_globals, + class_ids, + func_signatures, + func_synthetic_arguments, + module_boxed_vars, + closure_rest_params, + cross_module, + None, + false, + None, + Some(nonnegative_index_params), + false, + false, + true, + false, + ) + .with_context(|| { + format!( + "lowering guarded false-field-default indexed clone '{}::{}'", + class.name, method.name + ) + })?; + } + + // Compose the exact receiver-shape and nonnegative-index facts in one + // body. The externally published proven-receiver wrapper is emitted with + // the receiver-shaped generic body below; it guards the live index and + // routes here without giving up fixed receiver-field offsets. + if let Some(fact) = cross_module + .pshape_methods + .get(&(class.name.clone(), method.name.clone())) + { + compile_method( + llmod, + class, + method, + func_names, + strings, + classes, + methods, + module_globals, + module_global_types, + import_function_prefixes, + enums, + static_field_globals, + class_ids, + func_signatures, + func_synthetic_arguments, + module_boxed_vars, + closure_rest_params, + cross_module, + None, + false, + Some(fact.clone()), + Some(nonnegative_index_params), + false, + false, + false, + false, + ) + .with_context(|| { + format!( + "lowering receiver-shaped nonnegative-index clone '{}::{}'", + class.name, method.name + ) + })?; + } + if super::typed_abi::nonnegative_index_fast_array_params(method, nonnegative_index_params) .is_empty() { diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index 3435682882..8ed9c76462 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -14,7 +14,8 @@ use crate::types::{LlvmType, DOUBLE, I1, I32, I64, PTR}; use super::helpers::{node_stream_parent_kind, scoped_static_method_name}; use super::method_trampolines::{ - emit_guarded_undefined, emit_public_generic, emit_public_typed, guarded_undefined_name, + emit_guarded_nonnegative_index, emit_guarded_undefined, emit_public_generic, emit_public_typed, + guarded_falsy_field_default_name, guarded_undefined_name, }; use super::opts::CrossModuleCtx; use super::typed_abi::{ @@ -75,6 +76,11 @@ pub(super) fn compile_method( // primary (`proven_this: None`) invocation for this same method. let is_pshape_clone = proven_this.is_some() && !pshape_arg_clone; let is_index_clone = nonnegative_index_params.is_some(); + let guarded_index_family = !is_index_clone + && force_generic_body + && cross_module + .nonnegative_index_methods + .contains_key(&(class.name.clone(), method.name.clone())); let pshape_arg_plan = pshape_arg_clone .then(|| { cross_module @@ -90,6 +96,12 @@ pub(super) fn compile_method( .copied() }) .flatten(); + let guarded_falsy_field_default = cross_module + .guarded_falsy_field_default_methods + .get(&(class.name.clone(), method.name.clone())) + .copied(); + let guarded_clone_param = guarded_undefined_param + .or_else(|| guarded_falsy_field_default.map(|candidate| candidate.param_index)); let fast_array_param_ids = if fast_array_handle_clone { crate::codegen::typed_abi::nonnegative_index_fast_array_params( method, @@ -98,12 +110,14 @@ pub(super) fn compile_method( } else { Vec::new() }; - debug_assert!(!(is_pshape_clone && is_index_clone)); debug_assert!(!fast_array_handle_clone || is_index_clone); + debug_assert!(!fast_array_handle_clone || !is_pshape_clone); debug_assert!(!fast_array_handle_clone || !fast_array_param_ids.is_empty()); debug_assert!(!ptr_array_cache_clone || is_pshape_clone); - debug_assert!(!guarded_undefined_clone || guarded_undefined_param.is_some()); - debug_assert!(!guarded_undefined_clone || !is_index_clone); + debug_assert!(!guarded_undefined_clone || guarded_clone_param.is_some()); + debug_assert!( + !guarded_undefined_clone || !is_index_clone || guarded_falsy_field_default.is_some() + ); debug_assert!(!guarded_undefined_clone || !ptr_array_cache_clone); debug_assert!(!pshape_arg_clone || pshape_arg_plan.is_some()); debug_assert!(!pshape_arg_clone || !is_index_clone); @@ -125,7 +139,15 @@ pub(super) fn compile_method( nonnegative_index_params.expect("fast-array clone has index parameters"), ) } else if let Some(params) = nonnegative_index_params { - crate::codegen::nonnegative_index_method_name(&public_llvm_name, params) + let index_name = crate::codegen::nonnegative_index_method_name(&family_name, params); + if guarded_undefined_clone && guarded_falsy_field_default.is_some() { + guarded_falsy_field_default_name( + &index_name, + guarded_clone_param.expect("falsy-default clone parameter"), + ) + } else { + index_name + } } else if guarded_undefined_clone { guarded_undefined_name( &family_name, @@ -133,6 +155,8 @@ pub(super) fn compile_method( ) } else if guarded_undefined_param.is_some() { generic_method_body_name(&family_name) + } else if guarded_index_family { + generic_method_body_name(&family_name) } else if ptr_array_cache_clone || is_pshape_clone || pshape_arg_clone { family_name.clone() } else if typed_public_trampoline.is_some() || force_generic_body { @@ -154,6 +178,7 @@ pub(super) fn compile_method( let ic_base = llmod.ic_counter; let buffer_alias_base = llmod.buffer_alias_counter; + let lowered_function_index = llmod.function_count(); let lf = llmod.define_function(&llvm_name, DOUBLE, params); // Plain `$pshape` clones are producer-published capabilities and need // external linkage for guarded calls from importing modules. The stricter @@ -166,6 +191,7 @@ pub(super) fn compile_method( || typed_public_trampoline.is_some() || force_generic_body || guarded_undefined_param.is_some() + || (guarded_undefined_clone && guarded_falsy_field_default.is_some()) || pshape_arg_clone { lf.linkage = "internal".to_string(); @@ -175,6 +201,23 @@ pub(super) fn compile_method( lf.pre_statepoint_inline = true; } + // A false-field-default clone is entered only after its public wrapper + // proved the omitted argument, exact receiver layout, and live false slot. + // Remove exactly the corresponding synthetic default prologue; all other + // parameter defaults retain their source order and effects. + let specialized_body = guarded_falsy_field_default + .filter(|_| guarded_undefined_clone) + .map(|candidate| { + method + .body + .iter() + .enumerate() + .filter(|(index, _)| *index != candidate.prologue_stmt_index) + .map(|(_, stmt)| stmt.clone()) + .collect::>() + }); + let method_body = specialized_body.as_deref().unwrap_or(&method.body); + // gh #6206 / #6081: methods were compiled WITHOUT a shadow frame — same // exact-roots liveness hole as closures (see compile_closure). One extra // slot roots the receiver (`this` is a pointer value reachable from @@ -184,14 +227,14 @@ pub(super) fn compile_method( cross_module.flat_const_arrays.keys().copied().collect(); let m = crate::collectors::collect_pointer_typed_locals( &method.params, - &method.body, + method_body, &flat_const_ids, ); crate::codegen::helpers::maybe_spill_roots_to_shadow_frame( lf, &llvm_name, m.len() + 1, - &method.body, + method_body, ); lf.enable_shadow_frame(m.len() as u32 + 1); m @@ -200,7 +243,7 @@ pub(super) fn compile_method( }; let this_shadow_slot_idx = shadow_slot_map.len() as u32; let shadow_slot_clears_after_stmt = - crate::collectors::collect_shadow_slot_clear_points(&method.body, &shadow_slot_map); + crate::collectors::collect_shadow_slot_clear_points(method_body, &shadow_slot_map); let _ = lf.create_block("entry"); @@ -237,11 +280,13 @@ pub(super) fn compile_method( } map.insert(p.id, slot); if index_param_ids.contains(&p.id) { - // The clone is reachable only from a call site that proved - // this argument is in [0, i32::MAX]. The ordinary boxed ABI - // materializes such loop indices as plain doubles, so one - // entry conversion supplies the canonical raw index slot. - let raw_i32 = blk.fptosi(DOUBLE, &arg_name, I32); + // A statically proven route normally passes a plain double; + // the guarded stable public entry may also pass Perry's + // canonical INT32 NaN-box. Both entries prove the exact same + // signed-i32 value class before reaching this body, so use the + // shared already-guarded conversion rather than `fptosi` + // (which cannot consume a tagged INT32 value). + let raw_i32 = super::typed_abi::emit_typed_i32_raw_assuming_guarded(blk, &arg_name); let i32_slot = blk.alloca(I32); blk.store(I32, &raw_i32, &i32_slot); index_i32_param_slots.insert(p.id, i32_slot); @@ -257,7 +302,7 @@ pub(super) fn compile_method( for p in &method.params { local_types.insert(p.id, p.ty.clone()); } - if let Some(index) = guarded_undefined_param.filter(|_| guarded_undefined_clone) { + if let Some(index) = guarded_clone_param.filter(|_| guarded_undefined_clone) { local_types.insert(method.params[index].id, perry_hir::types::Type::Void); } @@ -278,7 +323,7 @@ pub(super) fn compile_method( .is_some(), ); let native_facts = crate::collectors::collect_native_region_fact_graph( - &method.body, + method_body, &[], &flat_const_ids, &clamp_fn_ids, @@ -311,18 +356,18 @@ pub(super) fn compile_method( let repsel_context_denial = repsel_flags.canonical_denial; let report_denial = repsel_flags.report_denial(); let repsel_closure_refs = if repsel_allows || repsel_str_allows || report_denial { - crate::expr::collect_closure_referenced_locals(&method.body) + crate::expr::collect_closure_referenced_locals(method_body) } else { std::collections::HashSet::new() }; let repsel_str_ineligible = if repsel_str_allows || report_denial { - crate::expr::collect_canonical_str_ineligible_locals(&method.body) + crate::expr::collect_canonical_str_ineligible_locals(method_body) } else { std::collections::HashSet::new() }; let mut guarded_param_proofs = index_param_proofs; - if let Some(index) = guarded_undefined_param.filter(|_| guarded_undefined_clone) { + if let Some(index) = guarded_clone_param.filter(|_| guarded_undefined_clone) { guarded_param_proofs.insert(method.params[index].id, perry_hir::types::Type::Void); } if let Some(plan) = pshape_arg_plan { @@ -338,8 +383,8 @@ pub(super) fn compile_method( ) })); } - let mut reassigned_locals = crate::collectors::reassigned_locals(&method.body); - if let Some(index) = guarded_undefined_param.filter(|_| guarded_undefined_clone) { + let mut reassigned_locals = crate::collectors::reassigned_locals(method_body); + if let Some(index) = guarded_clone_param.filter(|_| guarded_undefined_clone) { // Candidate discovery already rejected every user-authored write and // closure capture. The remaining assignment is TypeScript's lowered // optional-parameter prologue (`undefined = undefined`), which cannot @@ -367,6 +412,8 @@ pub(super) fn compile_method( current_block: 0, discard_expr_value: false, discard_this_expr: false, + truthy_call_result_requested: false, + pending_truthy_call_result: None, func_names, strings, loop_targets: Vec::new(), @@ -574,6 +621,7 @@ pub(super) fn compile_method( was_unrolled: method.was_unrolled, ic_site_counter: ic_base, ic_globals: Vec::new(), + property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), buffer_view_slots: HashMap::new(), @@ -648,7 +696,7 @@ pub(super) fn compile_method( super::arguments::materialize_arguments_object( &mut ctx, &method.params, - Some(&method.body), + Some(method_body), super::arguments::ArgumentsCallee::Undefined, ); @@ -675,7 +723,7 @@ pub(super) fn compile_method( // thread-local round trip per construction for a cell no one // reads. Measured: 1.89x on a two-class `new B(x, y)` loop, // 3.14x on `shapes.ts`. - if crate::collectors::body_contains_closure(&method.body) { + if crate::collectors::body_contains_closure(method_body) { crate::expr::this_super_call::push_shared_super_called_slot(&mut ctx); ctx.shared_super_scope_active = true; } else { @@ -809,7 +857,7 @@ pub(super) fn compile_method( (ctor.symbol, ctor.param_count) } else { // No callable ctor symbol — bail. - stmt::lower_stmts(&mut ctx, &method.body).with_context(|| { + stmt::lower_stmts(&mut ctx, method_body).with_context(|| { format!("lowering body of method '{}::{}'", class.name, method.name) })?; // Fall through to the default ret at end. @@ -1127,14 +1175,14 @@ pub(super) fn compile_method( .call(DOUBLE, "js_throw_reference_error_this_before_super", &[]); ctx.block().unreachable(); } else if method.is_async { - stmt::lower_async_rejecting_stmts(&mut ctx, &method.body).with_context(|| { + stmt::lower_async_rejecting_stmts(&mut ctx, method_body).with_context(|| { format!( "lowering async body of method '{}::{}'", class.name, method.name ) })?; } else { - stmt::lower_stmts(&mut ctx, &method.body).with_context(|| { + stmt::lower_stmts(&mut ctx, method_body).with_context(|| { format!("lowering body of method '{}::{}'", class.name, method.name) })?; } @@ -1199,6 +1247,24 @@ pub(super) fn compile_method( let buffer_alias_used = ctx.buffer_data_slots.len() as u32; let native_rep_records = std::mem::take(&mut ctx.native_rep_records); drop(ctx); + + // Under native roots, ordinary `force_inline` is intentionally only an + // LLVM hint: running the inliner after statepoint rewriting duplicates + // relocation scaffolding. Exact-receiver leaves are different. Once the + // body has actually been lowered, admit only compact bodies to the early + // inliner so direct method chains can flatten without guessing from HIR + // statement count. Indexed clones are already admitted above and + // pshape-argument clones do not carry an exact receiver proof. + if is_pshape_clone && !is_index_clone && !method.is_async && !method.is_generator { + let lowered = llmod + .function_mut(lowered_function_index) + .expect("just-lowered method function"); + if super::helpers::guarded_specialization_fits_preinline_budget( + lowered.estimated_ir_bytes(), + ) { + lowered.pre_statepoint_inline = true; + } + } llmod.ic_counter = ic_end; llmod.buffer_alias_counter += buffer_alias_used; llmod.native_rep_records.extend(native_rep_records); @@ -1221,11 +1287,48 @@ pub(super) fn compile_method( // twice. if let Some(param_index) = guarded_undefined_param.filter(|_| !guarded_undefined_clone) { emit_guarded_undefined(llmod, method, &family_name, &llvm_name, param_index); - } else if !is_pshape_clone && !is_index_clone && !guarded_undefined_clone { + } else if !is_index_clone + && !guarded_undefined_clone + && !pshape_arg_clone + && !ptr_array_cache_clone + { if let Some(kind) = typed_public_trampoline { emit_public_typed(llmod, method, &public_llvm_name, &llvm_name, kind); } else if force_generic_body { - emit_public_generic(llmod, method, &public_llvm_name, &llvm_name); + if let Some(params) = cross_module + .nonnegative_index_methods + .get(&(class.name.clone(), method.name.clone())) + { + let wrapper_name = if is_pshape_clone { + &family_name + } else { + &public_llvm_name + }; + let expected_class_id = *class_ids + .get(&class.name) + .expect("method class has a runtime class id"); + let keys_global = cross_module + .class_keys_globals + .get(&class.name) + .expect("method class has a canonical keys global"); + let expected_shape_global = + crate::typed_shape::shape_id_global_name_from_keys_global(keys_global); + let falsy_default = (!is_pshape_clone) + .then_some(guarded_falsy_field_default.as_ref()) + .flatten(); + emit_guarded_nonnegative_index( + llmod, + method, + wrapper_name, + &llvm_name, + params, + expected_class_id, + &expected_shape_global, + falsy_default, + ); + } else { + emit_public_generic(llmod, method, &public_llvm_name, &llvm_name); + } } } Ok(()) @@ -1656,6 +1759,8 @@ pub(super) fn compile_static_method( current_block: 0, discard_expr_value: false, discard_this_expr: false, + truthy_call_result_requested: false, + pending_truthy_call_result: None, func_names, strings, loop_targets: Vec::new(), @@ -1856,6 +1961,7 @@ pub(super) fn compile_static_method( was_unrolled: f.was_unrolled, ic_site_counter: ic_base, ic_globals: Vec::new(), + property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), buffer_view_slots: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/method_trampolines.rs b/crates/perry-codegen/src/codegen/method_trampolines.rs index 3ab1a6c265..6af0003e3c 100644 --- a/crates/perry-codegen/src/codegen/method_trampolines.rs +++ b/crates/perry-codegen/src/codegen/method_trampolines.rs @@ -199,6 +199,213 @@ pub(super) fn emit_public_generic( wf.block_mut(0).unwrap().ret(DOUBLE, &value); } +/// Emit the stable boxed-ABI entry for a method whose selected parameters are +/// consumed as non-negative signed-i32 array indices by `$idx_u31`. +/// +/// The source type is only a nomination: plain JavaScript packages erase these +/// parameters to `Any`, and even a declared `number` can hold a negative, +/// fraction, `-0`, string, Symbol, or BigInt at runtime. Every selected live +/// argument therefore passes the existing exact-i32 guard and a non-negative +/// check before the clone is entered. Any miss calls the original boxed body +/// with the original bits, preserving arbitrary property-key semantics. +pub(super) fn emit_guarded_nonnegative_index( + llmod: &mut LlModule, + method: &Function, + public_name: &str, + generic_body_name: &str, + index_param_ids: &[u32], + expected_class_id: u32, + expected_shape_global: &str, + falsy_field_default: Option<&super::param_guard::GuardedFalsyFieldDefaultMethodCandidate>, +) { + debug_assert!(!index_param_ids.is_empty()); + let clone_name = super::typed_abi::nonnegative_index_method_name(public_name, index_param_ids); + // The private clone has already been lowered. Use its real generated IR + // size—not source statement count—to decide whether this guard plus clone + // may flatten before statepoint rewriting. This makes tiny indexed leaves + // disappear at their direct call sites while keeping large mutation bodies + // behind one native call boundary. + let preinline = llmod + .function_estimated_ir_bytes(&clone_name) + .is_some_and(super::helpers::guarded_specialization_fits_preinline_budget); + let target_triple = llmod.target_triple.clone(); + let mut params: Vec<(LlvmType, String)> = Vec::with_capacity(method.params.len() + 1); + params.push((DOUBLE, "%this_arg".to_string())); + for p in &method.params { + params.push((DOUBLE, format!("%arg{}", p.id))); + } + let wf = llmod.define_function(public_name, DOUBLE, params); + wf.pre_statepoint_inline = preinline; + let _ = wf.create_block("entry"); + + let fast_idx = wf.num_blocks(); + let fast_label = wf + .create_block("nonnegative_index_method.fast") + .label + .clone(); + let generic_idx = wf.num_blocks(); + let generic_label = wf + .create_block("nonnegative_index_method.generic") + .label + .clone(); + + // The conversion helper's contract requires an already-guarded value. + // Build a short-circuiting proof chain so a non-number never reaches it, + // even in debug runtimes where that precondition is asserted. + let mut guard_block_idx = 0; + for (index, param_id) in index_param_ids.iter().enumerate() { + let arg = format!("%arg{param_id}"); + let (exact_i32, raw_i32) = super::typed_abi::emit_typed_i32_guard_and_raw( + wf.block_mut(guard_block_idx).unwrap(), + &arg, + ); + let admitted_idx = wf.num_blocks(); + let admitted_label = wf + .create_block(&format!("nonnegative_index_method.arg{index}.i32")) + .label + .clone(); + wf.block_mut(guard_block_idx) + .unwrap() + .cond_br(&exact_i32, &admitted_label, &generic_label); + + let nonnegative = wf + .block_mut(admitted_idx) + .unwrap() + .icmp_sge(I32, &raw_i32, "0"); + if index + 1 == index_param_ids.len() { + wf.block_mut(admitted_idx) + .unwrap() + .cond_br(&nonnegative, &fast_label, &generic_label); + } else { + guard_block_idx = wf.num_blocks(); + let next_guard_label = wf + .create_block(&format!("nonnegative_index_method.arg{}.guard", index + 1)) + .label + .clone(); + wf.block_mut(admitted_idx).unwrap().cond_br( + &nonnegative, + &next_guard_label, + &generic_label, + ); + } + } + + let mut arg_names: Vec = Vec::with_capacity(method.params.len() + 1); + arg_names.push("%this_arg".to_string()); + for p in &method.params { + arg_names.push(format!("%arg{}", p.id)); + } + let call_args: Vec<(LlvmType, &str)> = + arg_names.iter().map(|arg| (DOUBLE, arg.as_str())).collect(); + if let Some(candidate) = falsy_field_default { + // The index proof gets us here first. Keep the mutable default proof + // as a second guarded diamond: exact omitted argument, exact ordinary + // receiver layout, and exact live canonical-false slot. This wrapper + // runs no user code between the slot load and the private call. + let ordinary_idx = wf.num_blocks(); + let ordinary_label = wf + .create_block("falsy_field_default.ordinary") + .label + .clone(); + let deref_idx = wf.num_blocks(); + let deref_label = wf.create_block("falsy_field_default.deref").label.clone(); + let field_idx = wf.num_blocks(); + let field_label = wf.create_block("falsy_field_default.field").label.clone(); + let specialized_idx = wf.num_blocks(); + let specialized_label = wf + .create_block("falsy_field_default.specialized") + .label + .clone(); + + let guarded_arg = format!("%arg{}", method.params[candidate.param_index].id); + let recv_handle = { + let blk = wf.block_mut(fast_idx).unwrap(); + let arg_bits = blk.bitcast_double_to_i64(&guarded_arg); + let omitted = blk.icmp_eq(I64, &arg_bits, crate::nanbox::TAG_UNDEFINED_I64); + let recv_bits = blk.bitcast_double_to_i64("%this_arg"); + let recv_handle = blk.and(I64, &recv_bits, crate::nanbox::POINTER_MASK_I64); + let tag = blk.lshr(I64, &recv_bits, "48"); + let tagged = blk.icmp_eq(I64, &tag, "32765"); + let heap_floor = + crate::target_layout::heap_addr_lower_bound_inclusive(&target_triple).to_string(); + let heap_ceiling = + crate::target_layout::heap_addr_upper_bound_exclusive(&target_triple).to_string(); + let above_floor = blk.icmp_uge(I64, &recv_handle, &heap_floor); + let below_ceiling = blk.icmp_ult(I64, &recv_handle, &heap_ceiling); + let in_heap = blk.and(I1, &above_floor, &below_ceiling); + let safe = blk.and(I1, &tagged, &in_heap); + let enter = blk.and(I1, &omitted, &safe); + blk.cond_br(&enter, &deref_label, &ordinary_label); + recv_handle + }; + + let obj_ptr = { + let blk = wf.block_mut(deref_idx).unwrap(); + let obj_ptr = blk.inttoptr(I64, &recv_handle); + let gc_header_ptr = blk.gep(crate::types::I8, &obj_ptr, &[(I64, "-8")]); + let gc_header = blk.load(I32, &gc_header_ptr); + let guarded_gc = blk.and(I32, &gc_header, "142639359"); + let gc_ok = blk.icmp_eq(I32, &guarded_gc, "2"); + let class_shape = blk.load(I64, &obj_ptr); + let expected_shape = blk.load(I32, &format!("@{expected_shape_global}")); + let expected_shape_i64 = blk.zext(I32, &expected_shape, I64); + let expected_shape_high = blk.shl(I64, &expected_shape_i64, "32"); + let expected = blk.or(I64, &expected_shape_high, &expected_class_id.to_string()); + let shape_matches = blk.icmp_eq(I64, &class_shape, &expected); + let shape_rel = blk.add(I32, &expected_shape, "-2147483648"); + let shape_valid = blk.icmp_ult(I32, &shape_rel, "1073741824"); + let exact_layout = blk.and(I1, &gc_ok, &shape_matches); + let exact_layout = blk.and(I1, &exact_layout, &shape_valid); + blk.cond_br(&exact_layout, &field_label, &ordinary_label); + obj_ptr + }; + + { + let blk = wf.block_mut(field_idx).unwrap(); + let byte_offset = (16 + candidate.field_index * 8).to_string(); + let field_ptr = blk.gep(crate::types::I8, &obj_ptr, &[(I64, &byte_offset)]); + let field = blk.load(DOUBLE, &field_ptr); + let field_bits = blk.bitcast_double_to_i64(&field); + let is_false = blk.icmp_eq(I64, &field_bits, crate::nanbox::TAG_FALSE_I64); + blk.cond_br(&is_false, &specialized_label, &ordinary_label); + } + + let specialized_name = guarded_falsy_field_default_name(&clone_name, candidate.param_index); + let specialized_value = + wf.block_mut(specialized_idx) + .unwrap() + .call(DOUBLE, &specialized_name, &call_args); + wf.block_mut(specialized_idx) + .unwrap() + .ret(DOUBLE, &specialized_value); + + let ordinary_value = + wf.block_mut(ordinary_idx) + .unwrap() + .call(DOUBLE, &clone_name, &call_args); + wf.block_mut(ordinary_idx) + .unwrap() + .ret(DOUBLE, &ordinary_value); + } else { + let fast_value = wf + .block_mut(fast_idx) + .unwrap() + .call(DOUBLE, &clone_name, &call_args); + wf.block_mut(fast_idx).unwrap().ret(DOUBLE, &fast_value); + } + let generic_value = + wf.block_mut(generic_idx) + .unwrap() + .call(DOUBLE, generic_body_name, &call_args); + wf.block_mut(generic_idx) + .unwrap() + .ret(DOUBLE, &generic_value); +} + +pub(super) fn guarded_falsy_field_default_name(base_name: &str, param_index: usize) -> String { + format!("{base_name}$default_false{param_index}") +} + pub(super) fn guarded_undefined_name(base_name: &str, param_index: usize) -> String { format!("{base_name}$undef{param_index}") } diff --git a/crates/perry-codegen/src/codegen/mod.rs b/crates/perry-codegen/src/codegen/mod.rs index d800568c1e..64b19c29c2 100644 --- a/crates/perry-codegen/src/codegen/mod.rs +++ b/crates/perry-codegen/src/codegen/mod.rs @@ -194,6 +194,8 @@ pub mod entry_outline; pub(crate) mod func_registry; mod function; #[cfg(test)] +mod guarded_falsy_default_method_tests; +#[cfg(test)] mod guarded_undefined_method_tests; #[cfg(test)] mod hoisted_callback_method_tests; @@ -1790,7 +1792,7 @@ pub fn compile_module(hir: &HirModule, opts: CompileOptions) -> Result> .classes .iter() .flat_map(|class| { - class.methods.iter().filter_map(move |method| { + class.methods.iter().filter_map(|method| { let params = typed_abi::nonnegative_index_method_params(method); (!params.is_empty()).then(|| ((class.name.clone(), method.name.clone()), params)) }) @@ -1816,6 +1818,20 @@ pub fn compile_module(hir: &HirModule, opts: CompileOptions) -> Result> }) }) .collect(); + let mut guarded_falsy_field_default_candidates: Vec<_> = hir + .classes + .iter() + .flat_map(|class| { + class.methods.iter().filter_map(|method| { + let key = (class.name.clone(), method.name.clone()); + if !nonnegative_index_methods.contains_key(&key) { + return None; + } + param_guard::guarded_falsy_field_default_method_candidate(class, method) + .map(|candidate| (candidate.body_nodes, key, candidate)) + }) + }) + .collect(); progress.checkpoint("cross-module and typed-ABI analysis"); // Module-wide dispatch/barrier facts. Hoisted above the typed-clone @@ -1865,7 +1881,13 @@ pub fn compile_module(hir: &HirModule, opts: CompileOptions) -> Result> receiver_class_table, &module_dispatch_facts, ) { - pshape_methods.insert((class.name.clone(), method.name.clone()), fact); + let key = (class.name.clone(), method.name.clone()); + // Indexed methods compose this receiver proof with their + // exact nonnegative-i32 argument proof. Their published + // The published proven-receiver entry guards the live + // argument once, then selects its combined integer clone or + // its receiver-safe generic body. + pshape_methods.insert(key, fact); } match typed_abi::typed_f64_method_rejection_reason(method) { None => { @@ -2052,6 +2074,14 @@ pub fn compile_module(hir: &HirModule, opts: CompileOptions) -> Result> .take(16) .map(|(_, key, param_index)| (key, param_index)) .collect(); + guarded_falsy_field_default_candidates + .sort_unstable_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1))); + let guarded_falsy_field_default_methods: std::collections::HashMap<_, _> = + guarded_falsy_field_default_candidates + .into_iter() + .take(16) + .map(|(_, key, candidate)| (key, candidate)) + .collect(); // #8774: one non-combinatorial tagged-ABI clone per local method. Source // annotations or a unique unannotated field signature only nominate a // class; every routed call emits an exact runtime class+shape guard. Keep @@ -2076,6 +2106,7 @@ pub fn compile_module(hir: &HirModule, opts: CompileOptions) -> Result> || typed_f64_receiver_methods.contains_key(&key) || nonnegative_index_methods.contains_key(&key) || guarded_undefined_method_params.contains_key(&key) + || guarded_falsy_field_default_methods.contains_key(&key) { continue; } @@ -2446,6 +2477,7 @@ pub fn compile_module(hir: &HirModule, opts: CompileOptions) -> Result> typed_f64_receiver_methods, nonnegative_index_methods, guarded_undefined_method_params, + guarded_falsy_field_default_methods, pshape_methods, pshape_arg_methods, pshape_tower_routable, diff --git a/crates/perry-codegen/src/codegen/opts.rs b/crates/perry-codegen/src/codegen/opts.rs index 194ea2c3c9..97d2803a04 100644 --- a/crates/perry-codegen/src/codegen/opts.rs +++ b/crates/perry-codegen/src/codegen/opts.rs @@ -1019,6 +1019,14 @@ pub(crate) struct CrossModuleCtx { /// JSValue ABI and performs the bit-exact guard before entering the clone; /// every other runtime value falls through to the ordinary body. pub guarded_undefined_method_params: std::collections::HashMap<(String, String), usize>, + /// Indexed methods with one private omitted-argument/false-field clone. + /// The public index wrapper proves exact `undefined`, receiver class and + /// ShapeId, ordinary packed layout, and the live field's canonical-false + /// bits before skipping the default prologue and its falsy branch. + pub guarded_falsy_field_default_methods: std::collections::HashMap< + (String, String), + super::param_guard::GuardedFalsyFieldDefaultMethodCandidate, + >, /// Representation-selection Phase 5a: `(class, method)` pairs that have a /// generated proven-`this` clone (`collectors/proven_this.rs`). Local keys /// come from body analysis; imported keys come from an explicit capability diff --git a/crates/perry-codegen/src/codegen/ordinary_method_artifacts.rs b/crates/perry-codegen/src/codegen/ordinary_method_artifacts.rs index 1837a38aab..6c82a1a7a5 100644 --- a/crates/perry-codegen/src/codegen/ordinary_method_artifacts.rs +++ b/crates/perry-codegen/src/codegen/ordinary_method_artifacts.rs @@ -59,6 +59,11 @@ pub(super) fn compile_ordinary_method_artifacts( cross_module, } = c; + let guarded_index_public = typed_public_trampoline.is_none() + && cross_module + .nonnegative_index_methods + .contains_key(&(class.name.clone(), method.name.clone())); + compile_method( llmod, class, @@ -81,7 +86,8 @@ pub(super) fn compile_ordinary_method_artifacts( typed_public_trampoline, cross_module .typed_f64_receiver_methods - .contains_key(&(class.name.clone(), method.name.clone())), + .contains_key(&(class.name.clone(), method.name.clone())) + || guarded_index_public, None, None, false, @@ -139,6 +145,9 @@ pub(super) fn compile_ordinary_method_artifacts( .pshape_methods .get(&(class.name.clone(), method.name.clone())) { + let guarded_index_pshape = cross_module + .nonnegative_index_methods + .contains_key(&(class.name.clone(), method.name.clone())); compile_method( llmod, class, @@ -159,7 +168,7 @@ pub(super) fn compile_ordinary_method_artifacts( closure_rest_params, cross_module, None, - false, + guarded_index_pshape, Some(fact.clone()), None, false, diff --git a/crates/perry-codegen/src/codegen/param_guard.rs b/crates/perry-codegen/src/codegen/param_guard.rs index 405a4ba10e..c6d97a2c3c 100644 --- a/crates/perry-codegen/src/codegen/param_guard.rs +++ b/crates/perry-codegen/src/codegen/param_guard.rs @@ -802,6 +802,14 @@ pub(crate) struct GuardedUndefinedMethodCandidate { pub body_nodes: usize, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct GuardedFalsyFieldDefaultMethodCandidate { + pub param_index: usize, + pub prologue_stmt_index: usize, + pub field_index: usize, + pub body_nodes: usize, +} + const MAX_GUARDED_UNDEFINED_METHOD_NODES: usize = 1_024; /// Select one optional parameter whose exact-`undefined` value can profitably @@ -972,6 +980,206 @@ pub(crate) fn guarded_undefined_method_candidate( }) } +/// Select an indexed method whose omitted final-style parameter defaults from +/// one declared receiver field and is subsequently used only as a direct +/// condition. +/// +/// The candidate is not itself a value proof. The public indexed-method +/// wrapper must still prove all three mutable runtime facts before entering a +/// private clone: the actual argument is exactly `undefined`, the receiver has +/// this class's exact current ShapeId with ordinary packed fields, and the live +/// default slot is exactly canonical `false`. Every miss executes the original +/// body, including its ordinary property read and JavaScript truthiness. +pub(crate) fn guarded_falsy_field_default_method_candidate( + class: &perry_hir::Class, + method: &perry_hir::Function, +) -> Option { + use perry_hir::{CompareOp, Expr, Stmt}; + + if method.is_async + || method.is_generator + || class.extends.is_some() + || class.extends_name.is_some() + || class.native_extends.is_some() + || class.extends_expr.is_some() + { + return None; + } + let body_nodes = super::closure_collect::count_body_nodes(&method.body); + if body_nodes > MAX_GUARDED_UNDEFINED_METHOD_NODES { + return None; + } + let closure_refs = crate::expr::collect_closure_referenced_locals(&method.body); + + fn field_default(expr: &Expr) -> Option<&str> { + let Expr::PropertyGet { + object, property, .. + } = expr + else { + return None; + }; + matches!(object.as_ref(), Expr::This).then_some(property.as_str()) + } + + fn is_matching_prologue(stmt: &Stmt, id: u32, field: &str) -> bool { + let Stmt::If { + condition: + Expr::Compare { + op: CompareOp::Eq, + left, + right, + }, + then_branch, + else_branch: None, + } = stmt + else { + return false; + }; + let compares_undefined = matches!( + (left.as_ref(), right.as_ref()), + (Expr::LocalGet(local), Expr::Undefined) + | (Expr::Undefined, Expr::LocalGet(local)) if *local == id + ); + compares_undefined + && matches!( + then_branch.as_slice(), + [Stmt::Expr(Expr::LocalSet(local, value))] + if *local == id && field_default(value) == Some(field) + ) + } + + fn scan_only_direct_conditions(stmts: &[Stmt], id: u32, guards: &mut usize) -> bool { + use crate::collectors::expr_contains_local_get; + stmts.iter().all(|stmt| match stmt { + Stmt::If { + condition, + then_branch, + else_branch, + } => { + if matches!(condition, Expr::LocalGet(local) if *local == id) { + *guards += 1; + } else if expr_contains_local_get(condition, id) { + return false; + } + scan_only_direct_conditions(then_branch, id, guards) + && else_branch + .as_deref() + .is_none_or(|body| scan_only_direct_conditions(body, id, guards)) + } + Stmt::While { condition, body } | Stmt::DoWhile { condition, body } => { + !expr_contains_local_get(condition, id) + && scan_only_direct_conditions(body, id, guards) + } + Stmt::For { + init, + condition, + update, + body, + } => { + init.as_deref().is_none_or(|stmt| { + scan_only_direct_conditions(std::slice::from_ref(stmt), id, guards) + }) && condition + .as_ref() + .is_none_or(|expr| !expr_contains_local_get(expr, id)) + && update + .as_ref() + .is_none_or(|expr| !expr_contains_local_get(expr, id)) + && scan_only_direct_conditions(body, id, guards) + } + Stmt::Try { + body, + catch, + finally, + } => { + scan_only_direct_conditions(body, id, guards) + && catch + .as_ref() + .is_none_or(|catch| scan_only_direct_conditions(&catch.body, id, guards)) + && finally + .as_deref() + .is_none_or(|body| scan_only_direct_conditions(body, id, guards)) + } + Stmt::Switch { + discriminant, + cases, + } => { + !expr_contains_local_get(discriminant, id) + && cases.iter().all(|case| { + case.test + .as_ref() + .is_none_or(|expr| !expr_contains_local_get(expr, id)) + && scan_only_direct_conditions(&case.body, id, guards) + }) + } + Stmt::Labeled { body, .. } => { + scan_only_direct_conditions(std::slice::from_ref(body.as_ref()), id, guards) + } + Stmt::Expr(expr) | Stmt::Throw(expr) | Stmt::Return(Some(expr)) => { + !expr_contains_local_get(expr, id) + } + Stmt::Let { + init: Some(expr), .. + } => !expr_contains_local_get(expr, id), + Stmt::Return(None) + | Stmt::Let { init: None, .. } + | Stmt::Break + | Stmt::Continue + | Stmt::LabeledBreak(_) + | Stmt::LabeledContinue(_) + | Stmt::PreallocateBoxes(_) + | Stmt::PreallocateTdzBoxes(_) + | Stmt::ReleaseBoxes(_) => true, + }) + } + + method + .params + .iter() + .enumerate() + .find_map(|(param_index, param)| { + if param.is_rest || param.arguments_object.is_some() || closure_refs.contains(¶m.id) + { + return None; + } + let field_name = param.default.as_ref().and_then(field_default)?; + let (field_index, _) = class + .fields + .iter() + .filter(|field| field.key_expr.is_none()) + .enumerate() + .find(|(_, field)| { + !field.is_private && field.decorators.is_empty() && field.name == field_name + })?; + let prologue_stmt_index = method + .body + .iter() + .position(|stmt| is_matching_prologue(stmt, param.id, field_name))?; + let has_real_reassignment = method.body.iter().enumerate().any(|(index, stmt)| { + index != prologue_stmt_index + && crate::collectors::reassigned_locals(std::slice::from_ref(stmt)) + .contains(¶m.id) + }); + if has_real_reassignment { + return None; + } + let mut guards = 0; + let uses_are_safe = method.body.iter().enumerate().all(|(index, stmt)| { + index == prologue_stmt_index + || scan_only_direct_conditions( + std::slice::from_ref(stmt), + param.id, + &mut guards, + ) + }); + (uses_are_safe && guards > 0).then_some(GuardedFalsyFieldDefaultMethodCandidate { + param_index, + prologue_stmt_index, + field_index, + body_nodes, + }) + }) +} + /// Whether the current function body can suspend after its entry guard. /// `walk_expr_children` intentionally does not enter nested closure bodies; /// those execute under their own entry contracts and must not disqualify the diff --git a/crates/perry-codegen/src/codegen/typed_abi.rs b/crates/perry-codegen/src/codegen/typed_abi.rs index 8598b74646..41850e74d8 100644 --- a/crates/perry-codegen/src/codegen/typed_abi.rs +++ b/crates/perry-codegen/src/codegen/typed_abi.rs @@ -75,6 +75,72 @@ impl TypedParamRep { } } +/// Inline the exact contract of runtime `js_typed_i32_arg_guard` and return +/// both its predicate and the decoded signed lane. +/// +/// Typed/nonnegative method dispatch is itself hot enough that two leaf calls +/// (guard, then conversion) dominate small bodies such as ECS `SparseSet.has`. +/// Keep the proof in generated IR so LLVM can reuse `raw` for the subsequent +/// nonnegative test and specialized call. The safe-value select is +/// load-bearing: `fptosi` is poison for tagged NaNs, infinities, and +/// out-of-range doubles even when a later select would choose the tagged arm. +pub(crate) fn emit_typed_i32_guard_and_raw( + blk: &mut crate::block::LlBlock, + value: &str, +) -> (String, String) { + use crate::types::{DOUBLE, I1, I32, I64}; + + let bits = blk.bitcast_double_to_i64(value); + let int32_identity_mask = crate::nanbox::i64_literal(!crate::nanbox::INT32_MASK); + let tagged_identity = blk.and(I64, &bits, &int32_identity_mask); + let is_tagged = blk.icmp_eq(I64, &tagged_identity, crate::nanbox::INT32_TAG_I64); + + // JSValue::is_number: Perry-owned tags occupy positive-qNaN top words + // 0x7ff9..=0x7fff. Everything outside that interval (including negative + // IEEE values and canonical 0x7ff8 NaNs) remains a Number. + let top16 = blk.lshr(I64, &bits, "48"); + let below_tag_band = blk.icmp_ult(I64, &top16, crate::nanbox::SHORT_STRING_TAG_TOP16_I64); + let above_tag_band = blk.icmp_ugt(I64, &top16, crate::nanbox::STRING_TAG_TOP16_I64); + let is_plain_number = blk.or(I1, &below_tag_band, &above_tag_band); + let above_min = blk.fcmp("oge", value, "-2147483648.0"); + let below_max = blk.fcmp("ole", value, "2147483647.0"); + let in_range = blk.and(I1, &above_min, &below_max); + let plain_candidate = blk.and(I1, &is_plain_number, &in_range); + + // Never feed a rejected/tagged bit pattern to fptosi. + let safe_plain = blk.select(I1, &plain_candidate, DOUBLE, value, "0.0"); + let plain_raw = blk.fptosi(DOUBLE, &safe_plain, I32); + let roundtrip = blk.sitofp(I32, &plain_raw, DOUBLE); + let integral = blk.fcmp("oeq", &roundtrip, value); + let not_negative_zero = blk.icmp_ne(I64, &bits, "-9223372036854775808"); + let plain_ok = blk.and(I1, &plain_candidate, &integral); + let plain_ok = blk.and(I1, &plain_ok, ¬_negative_zero); + let admitted = blk.or(I1, &is_tagged, &plain_ok); + + let tagged_raw = blk.trunc(I64, &bits, I32); + let raw = blk.select(I1, &is_tagged, I32, &tagged_raw, &plain_raw); + (admitted, raw) +} + +/// Decode a value after an enclosing entry guard has established the contract +/// above. The select again shields `fptosi` from canonical INT32 NaN-boxes; +/// no range/integrality work is repeated in the specialized body. +pub(crate) fn emit_typed_i32_raw_assuming_guarded( + blk: &mut crate::block::LlBlock, + value: &str, +) -> String { + use crate::types::{DOUBLE, I1, I32, I64}; + + let bits = blk.bitcast_double_to_i64(value); + let int32_identity_mask = crate::nanbox::i64_literal(!crate::nanbox::INT32_MASK); + let tagged_identity = blk.and(I64, &bits, &int32_identity_mask); + let is_tagged = blk.icmp_eq(I64, &tagged_identity, crate::nanbox::INT32_TAG_I64); + let safe_plain = blk.select(I1, &is_tagged, DOUBLE, "0.0", value); + let plain_raw = blk.fptosi(DOUBLE, &safe_plain, I32); + let tagged_raw = blk.trunc(I64, &bits, I32); + blk.select(I1, &is_tagged, I32, &tagged_raw, &plain_raw) +} + pub(crate) fn typed_param_rep_for_type(ty: &Type) -> Option { if matches!(ty, Type::Int32) { Some(TypedParamRep::I32) @@ -459,11 +525,25 @@ pub(crate) fn nonnegative_index_fast_array_method_name( format!("{generic_name}$idx_fast_array_u31_{suffix}") } -/// Select a deliberately small method family for call-site-proven index -/// specialization. Source `number` annotations nominate candidates but never -/// license the clone: routing requires a separate nonnegative-i32 proof at the -/// concrete call site, and every other caller keeps the public boxed body. +/// Select a deliberately small method family for guarded index +/// specialization. Source `number`/`Int32` annotations nominate transitive +/// numeric flows; erased `Any`/`Unknown` JavaScript parameters must occur in an +/// index expression directly, so an object whose field produces an index is +/// not mistaken for the index itself. These facts only nominate candidates: +/// direct routing requires a separate nonnegative-i32 proof, while the stable +/// public entry validates erased arguments at runtime and sends every miss to +/// the unchanged boxed body. pub(crate) fn nonnegative_index_method_params(method: &Function) -> Vec { + let direct_index_used = crate::collectors::collect_direct_index_used_locals(&method.body); + let index_used = crate::collectors::collect_index_used_locals(&method.body); + nonnegative_index_method_params_from_uses(method, &direct_index_used, &index_used) +} + +fn nonnegative_index_method_params_from_uses( + method: &Function, + direct_index_used: &HashSet, + index_used: &HashSet, +) -> Vec { if method.is_async || method.is_generator || method.was_plain_async @@ -471,19 +551,26 @@ pub(crate) fn nonnegative_index_method_params(method: &Function) -> Vec { || method .params .iter() - .any(|p| p.default.is_some() || p.is_rest || p.arguments_object.is_some()) + .any(|p| p.is_rest || p.arguments_object.is_some()) { return Vec::new(); } - let index_used = crate::collectors::collect_index_used_locals(&method.body); let reassigned = crate::collectors::reassigned_locals(&method.body); let closure_referenced = crate::expr::collect_closure_referenced_locals(&method.body); method .params .iter() .filter(|param| { - matches!(param.ty, Type::Number | Type::Int32) + let eligible_type = matches!( + param.ty, + Type::Number | Type::Int32 | Type::Any | Type::Unknown + ); + let credible_numeric_flow = direct_index_used.contains(¶m.id) + || matches!(param.ty, Type::Number | Type::Int32); + eligible_type + && credible_numeric_flow + && param.default.is_none() && index_used.contains(¶m.id) && !reassigned.contains(¶m.id) && !closure_referenced.contains(¶m.id) diff --git a/crates/perry-codegen/src/collectors/index_uses.rs b/crates/perry-codegen/src/collectors/index_uses.rs index b354ec40a3..550ce87edd 100644 --- a/crates/perry-codegen/src/collectors/index_uses.rs +++ b/crates/perry-codegen/src/collectors/index_uses.rs @@ -3,8 +3,7 @@ use std::collections::HashSet; use super::*; pub fn collect_index_used_locals(stmts: &[perry_hir::Stmt]) -> HashSet { - let mut out: HashSet = HashSet::new(); - walk_index_uses_in_stmts(stmts, &mut out); + let mut out = collect_direct_index_used_locals(stmts); // Issue #435: take the transitive closure backward through writes so // that locals which feed an array index via arithmetic are also // marked. Image-convolution's `xx → idx → array[idx]` shape relies @@ -21,6 +20,20 @@ pub fn collect_index_used_locals(stmts: &[perry_hir::Stmt]) -> HashSet { out } +/// Locals mentioned by an index expression itself, before walking backward +/// through assignments that produced that value. +/// +/// Most representation selectors want the transitive set above. Method-entry +/// guards also need this seed set to distinguish an erased numeric parameter +/// used *as* an index from an arbitrary object whose property merely produces +/// one (`let i = component.meta.id; rows[i]`). Guarding the latter object as if +/// it were a nonnegative integer makes the generated fast path unreachable. +pub fn collect_direct_index_used_locals(stmts: &[perry_hir::Stmt]) -> HashSet { + let mut out = HashSet::new(); + walk_index_uses_in_stmts(stmts, &mut out); + out +} + /// Iterate the `Stmt::Let` / `Expr::LocalSet` write graph to a fixed /// point: when a write target is in `out`, pull every `LocalGet` / /// `LocalSet` / `Update` id from the rhs into `out` as well. The result diff --git a/crates/perry-codegen/src/collectors/mod.rs b/crates/perry-codegen/src/collectors/mod.rs index 4824bb1127..6d38e022ff 100644 --- a/crates/perry-codegen/src/collectors/mod.rs +++ b/crates/perry-codegen/src/collectors/mod.rs @@ -78,7 +78,7 @@ pub(crate) use i32_locals::{ collect_integer_let_ids, collect_localset_ids_in_stmts, is_strictly_i32_bounded_expr, is_ushr_zero, }; -pub(crate) use index_uses::collect_index_used_locals; +pub(crate) use index_uses::{collect_direct_index_used_locals, collect_index_used_locals}; pub(crate) use int_valued_i64_locals::ceil_log2_abs; pub(crate) use integer_locals::{ collect_flat_row_aliases, is_int32_producing_expr, static_index_window, diff --git a/crates/perry-codegen/src/collectors/proven_this.rs b/crates/perry-codegen/src/collectors/proven_this.rs index 58a621b5c6..9c84a1be15 100644 --- a/crates/perry-codegen/src/collectors/proven_this.rs +++ b/crates/perry-codegen/src/collectors/proven_this.rs @@ -544,7 +544,7 @@ pub(crate) fn method_proven_this( return None; } for param in &method.params { - if param.default.is_some() || param.is_rest || param.arguments_object.is_some() { + if param.is_rest || param.arguments_object.is_some() { return None; } } @@ -564,13 +564,34 @@ pub(crate) fn method_proven_this( let fields = chain_field_names(&chain); let methods = chain_method_map(&chain); + // A default that only reads a declared receiver field cannot invalidate + // the exact receiver shape. The parser also lowers that read into the + // explicit `arg === undefined` prologue walked below. Keep every other + // default conservative-off: an arbitrary initializer could run user code + // and mutate an aliased receiver after the call-site shape guard. + if method.params.iter().any(|param| { + param.default.as_ref().is_some_and(|default| { + !matches!( + default, + Expr::PropertyGet { + object, + property, + .. + } if matches!(object.as_ref(), Expr::This) + && fields.contains(property.as_str()) + ) + }) + }) { + return None; + } + // `this`-flow safety: `this` never used as a value (`Expr::This` in value // position rejects), no closure mentioning `this`, every `this.f = v` // write to a DECLARED chain field, every internally-invoked `this.m()` / // `super.m()` vetted transitively. This is the same walk Phase 3b runs // over the methods called on a proven local. let mut analysis = ThisFlowAnalysis::new(&chain, &fields, &methods); - if !analysis.method_safe(&class.name, method) { + if !analysis.method_safe_with_terminal_this_return(&class.name, method) { return None; } diff --git a/crates/perry-codegen/src/collectors/proven_this_routing_tests.rs b/crates/perry-codegen/src/collectors/proven_this_routing_tests.rs index 7ef6ddb43e..6e7daca56c 100644 --- a/crates/perry-codegen/src/collectors/proven_this_routing_tests.rs +++ b/crates/perry-codegen/src/collectors/proven_this_routing_tests.rs @@ -559,6 +559,306 @@ fn guarded_site_module() -> Module { m } +fn guarded_boolean_site_module(boolean_body: bool) -> Module { + let predicate = func( + 130, + "accepts", + vec![param(131, "value", Type::Any)], + // Deliberately Boolean in both variants: the negative proves codegen + // does not trust an erased source annotation. + Type::Boolean, + vec![Stmt::Return(Some(if boolean_body { + Expr::Compare { + op: perry_hir::CompareOp::Gt, + left: Box::new(Expr::LocalGet(131)), + right: Box::new(this_get("limit")), + } + } else { + Expr::Integer(1) + }))], + ); + let mut m = Module::new(if boolean_body { + "guarded_boolean_result.ts" + } else { + "guarded_lying_boolean_result.ts" + }); + m.classes = vec![class( + 104, + "Predicate", + vec![field("limit", Type::Number)], + vec![predicate], + )]; + m.functions = vec![func( + 132, + "probeBoolean", + vec![ + param(133, "predicate", Type::Named("Predicate".to_string())), + param(134, "value", Type::Any), + ], + Type::Number, + vec![ + Stmt::If { + condition: call(Expr::LocalGet(133), "accepts", vec![Expr::LocalGet(134)]), + then_branch: vec![Stmt::Return(Some(Expr::Integer(1)))], + else_branch: None, + }, + Stmt::Return(Some(Expr::Integer(0))), + ], + )]; + m.init_kind = ModuleInitKind::Eager; + m +} + +fn bitset_truthiness_site_module(proven_index: bool) -> Module { + const MASK_ID: u32 = 141; + const INDEX_ID: u32 = 142; + const RECEIVER_ID: u32 = 144; + const CALLER_MASK_ID: u32 = 145; + const CALLER_INDEX_ID: u32 = 146; + + let bitset_test = Expr::Binary { + op: BinaryOp::BitAnd, + left: Box::new(Expr::IndexGet { + object: Box::new(Expr::LocalGet(MASK_ID)), + index: Box::new(Expr::Unary { + op: perry_hir::UnaryOp::BitNot, + operand: Box::new(Expr::Unary { + op: perry_hir::UnaryOp::BitNot, + operand: Box::new(Expr::Binary { + op: BinaryOp::Div, + left: Box::new(Expr::LocalGet(INDEX_ID)), + right: Box::new(Expr::Integer(32)), + }), + }), + }), + }), + right: Box::new(Expr::Binary { + op: BinaryOp::Shl, + left: Box::new(Expr::Integer(1)), + right: Box::new(Expr::Binary { + op: BinaryOp::Mod, + left: Box::new(Expr::LocalGet(INDEX_ID)), + right: Box::new(Expr::Integer(32)), + }), + }), + }; + let has = func( + 140, + "has", + vec![ + param(MASK_ID, "mask", Type::Any), + param(INDEX_ID, "index", Type::Any), + ], + Type::Any, + vec![Stmt::Return(Some(bitset_test))], + ); + let call_index = if proven_index { + Expr::Integer(7) + } else { + Expr::LocalGet(CALLER_INDEX_ID) + }; + let mut caller_params = vec![ + param(RECEIVER_ID, "set", Type::Named("Bitset".to_string())), + param(CALLER_MASK_ID, "mask", Type::Any), + ]; + if !proven_index { + caller_params.push(param(CALLER_INDEX_ID, "index", Type::Any)); + } + let caller = func( + 143, + "probeBitset", + caller_params, + Type::Number, + vec![ + Stmt::If { + condition: call( + Expr::LocalGet(RECEIVER_ID), + "has", + vec![Expr::LocalGet(CALLER_MASK_ID), call_index], + ), + then_branch: vec![Stmt::Return(Some(Expr::Integer(1)))], + else_branch: None, + }, + Stmt::Return(Some(Expr::Integer(0))), + ], + ); + + let mut m = Module::new(if proven_index { + "guarded_bitset_truthiness.ts" + } else { + "guarded_unproven_bitset_truthiness.ts" + }); + m.classes = vec![class(139, "Bitset", Vec::new(), vec![has])]; + m.functions = vec![caller]; + m.init_kind = ModuleInitKind::Eager; + m +} + +/// A condition may consume a constructively-Boolean direct method result as +/// `i1`, but only inside the guarded static arm. The dynamic override arm must +/// retain total JavaScript truthiness because an own/prototype replacement can +/// return any value at runtime. +#[test] +fn guarded_boolean_method_truthiness_is_native_only_on_the_proven_arm() { + let ir = emit(&guarded_boolean_site_module(true), false); + let probe = function_body(&ir, "__probeBoolean("); + let bs = blocks(&probe); + let fast = bs + .iter() + .find(|(_, body)| { + body.iter().any(|line| { + line.contains("call double @") + && line.contains("__accepts") + && !line.contains("js_native_call_method") + }) + }) + .unwrap_or_else(|| panic!("no guarded direct predicate arm in:\n{probe}")); + assert!( + fast.1.iter().any(|line| line.contains("icmp eq i64")), + "the proven Boolean arm boxed its return and called the total predicate:\n{probe}" + ); + assert!( + !fast.1.iter().any(|line| line.contains("@js_is_truthy(")), + "the proven Boolean arm still calls js_is_truthy:\n{probe}" + ); + let fallback = bs + .iter() + .find(|(_, body)| { + body.iter() + .any(|line| line.contains("@js_native_call_method_by_id(")) + }) + .unwrap_or_else(|| panic!("no dynamic override fallback in:\n{probe}")); + assert!( + fallback + .1 + .iter() + .any(|line| line.contains("@js_is_truthy(")), + "the arbitrary override result was not tested with full JS truthiness:\n{probe}" + ); + assert!( + bs.iter() + .any(|(_, body)| body.iter().any(|line| line.contains(" = phi i1 "))), + "the guarded Boolean and dynamic truthiness arms do not merge natively:\n{probe}" + ); + let merge = bs + .iter() + .find(|(label, _)| label.starts_with("method_direct.merge")) + .unwrap_or_else(|| panic!("no guarded method merge in:\n{probe}")); + assert!( + merge + .1 + .iter() + .any(|line| line.contains(" = phi double ")), + "truthiness publication replaced the override's actual JS value instead of merging it in parallel:\n{probe}" + ); +} + +/// The canonical ECS bitset method can publish Number truthiness when its body +/// was resolved. The dynamic override remains unconstrained. +#[test] +fn guarded_bitset_method_truthiness_uses_raw_number_only_on_the_proven_arm() { + let ir = emit(&bitset_truthiness_site_module(true), false); + let probe = function_body(&ir, "__probeBitset("); + let bs = blocks(&probe); + let fast = bs + .iter() + .find(|(_, body)| { + body.iter().any(|line| { + line.contains("call double @") + && line.contains("__has") + && line.contains("$idx_u31_") + }) + }) + .unwrap_or_else(|| panic!("no guarded indexed bitset arm in:\n{probe}")); + assert!( + fast.1.iter().any(|line| line.contains("fcmp one double")), + "the exact Number result still used total JS truthiness:\n{probe}" + ); + assert!( + !fast.1.iter().any(|line| line.contains("@js_is_truthy(")), + "the proven Number arm still calls js_is_truthy:\n{probe}" + ); + let fallback = bs + .iter() + .find(|(_, body)| { + body.iter() + .any(|line| line.contains("@js_native_call_method_by_id(")) + }) + .unwrap_or_else(|| panic!("no dynamic bitset override fallback in:\n{probe}")); + assert!( + fallback + .1 + .iter() + .any(|line| line.contains("@js_is_truthy(")), + "the arbitrary bitset override skipped full JavaScript truthiness:\n{probe}" + ); + let merge = bs + .iter() + .find(|(label, _)| label.starts_with("method_direct.merge")) + .unwrap_or_else(|| panic!("no guarded bitset merge in:\n{probe}")); + assert!( + merge.1.iter().any(|line| line.contains(" = phi double ")) + && merge.1.iter().any(|line| line.contains(" = phi i1 ")), + "the bitset value and truthiness were not merged independently:\n{probe}" + ); +} + +/// The result-kind proof does not require the native-index lowering proof: for +/// an arbitrary index the canonical source expression still either returns a +/// Number or throws. +#[test] +fn unproven_bitset_index_still_has_raw_number_truthiness() { + let ir = emit(&bitset_truthiness_site_module(false), false); + let probe = function_body(&ir, "__probeBitset("); + assert!( + probe.contains("@js_is_truthy("), + "the arbitrary dynamic override skipped total JavaScript truthiness:\n{probe}" + ); + assert!( + probe.contains("fcmp one double"), + "the canonical bitset result lost its input-independent Number proof:\n{probe}" + ); +} + +/// A generic bitwise method is not enough. Keep using total truthiness unless +/// the full Number-or-throw bitset tree matched structurally. +#[test] +fn noncanonical_bitwise_method_does_not_gain_raw_number_truthiness() { + let mut module = bitset_truthiness_site_module(true); + module.classes[0].methods[0].body = vec![Stmt::Return(Some(Expr::Binary { + op: BinaryOp::BitAnd, + left: Box::new(Expr::LocalGet(141)), + right: Box::new(Expr::LocalGet(142)), + }))]; + let ir = emit(&module, false); + let probe = function_body(&ir, "__probeBitset("); + assert!( + probe.contains("@js_is_truthy("), + "a noncanonical bitwise return bypassed total JavaScript truthiness:\n{probe}" + ); + assert!( + !probe.contains("fcmp one double"), + "an arbitrary bitwise return was mistaken for the canonical bitset test:\n{probe}" + ); +} + +/// The constructive proof, not `: boolean`, licenses the direct tag test. +#[test] +fn erased_boolean_return_annotation_does_not_license_a_native_result() { + let ir = emit(&guarded_boolean_site_module(false), false); + let probe = function_body(&ir, "__probeBoolean("); + let dynamic_call = probe + .find("@js_native_call_method_by_id(") + .unwrap_or_else(|| panic!("no guarded method fallback in:\n{probe}")); + let truthy = probe + .rfind("@js_is_truthy(") + .unwrap_or_else(|| panic!("lying Boolean annotation bypassed js_is_truthy:\n{probe}")); + assert!( + dynamic_call < truthy, + "an annotation-only Boolean result was canonicalized before the guard merge:\n{probe}" + ); +} + /// Regression: a method with NO eligible typed clone routes to its clone. /// /// This half was already true before #7128 and is kept as the control — if it @@ -981,6 +1281,45 @@ fn tower_site_module() -> Module { m } +/// The same interface-dispatch shape with an indexed method. The literal +/// argument supplies the nonnegative-i32 proof while the receiver stays +/// runtime-typed, matching `this._ent[id].sset.add(id)` in wolf-ecs. +fn indexed_tower_site_module() -> Module { + const INDEX_ID: u32 = 83; + let mut row = row_class(); + row.fields.push(array_field("values")); + row.methods.push(func( + 99, + "lookup", + vec![param(INDEX_ID, "index", Type::Any)], + Type::Any, + vec![Stmt::Return(Some(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::IndexGet { + object: Box::new(this_get("values")), + index: Box::new(Expr::LocalGet(INDEX_ID)), + }), + right: Box::new(this_get("id")), + }))], + )); + + let mut module = Module::new("pshape_index_tower.ts"); + module.classes = vec![row]; + module.functions = vec![func( + 1, + "probeIndex", + vec![param(2, "row", Type::Named("Shaped".to_string()))], + Type::Any, + vec![Stmt::Return(Some(call( + Expr::LocalGet(2), + "lookup", + vec![Expr::Integer(0)], + )))], + )]; + module.init_kind = ModuleInitKind::Eager; + module +} + /// Split rendered IR into `(label, body_lines)` — block labels render /// unindented and colon-terminated, every instruction is indented. fn blocks(ir: &str) -> Vec<(String, Vec<&str>)> { @@ -1040,6 +1379,28 @@ fn tower_case_routes_to_proven_this_clone() { ); } +#[test] +fn tower_case_composes_receiver_shape_and_proven_index_clones() { + const INDEX_ID: u32 = 83; + let ir = emit(&indexed_tower_site_module(), false); + let caller = function_body(&ir, "__probeIndex("); + let public = "perry_method_pshape_index_tower_ts__Row__lookup"; + let indexed = format!("{public}$idx_u31_{INDEX_ID}"); + let shaped = format!("{public}$pshape"); + let combined = format!("{shaped}$idx_u31_{INDEX_ID}"); + + assert!( + caller.contains(&format!("call double @{combined}(")) + && caller.contains(&format!("call double @{indexed}(")), + "the shape hit must consume both proofs and the shape miss must retain the index proof:\n{caller}" + ); + assert!( + !caller.contains(&format!("call double @{shaped}(")) + && !caller.contains(&format!("call double @{public}(")), + "a proven index must not be re-guarded by either public tower target:\n{caller}" + ); +} + /// Soundness ratchet (#7142): the routed call is dominated by a compare of the /// receiver's authoritative ShapeId against `@perry_class_shape_id_*`. /// diff --git a/crates/perry-codegen/src/collectors/ptr_shape.rs b/crates/perry-codegen/src/collectors/ptr_shape.rs index 863526517b..bd575f421c 100644 --- a/crates/perry-codegen/src/collectors/ptr_shape.rs +++ b/crates/perry-codegen/src/collectors/ptr_shape.rs @@ -1521,7 +1521,7 @@ impl<'a, 'b> ThisFlowAnalysis<'a, 'b> { } for class in self.chain { if let Some(ctor) = &class.constructor { - if !self.function_this_safe(&class.name, "constructor", ctor) { + if !self.function_this_safe(&class.name, "constructor", ctor, false) { return false; } } @@ -1530,7 +1530,20 @@ impl<'a, 'b> ThisFlowAnalysis<'a, 'b> { } pub(super) fn method_safe(&mut self, owner: &str, func: &'a perry_hir::Function) -> bool { - self.function_this_safe(owner, &func.name, func) + self.function_this_safe(owner, &func.name, func, false) + } + + /// Phase 5a root-method variant: `return this` is safe only as the final + /// statement of the method whose receiver was already guarded. It creates + /// no alias until every specialized field access has completed. Nested + /// methods continue through [`Self::method_safe`] and may not return the + /// receiver, preserving Phase 3b's no-escape contract. + pub(super) fn method_safe_with_terminal_this_return( + &mut self, + owner: &str, + func: &'a perry_hir::Function, + ) -> bool { + self.function_this_safe(owner, &func.name, func, true) } fn function_this_safe( @@ -1538,6 +1551,7 @@ impl<'a, 'b> ThisFlowAnalysis<'a, 'b> { owner: &str, name: &str, func: &'a perry_hir::Function, + allow_terminal_this_return: bool, ) -> bool { let key = (owner.to_string(), name.to_string()); if !self.visited.insert(key) { @@ -1552,11 +1566,14 @@ impl<'a, 'b> ThisFlowAnalysis<'a, 'b> { let param_ids: Vec = func.params.iter().map(|p| p.id).collect(); let ctx = (owner.to_string(), name.to_string(), param_ids); let mut safe = true; - for s in &func.body { + for (index, s) in func.body.iter().enumerate() { if !safe { break; } - safe &= self.stmt_this_safe(s, &ctx); + let terminal_this_return = allow_terminal_this_return + && index + 1 == func.body.len() + && matches!(s, Stmt::Return(Some(Expr::This))); + safe &= terminal_this_return || self.stmt_this_safe(s, &ctx); } safe } @@ -1731,7 +1748,7 @@ impl<'a, 'b> ThisFlowAnalysis<'a, 'b> { return false; }; self.internally_invoked.insert(property.clone()); - if !self.function_this_safe(&owner, property, func) { + if !self.function_this_safe(&owner, property, func, false) { return false; } args.iter() @@ -1761,7 +1778,7 @@ impl<'a, 'b> ThisFlowAnalysis<'a, 'b> { return false; }; self.internally_invoked.insert(method.clone()); - if !self.function_this_safe(&owner, method, func) { + if !self.function_this_safe(&owner, method, func, false) { return false; } args.iter() diff --git a/crates/perry-codegen/src/collectors/scalar_method_dispatch.rs b/crates/perry-codegen/src/collectors/scalar_method_dispatch.rs index 7a902109c6..2f985ef075 100644 --- a/crates/perry-codegen/src/collectors/scalar_method_dispatch.rs +++ b/crates/perry-codegen/src/collectors/scalar_method_dispatch.rs @@ -24,9 +24,13 @@ //! Two fact sets restore the missing half: //! //! * [`ModuleDispatchFacts`] — module-scoped: which classes' prototypes are -//! named anywhere in the module. Naming is enough, because a named prototype -//! can be aliased and written through. A per-function walk cannot see this; -//! the mutation typically lives in a helper the constructor calls. +//! named anywhere in the module. Naming is normally enough, because a named +//! prototype can be aliased and written through. The one exception is a +//! fully-contained, immutable alias used only by the lowered +//! `Object.getOwnPropertyNames(proto)` / `typeof proto[key]` / +//! `proto[key].bind(...)` introspection pattern; that pattern never exposes +//! or mutates the prototype object. A per-function mutation walk cannot see +//! this; the mutation typically lives in a helper the constructor calls. //! * [`collect_candidate_property_writes`] — function-scoped: which property //! names are written directly on each scalar-replacement candidate. //! @@ -44,9 +48,11 @@ use super::cjs_scaffolding::CjsScaffolding; #[derive(Debug, Clone)] pub struct ModuleDispatchFacts { /// Classes whose prototype object is named — read or written — anywhere in - /// the module. Reading is enough: `let p = C.prototype; p.m = fn` mutates - /// through an alias, and `.prototype. = fn` lowers to - /// `Expr::RegisterPrototypeMethod` only when the recogniser matches. + /// the module. Reading is normally enough: `let p = C.prototype; p.m = fn` + /// mutates through an alias, and `.prototype. = fn` lowers to + /// `Expr::RegisterPrototypeMethod` only when the recogniser matches. The + /// contained read-only reflection exception is proved before this set is + /// populated; see [`read_only_prototype_reads`]. prototype_touched_classes: HashSet, /// A prototype was named through an expression that cannot be attributed to /// a declared class (`k.prototype`, `x.constructor.prototype`, …). Nothing @@ -325,6 +331,137 @@ impl ModuleDispatchFacts { } } +/// Exact `Class.prototype` expression nodes whose value is held by one +/// immutable local and used only for the read-only reflection pattern emitted +/// by libraries' `bind()` helpers: +/// +/// ```text +/// const proto = C.prototype; +/// Object.getOwnPropertyNames(proto); +/// typeof proto[key]; +/// proto[key].bind(receiver); +/// ``` +/// +/// Merely seeing the first line is not enough: every reference to the local is +/// counted with HIR's exhaustive local-reference walker, and every one must be +/// the exact `LocalGet` consumed by one of the three read-only forms above. +/// The class must also have a plain method-only prototype; otherwise the +/// indexed reads could invoke an accessor with arbitrary side effects. Any +/// write, return, call argument, alias, specialized local-id operation, or +/// unrecognised use keeps the historical conservative prototype kill. +fn read_only_prototype_reads(hir: &Module) -> HashSet { + let plain_prototype_classes: HashSet<&str> = hir + .classes + .iter() + .filter(|class| { + class.getters.is_empty() + && class.setters.is_empty() + && class.computed_members.is_empty() + }) + .map(|class| class.name.as_str()) + .collect(); + let mut reads = HashSet::new(); + + let mut inspect_scope = |stmts: &[Stmt]| { + for stmt in stmts { + let Stmt::Let { + id, + mutable: false, + init: Some(init), + .. + } = stmt + else { + continue; + }; + let Expr::PropertyGet { + object, property, .. + } = init + else { + continue; + }; + let Expr::ClassRef(class_name) = object.as_ref() else { + continue; + }; + if !is_prototype_key(property) || !plain_prototype_classes.contains(class_name.as_str()) + { + continue; + } + + let mut allowed_local_gets = HashSet::new(); + let mut saw_names = false; + let mut saw_typeof_index = false; + let mut saw_bound_index = false; + for_each_expr_in_stmts(stmts, &mut |expr| match expr { + Expr::ObjectGetOwnPropertyNames(value) if matches!(value.as_ref(), Expr::LocalGet(local) if local == id) => + { + allowed_local_gets.insert(value.as_ref() as *const Expr as usize); + saw_names = true; + } + Expr::TypeOf(value) => { + if let Expr::IndexGet { object, .. } = value.as_ref() { + if matches!(object.as_ref(), Expr::LocalGet(local) if local == id) { + allowed_local_gets.insert(object.as_ref() as *const Expr as usize); + saw_typeof_index = true; + } + } + } + Expr::Call { callee, .. } => { + if let Expr::PropertyGet { + object, property, .. + } = callee.as_ref() + { + if property == "bind" { + if let Expr::IndexGet { object, .. } = object.as_ref() { + if matches!(object.as_ref(), Expr::LocalGet(local) if local == id) { + allowed_local_gets + .insert(object.as_ref() as *const Expr as usize); + saw_bound_index = true; + } + } + } + } + } + _ => {} + }); + + let mut refs = Vec::new(); + let mut visited = HashSet::new(); + for stmt in stmts { + perry_hir::analysis::collect_local_refs_stmt(stmt, &mut refs, &mut visited); + } + let reference_count = refs.iter().filter(|local| *local == id).count(); + if saw_names + && saw_typeof_index + && saw_bound_index + && reference_count == allowed_local_gets.len() + { + reads.insert(init as *const Expr as usize); + } + } + }; + + inspect_scope(&hir.init); + for function in &hir.functions { + inspect_scope(&function.body); + } + for class in &hir.classes { + if let Some(ctor) = &class.constructor { + inspect_scope(&ctor.body); + } + for method in class + .methods + .iter() + .chain(class.static_methods.iter()) + .chain(class.getters.iter().map(|(_, f)| f)) + .chain(class.setters.iter().map(|(_, f)| f)) + .chain(class.computed_members.iter().map(|m| &m.function)) + { + inspect_scope(&method.body); + } + } + reads +} + /// Scan a whole module — top-level init, every function, and every class body /// (constructor, field initializers, methods, accessors, computed members) — /// for expressions that can rewrite a class's prototype. @@ -350,14 +487,15 @@ pub fn collect_module_dispatch_facts(hir: &Module) -> ModuleDispatchFacts { // bindings first — the barrier classifier below consults them to skip the // two `defineProperty` sites every `cjs_wrap`-compiled module contains. let cjs = super::cjs_scaffolding::collect(hir); + let read_only_prototype_reads = read_only_prototype_reads(hir); - note_stmts(&hir.init, &mut facts, &cjs); + note_stmts(&hir.init, &mut facts, &cjs, &read_only_prototype_reads); for function in &hir.functions { - note_stmts(&function.body, &mut facts, &cjs); + note_stmts(&function.body, &mut facts, &cjs, &read_only_prototype_reads); } for class in &hir.classes { if let Some(ctor) = &class.constructor { - note_stmts(&ctor.body, &mut facts, &cjs); + note_stmts(&ctor.body, &mut facts, &cjs, &read_only_prototype_reads); } for method in class .methods @@ -367,18 +505,23 @@ pub fn collect_module_dispatch_facts(hir: &Module) -> ModuleDispatchFacts { .chain(class.setters.iter().map(|(_, f)| f)) .chain(class.computed_members.iter().map(|m| &m.function)) { - note_stmts(&method.body, &mut facts, &cjs); + note_stmts(&method.body, &mut facts, &cjs, &read_only_prototype_reads); } for field in class.fields.iter().chain(class.static_fields.iter()) { if let Some(init) = &field.init { - note_expr_tree(init, &mut facts, &cjs); + note_expr_tree(init, &mut facts, &cjs, &read_only_prototype_reads); } if let Some(key) = &field.key_expr { - note_expr_tree(key, &mut facts, &cjs); + note_expr_tree(key, &mut facts, &cjs, &read_only_prototype_reads); } } for member in &class.computed_members { - note_expr_tree(&member.key_expr, &mut facts, &cjs); + note_expr_tree( + &member.key_expr, + &mut facts, + &cjs, + &read_only_prototype_reads, + ); } } @@ -397,17 +540,36 @@ pub fn collect_module_dispatch_facts(hir: &Module) -> ModuleDispatchFacts { facts } -fn note_stmts(stmts: &[Stmt], facts: &mut ModuleDispatchFacts, cjs: &CjsScaffolding) { - for_each_expr_in_stmts(stmts, &mut |expr| note_expr(expr, facts, cjs)); +fn note_stmts( + stmts: &[Stmt], + facts: &mut ModuleDispatchFacts, + cjs: &CjsScaffolding, + read_only_prototype_reads: &HashSet, +) { + for_each_expr_in_stmts(stmts, &mut |expr| { + note_expr(expr, facts, cjs, read_only_prototype_reads) + }); } -fn note_expr_tree(expr: &Expr, facts: &mut ModuleDispatchFacts, cjs: &CjsScaffolding) { - for_each_expr(expr, &mut |node| note_expr(node, facts, cjs)); +fn note_expr_tree( + expr: &Expr, + facts: &mut ModuleDispatchFacts, + cjs: &CjsScaffolding, + read_only_prototype_reads: &HashSet, +) { + for_each_expr(expr, &mut |node| { + note_expr(node, facts, cjs, read_only_prototype_reads) + }); } /// Classify one already-visited expression node. -fn note_expr(expr: &Expr, facts: &mut ModuleDispatchFacts, cjs: &CjsScaffolding) { - note_prototype_effect(expr, facts); +fn note_expr( + expr: &Expr, + facts: &mut ModuleDispatchFacts, + cjs: &CjsScaffolding, + read_only_prototype_reads: &HashSet, +) { + note_prototype_effect(expr, facts, read_only_prototype_reads); // #7139: the CommonJS wrap's own `defineProperty(require, 'name', …)` // preamble and the transpiled-CJS `defineProperty(exports, "__esModule", // …)` marker target module scaffolding that can never be a `Ptr` @@ -430,7 +592,11 @@ fn note_expr(expr: &Expr, facts: &mut ModuleDispatchFacts, cjs: &CjsScaffolding) /// /// Only the node itself is classified — [`for_each_expr`] supplies every node /// in the tree, including closure bodies. -fn note_prototype_effect(expr: &Expr, facts: &mut ModuleDispatchFacts) { +fn note_prototype_effect( + expr: &Expr, + facts: &mut ModuleDispatchFacts, + read_only_prototype_reads: &HashSet, +) { match expr { // `.prototype. = fn` (and its aliased `let p = C.prototype` // shape) — issue #838's recogniser resolves the class by name. @@ -447,7 +613,9 @@ fn note_prototype_effect(expr: &Expr, facts: &mut ModuleDispatchFacts) { // can be aliased into a local and written through later. Expr::PropertyGet { object, property, .. - } if is_prototype_key(property) => { + } if is_prototype_key(property) + && !read_only_prototype_reads.contains(&(expr as *const Expr as usize)) => + { note_prototype_holder(object, facts); } Expr::PropertySet { @@ -924,6 +1092,73 @@ mod tests { assert!(!facts.prototype_is_stable(&classes, "C")); } + fn read_only_bind_introspection_stmts() -> Vec { + const PROTO: u32 = 41; + vec![ + Stmt::Let { + id: PROTO, + name: "proto".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::PropertyGet { + byte_offset: 77, + object: Box::new(Expr::ClassRef("C".to_string())), + property: "prototype".to_string(), + }), + }, + Stmt::Expr(Expr::ObjectGetOwnPropertyNames(Box::new(Expr::LocalGet( + PROTO, + )))), + Stmt::Expr(Expr::TypeOf(Box::new(Expr::IndexGet { + object: Box::new(Expr::LocalGet(PROTO)), + index: Box::new(Expr::String("getValue".to_string())), + }))), + Stmt::Expr(Expr::Call { + callee: Box::new(Expr::PropertyGet { + byte_offset: 78, + object: Box::new(Expr::IndexGet { + object: Box::new(Expr::LocalGet(PROTO)), + index: Box::new(Expr::String("getValue".to_string())), + }), + property: "bind".to_string(), + }), + args: vec![Expr::Undefined], + type_args: Vec::new(), + byte_offset: 78, + }), + ] + } + + #[test] + fn contained_bind_introspection_does_not_mark_the_prototype_unstable() { + let class = summarizable_class("C"); + let mut module = Module::new("m.ts"); + module.init = read_only_bind_introspection_stmts(); + module.classes.push(class.clone()); + + let facts = collect_module_dispatch_facts(&module); + let classes = HashMap::from([(class.name.clone(), &class)]); + assert!(facts.prototype_is_stable(&classes, "C")); + } + + #[test] + fn a_write_through_the_introspection_alias_keeps_the_prototype_unstable() { + const PROTO: u32 = 41; + let class = summarizable_class("C"); + let mut module = Module::new("m.ts"); + module.init = read_only_bind_introspection_stmts(); + module.init.push(Stmt::Expr(Expr::PropertySet { + object: Box::new(Expr::LocalGet(PROTO)), + property: "getValue".to_string(), + value: Box::new(Expr::Number(9.0)), + })); + module.classes.push(class.clone()); + + let facts = collect_module_dispatch_facts(&module); + let classes = HashMap::from([(class.name.clone(), &class)]); + assert!(!facts.prototype_is_stable(&classes, "C")); + } + /// A prototype named through something other than a class ref can't be /// attributed, so nothing in the module may be summarized. #[test] diff --git a/crates/perry-codegen/src/expr/arrays_finds.rs b/crates/perry-codegen/src/expr/arrays_finds.rs index 1e3723419c..09f2c0b6c4 100644 --- a/crates/perry-codegen/src/expr/arrays_finds.rs +++ b/crates/perry-codegen/src/expr/arrays_finds.rs @@ -806,10 +806,8 @@ pub(crate) fn lower( return rooting::with_operands_rooted(ctx, &[array, index], |ctx, vals| { let a = vals[0].clone(); let key = vals[1].clone(); - Ok(ctx.block().call( - DOUBLE, - "js_object_get_symbol_property", - &[(DOUBLE, &a), (DOUBLE, &key)], + Ok(super::index_get::lower_symbol_property_get_ic( + ctx, &a, &key, )) }); } diff --git a/crates/perry-codegen/src/expr/binary.rs b/crates/perry-codegen/src/expr/binary.rs index f799aa4088..eb3865fbb3 100644 --- a/crates/perry-codegen/src/expr/binary.rs +++ b/crates/perry-codegen/src/expr/binary.rs @@ -785,6 +785,14 @@ fn try_lower_small_bigint_literal_binary( pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { match expr { Expr::Binary { op, left, right } => { + // A proven-u31 index turns the canonical 32-bit ECS bitset test + // into one guarded Uint32Array load. Match before the erased + // receiver makes the ordinary BitAnd arm choose its fully dynamic + // BigInt-capable lowering; every guard miss retains that lowering + // in the peephole's slow arm. + if let Some(value) = super::bitset_test::try_lower_u32_bitset_test(ctx, expr)? { + return Ok(value); + } if matches!(op, BinaryOp::Add) { // Use the stricter `is_definitely_string_expr` check for // the string-concat fast path. A union type `string|number` diff --git a/crates/perry-codegen/src/expr/bitset_test.rs b/crates/perry-codegen/src/expr/bitset_test.rs new file mode 100644 index 0000000000..0bec9aaba7 --- /dev/null +++ b/crates/perry-codegen/src/expr/bitset_test.rs @@ -0,0 +1,235 @@ +//! Guarded native lowering for the canonical 32-bit bitset membership idiom. +//! +//! ECS implementations commonly store component masks in a `Uint32Array` and +//! test one component with +//! +//! ```text +//! mask[~~(index / 32)] & (1 << (index % 32)) +//! ``` +//! +//! When `index` is an erased method parameter, lowering the expression one +//! node at a time loses the relationship between its parts: division and +//! remainder round-trip through doubles, the unknown receiver emits the full +//! eight-kind typed-array dispatch, and the final `&` calls the BigInt-capable +//! dynamic helper. A non-negative native-i32 proof makes the index arithmetic +//! exact. This module recognizes only that exact, side-effect-free tree and +//! emits one monomorphic Uint32Array guard plus a canonical slow path. + +use anyhow::Result; +use perry_hir::{BinaryOp, Expr, UnaryOp}; + +use crate::types::{DOUBLE, I1, I32, I64}; + +use super::{lower_expr, FnCtx}; + +/// Return the local ids `(mask, index)` when `expr` is exactly +/// `mask[~~(index / 32)] & (1 << (index % 32))`. +fn match_u32_bitset_test(expr: &Expr) -> Option<(u32, u32)> { + let Expr::Binary { + op: BinaryOp::BitAnd, + left, + right, + } = expr + else { + return None; + }; + let Expr::IndexGet { object, index } = left.as_ref() else { + return None; + }; + let Expr::LocalGet(mask_id) = object.as_ref() else { + return None; + }; + let Expr::Unary { + op: UnaryOp::BitNot, + operand: outer_not, + } = index.as_ref() + else { + return None; + }; + let Expr::Unary { + op: UnaryOp::BitNot, + operand: inner_not, + } = outer_not.as_ref() + else { + return None; + }; + let Expr::Binary { + op: BinaryOp::Div, + left: div_left, + right: div_right, + } = inner_not.as_ref() + else { + return None; + }; + let Expr::LocalGet(index_id) = div_left.as_ref() else { + return None; + }; + if !matches!(div_right.as_ref(), Expr::Integer(32)) { + return None; + } + + let Expr::Binary { + op: BinaryOp::Shl, + left: shift_left, + right: shift_right, + } = right.as_ref() + else { + return None; + }; + if !matches!(shift_left.as_ref(), Expr::Integer(1)) { + return None; + } + let Expr::Binary { + op: BinaryOp::Mod, + left: mod_left, + right: mod_right, + } = shift_right.as_ref() + else { + return None; + }; + if !matches!(mod_left.as_ref(), Expr::LocalGet(id) if id == index_id) + || !matches!(mod_right.as_ref(), Expr::Integer(32)) + { + return None; + } + + Some((*mask_id, *index_id)) +} + +/// Whether `expr` is the exact Number-producing bitset test recognized by +/// [`try_lower_u32_bitset_test`]. +/// +/// This deliberately exposes only the structural fact, not the matched local +/// ids. On every normal JavaScript exit the expression is a Number for every +/// index value: `/`, `%`, and `<<` either produce Numbers or throw, and the +/// Number shift operand means a BigInt loaded from `mask` throws at `&` +/// instead of producing a BigInt result. The non-negative proof is required +/// only by the native indexing peephole, not by this result-kind fact. +pub(crate) fn is_u32_bitset_test(expr: &Expr) -> bool { + match_u32_bitset_test(expr).is_some() +} + +/// Try the native Uint32Array bitset path. +/// +/// The index proof is deliberately two-part: the local must have a native i32 +/// slot *and* be known non-negative in this body. The `$idx_u31` method clone +/// supplies both. Without the lower-bound proof, `lshr index, 5` would differ +/// from JavaScript's truncation-toward-zero `~~(index / 32)` for negatives. +pub(super) fn try_lower_u32_bitset_test( + ctx: &mut FnCtx<'_>, + expr: &Expr, +) -> Result> { + let Some((mask_id, index_id)) = match_u32_bitset_test(expr) else { + return Ok(None); + }; + let Some(index_slot) = ctx.i32_counter_slots.get(&index_id).cloned() else { + return Ok(None); + }; + if !ctx.nonnegative_integer_locals.contains(&index_id) { + return Ok(None); + } + + // Both source operands are pure local/arithmetic trees. Evaluate the + // receiver once in source order, then keep all index work in native i32. + let mask_box = lower_expr(ctx, &Expr::LocalGet(mask_id))?; + let index_i32 = ctx.block().load(I32, &index_slot); + let word_i32 = ctx.block().lshr(I32, &index_i32, "5"); + let shift_i32 = ctx.block().and(I32, &index_i32, "31"); + let bit_i32 = ctx.block().shl(I32, "1", &shift_i32); + + let header_idx = ctx.new_block("u32bitset.header"); + let fast_idx = ctx.new_block("u32bitset.fast"); + let slow_idx = ctx.new_block("u32bitset.slow"); + let merge_idx = ctx.new_block("u32bitset.merge"); + let header_label = ctx.block_label(header_idx); + let fast_label = ctx.block_label(fast_idx); + let slow_label = ctx.block_label(slow_idx); + let merge_label = ctx.block_label(merge_idx); + + // The process-global cache uses `(receiver_address << 8) | kind`. An + // exact address hit is also what makes the later header load safe; no + // receiver-derived address is dereferenced before this branch. + let raw = { + let blk = ctx.block(); + let bits = blk.bitcast_double_to_i64(&mask_box); + let raw = blk.and(I64, &bits, crate::nanbox::POINTER_MASK_I64); + let tag = blk.and( + I64, + &bits, + &crate::nanbox::i64_literal(crate::nanbox::TAG_MASK), + ); + let is_pointer = blk.icmp_eq(I64, &tag, crate::nanbox::POINTER_TAG_I64); + let view_guard = blk.load(I64, "@PERRY_TA_VIEW_GUARD"); + let owns_inline_storage = blk.icmp_eq(I64, &view_guard, "0"); + let slot = blk.lshr(I64, &raw, "3"); + let slot = blk.and(I64, &slot, "63"); + let cache_ptr = blk.gep( + "[64 x i64]", + "@PERRY_TA_KIND_CACHE", + &[(I64, "0"), (I64, &slot)], + ); + let cache_entry = blk.load(I64, &cache_ptr); + let cached_addr = blk.lshr(I64, &cache_entry, "8"); + let address_matches = blk.icmp_eq(I64, &cached_addr, &raw); + let kind = blk.and(I64, &cache_entry, "255"); + // Numeric typed-array kind 5 is Uint32Array. Other kinds retain the + // canonical property read and ToNumeric behavior in the slow arm. + let is_uint32 = blk.icmp_eq(I64, &kind, "5"); + let guard = blk.and(I1, &is_pointer, &owns_inline_storage); + let guard = blk.and(I1, &guard, &address_matches); + let guard = blk.and(I1, &guard, &is_uint32); + blk.cond_br(&guard, &header_label, &slow_label); + raw + }; + + // The cache hit above proves `raw` is the live Uint32Array header. Its + // first word is the u32 length; only an in-bounds access may bypass + // `[[Get]]` because OOB access can observe prototype semantics. + ctx.current_block = header_idx; + let header_ptr = ctx.block().inttoptr(I64, &raw); + let length = ctx.block().load(I32, &header_ptr); + let in_bounds = ctx.block().icmp_ult(I32, &word_i32, &length); + ctx.block().cond_br(&in_bounds, &fast_label, &slow_label); + + // Owning Uint32Array data begins 16 bytes after its header. The bitwise + // result is interpreted as signed i32 by JavaScript's `&` operator. + ctx.current_block = fast_idx; + let word_i64 = ctx.block().zext(I32, &word_i32, I64); + let byte_offset = ctx.block().shl(I64, &word_i64, "2"); + let data_addr = ctx.block().add(I64, &raw, "16"); + let elem_addr = ctx.block().add(I64, &data_addr, &byte_offset); + let elem_ptr = ctx.block().inttoptr(I64, &elem_addr); + let lane = ctx.block().load(I32, &elem_ptr); + let native_result = ctx.block().and(I32, &lane, &bit_i32); + let fast_value = ctx.block().sitofp(I32, &native_result, DOUBLE); + let fast_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + // Preserve the original expression on every miss. `js_dyn_index_get` + // implements RequireObjectCoercible, arbitrary property keys, proxies, + // typed-array views/kinds, OOB behavior, and accessors. The dynamic `&` + // retains BigInt and mixed-BigInt TypeError semantics. There is no + // allocation between the first return and the second call; the latter + // roots both arguments before ToNumeric can allocate. + ctx.current_block = slow_idx; + let word_double = ctx.block().uitofp(I32, &word_i32, DOUBLE); + let bit_double = ctx.block().sitofp(I32, &bit_i32, DOUBLE); + let loaded = ctx.block().call( + DOUBLE, + "js_dyn_index_get", + &[(DOUBLE, &mask_box), (DOUBLE, &word_double)], + ); + let slow_value = ctx.block().call( + DOUBLE, + "js_dynamic_bitand", + &[(DOUBLE, &loaded), (DOUBLE, &bit_double)], + ); + let slow_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + Ok(Some(ctx.block().phi( + DOUBLE, + &[(&fast_value, &fast_end), (&slow_value, &slow_end)], + ))) +} diff --git a/crates/perry-codegen/src/expr/call_return_array_index_tests.rs b/crates/perry-codegen/src/expr/call_return_array_index_tests.rs index 68802a99f7..8e45f05903 100644 --- a/crates/perry-codegen/src/expr/call_return_array_index_tests.rs +++ b/crates/perry-codegen/src/expr/call_return_array_index_tests.rs @@ -119,8 +119,23 @@ fn compile_store_ir(receiver_selector: i64) -> String { } fn write_method_ir(ir: &str) -> &str { - let signature = "define double @perry_method_call_return_array_put_value_ts__Store__write("; - let start = ir.find(signature).expect("write method is present in IR"); + // Index-specialized methods publish a small guard wrapper and retain the + // original semantics in `$generic`; assertions about assignment lowering + // belong to that body rather than the wrapper. + let generic = "@perry_method_call_return_array_put_value_ts__Store__write$generic("; + let public = "@perry_method_call_return_array_put_value_ts__Store__write("; + let start = ir + .match_indices("define ") + .map(|(start, _)| start) + .find(|start| { + let line_end = ir[*start..] + .find('\n') + .map(|len| *start + len) + .unwrap_or(ir.len()); + let signature = &ir[*start..line_end]; + signature.contains(generic) || signature.contains(public) + }) + .expect("write method body is present in IR"); let method_and_rest = &ir[start..]; let end = method_and_rest .find("\n}\n") diff --git a/crates/perry-codegen/src/expr/compare.rs b/crates/perry-codegen/src/expr/compare.rs index 3435d161a3..40826f145d 100644 --- a/crates/perry-codegen/src/expr/compare.rs +++ b/crates/perry-codegen/src/expr/compare.rs @@ -16,7 +16,44 @@ use crate::type_analysis::{ }; use crate::types::{DOUBLE, I1, I32, I64, I8}; -use super::{unbox_str_handle, unbox_to_i64, FnCtx}; +use super::{lower_expr, unbox_str_handle, unbox_to_i64, FnCtx}; + +/// Runtime ABI values returned by `js_value_typeof_tag`. The classifier owns +/// Perry's representation exceptions; this table merely maps the eight +/// ECMAScript result strings to that stable integer ABI. +fn typeof_literal_tag(literal: &str) -> Option { + match literal { + "undefined" => Some(0), + "object" => Some(1), + "boolean" => Some(2), + "number" => Some(3), + "string" => Some(4), + "function" => Some(5), + "bigint" => Some(6), + "symbol" => Some(7), + _ => None, + } +} + +/// Recognize the safe, high-value subset of literal `typeof` comparisons. +/// +/// Restricting the operand to a local is intentional. `Expr::TypeOf` has +/// compile-time representation corrections for namespace/class/native-module +/// expressions in `literals_vars.rs`; intercepting those before ordinary +/// lowering would bypass those corrections. A local already takes the runtime +/// classifier today, so replacing its returned string with the same +/// classifier's integer tag changes no semantic route. +fn local_typeof_literal_pair<'a>(left: &'a Expr, right: &'a Expr) -> Option<(&'a Expr, u32)> { + match (left, right) { + (Expr::TypeOf(operand), Expr::String(literal)) + | (Expr::String(literal), Expr::TypeOf(operand)) + if matches!(operand.as_ref(), Expr::LocalGet(_)) => + { + typeof_literal_tag(literal).map(|tag| (operand.as_ref(), tag)) + } + _ => None, + } +} /// True only when compiler-owned initializer provenance establishes that this /// expression currently contains a Symbol identity. @@ -27,7 +64,7 @@ use super::{unbox_str_handle, unbox_to_i64, FnCtx}; /// storage (reclaimable but non-moving), while `Symbol.for()` values are /// process-lifetime `Box` allocations. Therefore a proven Symbol can equal /// another JS value iff their NaN-boxed pointer bits are identical. -fn is_proven_symbol_expr(ctx: &FnCtx<'_>, expr: &Expr) -> bool { +pub(crate) fn is_proven_symbol_expr(ctx: &FnCtx<'_>, expr: &Expr) -> bool { match expr { Expr::SymbolNew(_) | Expr::SymbolFor(_) => true, Expr::LocalGet(id) => { @@ -401,6 +438,150 @@ fn lower_strict_eq_inline_any(ctx: &mut FnCtx<'_>, l: &str, r: &str) -> String { ) } +/// Normalize Perry's compact INT32 immediate to an ordinary IEEE double. +/// Every other bit pattern is left unchanged. That makes a subsequent `fcmp` +/// exact for an arbitrary JS value against a proven Number: non-number tags +/// remain NaNs (and therefore compare unequal), while both Number encodings +/// participate in the numeric comparison. +fn normalize_int32_immediate(ctx: &mut FnCtx<'_>, value: &str) -> String { + let bits = ctx.block().bitcast_double_to_i64(value); + let top16 = ctx.block().lshr(I64, &bits, "48"); + let is_i32 = ctx + .block() + .icmp_eq(I64, &top16, crate::nanbox::INT32_TAG_TOP16_I64); + let raw = ctx.block().trunc(I64, &bits, I32); + let decoded = ctx.block().sitofp(I32, &raw, DOUBLE); + ctx.block().select(I1, &is_i32, DOUBLE, &decoded, value) +} + +/// Strict equality where exactly one operand is a proven Number. +/// +/// `fcmp` already rejects every Perry non-number tag because those encodings +/// are NaNs. The only representation it cannot consume directly is the +/// canonical INT32 immediate, which we normalize above. This therefore +/// replaces `js_eq` without speculating on an object's shape or invoking any +/// coercion (strict equality never coerces). +fn lower_strict_eq_against_number(ctx: &mut FnCtx<'_>, op: CompareOp, l: &str, r: &str) -> String { + let l = normalize_int32_immediate(ctx, l); + let r = normalize_int32_immediate(ctx, r); + let pred = if matches!(op, CompareOp::Ne) { + // `une` is required for both ordinary NaN and every non-number tag. + "une" + } else { + "oeq" + }; + let bit = ctx.block().fcmp(pred, &l, &r); + let tagged = ctx.block().select( + I1, + &bit, + I64, + crate::nanbox::TAG_TRUE_I64, + crate::nanbox::TAG_FALSE_I64, + ); + ctx.block().bitcast_i64_to_double(&tagged) +} + +/// Inline the primitive-number arm of Abstract Relational Comparison when one +/// operand is already proven numeric. +/// +/// Ordinary Numbers, compact INT32 values, `undefined`, `null`, and booleans +/// have a side-effect-free `ToPrimitive`/`ToNumber` result. Strings, BigInts, +/// Symbols, and objects retain the complete runtime helper because their +/// coercion either follows different rules or may execute user code. +fn lower_relational_against_number( + ctx: &mut FnCtx<'_>, + op: CompareOp, + l: &str, + r: &str, + dynamic_is_left: bool, + fallback_fn: &str, +) -> String { + let dynamic = if dynamic_is_left { l } else { r }; + let bits = ctx.block().bitcast_double_to_i64(dynamic); + let top16 = ctx.block().lshr(I64, &bits, "48"); + let below_tag_band = + ctx.block() + .icmp_ult(I64, &top16, crate::nanbox::SHORT_STRING_TAG_TOP16_I64); + let above_tag_band = ctx + .block() + .icmp_ugt(I64, &top16, crate::nanbox::STRING_TAG_TOP16_I64); + let is_plain_number = ctx.block().or(I1, &below_tag_band, &above_tag_band); + let is_i32 = ctx + .block() + .icmp_eq(I64, &top16, crate::nanbox::INT32_TAG_TOP16_I64); + let is_undefined = ctx + .block() + .icmp_eq(I64, &bits, crate::nanbox::TAG_UNDEFINED_I64); + let is_null = ctx.block().icmp_eq(I64, &bits, crate::nanbox::TAG_NULL_I64); + let is_false = ctx + .block() + .icmp_eq(I64, &bits, crate::nanbox::TAG_FALSE_I64); + let is_true = ctx.block().icmp_eq(I64, &bits, crate::nanbox::TAG_TRUE_I64); + let is_zero = ctx.block().or(I1, &is_null, &is_false); + let primitive = ctx.block().or(I1, &is_plain_number, &is_i32); + let primitive = ctx.block().or(I1, &primitive, &is_undefined); + let primitive = ctx.block().or(I1, &primitive, &is_zero); + let primitive = ctx.block().or(I1, &primitive, &is_true); + + let fast_idx = ctx.new_block("relnum.fast"); + let slow_idx = ctx.new_block("relnum.slow"); + let merge_idx = ctx.new_block("relnum.merge"); + let fast_l = ctx.block_label(fast_idx); + let slow_l = ctx.block_label(slow_idx); + let merge_l = ctx.block_label(merge_idx); + ctx.block().cond_br(&primitive, &fast_l, &slow_l); + + ctx.current_block = fast_idx; + let raw = ctx.block().trunc(I64, &bits, I32); + let int_value = ctx.block().sitofp(I32, &raw, DOUBLE); + let dynamic_number = ctx.block().select(I1, &is_i32, DOUBLE, &int_value, dynamic); + let dynamic_number = ctx + .block() + .select(I1, &is_zero, DOUBLE, "0.0", &dynamic_number); + let dynamic_number = ctx + .block() + .select(I1, &is_true, DOUBLE, "1.0", &dynamic_number); + // `undefined` deliberately retains its tagged NaN. Ordered fcmp then + // returns false for all four relational operators, exactly as ToNumber. + let proven_number = normalize_int32_immediate(ctx, if dynamic_is_left { r } else { l }); + let (left_number, right_number) = if dynamic_is_left { + (&dynamic_number, &proven_number) + } else { + (&proven_number, &dynamic_number) + }; + let pred = match op { + CompareOp::Lt => "olt", + CompareOp::Le => "ole", + CompareOp::Gt => "ogt", + CompareOp::Ge => "oge", + _ => unreachable!("relational-number helper received equality op"), + }; + let fast_bit = ctx.block().fcmp(pred, left_number, right_number); + let fast_bits = ctx.block().select( + I1, + &fast_bit, + I64, + crate::nanbox::TAG_TRUE_I64, + crate::nanbox::TAG_FALSE_I64, + ); + let fast_result = ctx.block().bitcast_i64_to_double(&fast_bits); + let fast_pred = ctx.block().label.clone(); + ctx.block().br(&merge_l); + + ctx.current_block = slow_idx; + let slow_result = ctx + .block() + .call(DOUBLE, fallback_fn, &[(DOUBLE, l), (DOUBLE, r)]); + let slow_pred = ctx.block().label.clone(); + ctx.block().br(&merge_l); + + ctx.current_block = merge_idx; + ctx.block().phi( + DOUBLE, + &[(&fast_result, &fast_pred), (&slow_result, &slow_pred)], + ) +} + /// Resolve equal-length heap strings of up to three bytes inline, and reject /// longer pairs on a length or endpoint mismatch before using the full helper. /// The caller has already proven both values carry `STRING_TAG`. @@ -645,6 +826,35 @@ fn lower_string_strict_eq_inline( pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { match expr { Expr::Compare { op, left, right } => { + if matches!(op, CompareOp::Eq | CompareOp::Ne) { + if let Some((operand, expected_tag)) = + local_typeof_literal_pair(left.as_ref(), right.as_ref()) + { + // The literal has no evaluation side effects. Lower the + // local operand exactly once, then compare the shared + // classifier's integer result instead of materializing a + // heap string and entering string equality. + let value = lower_expr(ctx, operand)?; + let tag = ctx.block().call( + I32, + "js_value_typeof_tag", + &[(crate::types::DOUBLE, &value)], + ); + let bit = if matches!(op, CompareOp::Ne) { + ctx.block().icmp_ne(I32, &tag, &expected_tag.to_string()) + } else { + ctx.block().icmp_eq(I32, &tag, &expected_tag.to_string()) + }; + let tagged = ctx.block().select( + I1, + &bit, + I64, + crate::nanbox::TAG_TRUE_I64, + crate::nanbox::TAG_FALSE_I64, + ); + return Ok(ctx.block().bitcast_i64_to_double(&tagged)); + } + } // #7979: every arm below used to lower `left`, then lower `right`, // then consume the original left SSA value. A call-result string // therefore named retired from-space whenever the right call @@ -1170,14 +1380,18 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // through the runtime `js_rel_*` helpers, which return a NaN-boxed // boolean. The statically-numeric case keeps the bare `fcmp` fast // path below (and Dates are subsumed — they aren't numeric_expr). - let both_numeric = is_numeric_expr(ctx, left) - && is_numeric_expr(ctx, right) + let left_numeric = is_numeric_expr(ctx, left) && !expr_may_return_boxed_value_from_raw_f64_fallback(ctx, left) + && !is_bigint_expr(ctx, left); + let right_numeric = is_numeric_expr(ctx, right) && !expr_may_return_boxed_value_from_raw_f64_fallback(ctx, right) - && !is_bigint_expr(ctx, left) && !is_bigint_expr(ctx, right); + let both_numeric = left_numeric && right_numeric; + let exactly_one_numeric = left_numeric ^ right_numeric; + if matches!(op, CompareOp::Eq | CompareOp::Ne) && exactly_one_numeric { + return Ok(lower_strict_eq_against_number(ctx, *op, &l, &r)); + } if is_relational_op && !both_numeric { - let blk = ctx.block(); let fname = match op { CompareOp::Lt => "js_rel_lt", CompareOp::Le => "js_rel_le", @@ -1185,6 +1399,17 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { CompareOp::Ge => "js_rel_ge", _ => unreachable!(), }; + if exactly_one_numeric { + return Ok(lower_relational_against_number( + ctx, + *op, + &l, + &r, + !left_numeric, + fname, + )); + } + let blk = ctx.block(); let res = blk.call(DOUBLE, fname, &[(DOUBLE, &l), (DOUBLE, &r)]); return Ok(res); } diff --git a/crates/perry-codegen/src/expr/compare_tests.rs b/crates/perry-codegen/src/expr/compare_tests.rs index 08afb63337..36d0bd5d7a 100644 --- a/crates/perry-codegen/src/expr/compare_tests.rs +++ b/crates/perry-codegen/src/expr/compare_tests.rs @@ -539,3 +539,125 @@ fn i8_literal_writes_high_bytes_in_twos_complement() { assert_eq!(i8_literal(0xC3), "-61"); assert_eq!(i8_literal(0xFF), "-1"); } + +#[test] +fn local_typeof_strict_ne_literal_uses_the_integer_classifier() { + let ir = cmp_ir( + "typeof_local_ne_number", + CompareOp::Ne, + Expr::TypeOf(Box::new(Expr::LocalGet(X))), + Expr::String("number".to_string()), + ); + assert!( + ir.contains("call i32 @js_value_typeof_tag("), + "literal typeof comparison did not use the integer classifier:\n{ir}" + ); + assert!( + !ir.contains("call i64 @js_value_typeof("), + "literal typeof comparison still materialized a typeof string:\n{ir}" + ); + assert!( + !ir.contains("call i32 @js_string_equals("), + "literal typeof comparison still entered string equality:\n{ir}" + ); +} + +#[test] +fn reversed_local_typeof_strict_eq_uses_the_same_integer_classifier() { + let ir = cmp_ir( + "typeof_local_eq_reversed", + CompareOp::Eq, + Expr::String("string".to_string()), + Expr::TypeOf(Box::new(Expr::LocalGet(X))), + ); + assert!(ir.contains("call i32 @js_value_typeof_tag("), "{ir}"); + assert!(!ir.contains("call i64 @js_value_typeof("), "{ir}"); +} + +#[test] +fn nonliteral_typeof_comparison_keeps_runtime_string_semantics() { + let ir = cmp_ir( + "typeof_nonliteral_compare", + CompareOp::Eq, + Expr::TypeOf(Box::new(Expr::LocalGet(X))), + Expr::LocalGet(Y), + ); + assert!( + ir.contains("call i64 @js_value_typeof("), + "a nonliteral comparison incorrectly took the integer-tag ABI:\n{ir}" + ); + assert!(!ir.contains("call i32 @js_value_typeof_tag("), "{ir}"); +} + +#[test] +fn dynamic_strict_eq_against_number_normalizes_int32_without_js_eq() { + let ir = cmp_ir( + "dynamic_strict_eq_number", + CompareOp::Eq, + Expr::LocalGet(X), + Expr::Number(7.0), + ); + assert!( + ir.contains(crate::nanbox::INT32_TAG_TOP16_I64), + "the compact-INT32 normalization guard is absent:\n{ir}" + ); + assert!( + ir.contains("fcmp oeq double"), + "dynamic-vs-number equality did not become numeric fcmp:\n{ir}" + ); + assert!( + !ir.contains(JS_EQ_CALL), + "dynamic-vs-number strict equality retained js_eq:\n{ir}" + ); +} + +#[test] +fn reversed_dynamic_strict_ne_against_number_uses_unordered_numeric_compare() { + let ir = cmp_ir( + "dynamic_strict_ne_number_reversed", + CompareOp::Ne, + Expr::Number(7.0), + Expr::LocalGet(X), + ); + assert!( + ir.contains("fcmp une double"), + "strict !== must treat NaN and every non-number tag as unequal:\n{ir}" + ); + assert!(!ir.contains(JS_EQ_CALL), "{ir}"); +} + +#[test] +fn dynamic_relational_against_number_inlines_primitive_arm_and_keeps_coercing_fallback() { + let ir = cmp_ir( + "dynamic_lt_number", + CompareOp::Lt, + Expr::LocalGet(X), + Expr::Number(7.0), + ); + assert!( + ir.contains("relnum.fast") && ir.contains("relnum.slow"), + "dynamic-vs-number relational compare lacks guarded primitive dispatch:\n{ir}" + ); + assert!( + ir.contains("fcmp olt double"), + "the admitted primitive arm did not lower to fcmp:\n{ir}" + ); + assert!( + ir.contains("call double @js_rel_lt("), + "objects, strings, BigInts, and Symbols lost their coercing fallback:\n{ir}" + ); +} + +#[test] +fn loose_equality_against_number_keeps_coercion() { + let ir = cmp_ir( + "dynamic_loose_eq_number", + CompareOp::LooseEq, + Expr::LocalGet(X), + Expr::Number(7.0), + ); + assert!( + ir.contains(JS_LOOSE_EQ_CALL), + "dynamic == number incorrectly bypassed coercion:\n{ir}" + ); +} diff --git a/crates/perry-codegen/src/expr/index_get.rs b/crates/perry-codegen/src/expr/index_get.rs index 5c007fa0f0..98fb37c6e6 100644 --- a/crates/perry-codegen/src/expr/index_get.rs +++ b/crates/perry-codegen/src/expr/index_get.rs @@ -28,7 +28,7 @@ use crate::native_value::{ }; use crate::rooting; use crate::type_analysis::{is_array_expr, is_numeric_expr, is_string_expr, receiver_class_name}; -use crate::types::{DOUBLE, I1, I16, I32, I64, I8}; +use crate::types::{DOUBLE, I1, I16, I32, I64, I8, PTR}; use super::{ array_kind_fact, attach_buffer_view_pointer_state_for_expr, @@ -48,6 +48,70 @@ use guarded_array::{ }; use inline_dyn_typed_array::lower_inline_dyn_typed_array_get; +/// Emit a weak monomorphic IC for an exact own Symbol-keyed data property. +/// +/// The cache stores raw bits, not roots. Its epoch is advanced by every +/// Symbol-property mutation and completed GC, so a moved/reclaimed receiver or +/// value cannot hit and the cache cannot keep otherwise-dead objects alive. +pub(crate) fn lower_symbol_property_get_ic( + ctx: &mut FnCtx<'_>, + obj_box: &str, + sym_box: &str, +) -> String { + let site_id = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let cache_name = super::inline_cache_global_name(ctx, site_id); + ctx.ic_globals.push(cache_name.clone()); + let cache_ref = format!("@{cache_name}"); + + let hit_idx = ctx.new_block("symic.hit"); + let miss_idx = ctx.new_block("symic.miss"); + let merge_idx = ctx.new_block("symic.merge"); + let hit_label = ctx.block_label(hit_idx); + let miss_label = ctx.block_label(miss_idx); + let merge_label = ctx.block_label(merge_idx); + + let epoch = ctx + .block() + .load_atomic_acquire(I64, "@PERRY_SYMBOL_PROPERTY_IC_EPOCH", 8); + let cached_epoch_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]); + let cached_epoch = ctx.block().load_atomic_acquire(I64, &cached_epoch_ptr, 8); + let epoch_matches = ctx.block().icmp_eq(I64, &epoch, &cached_epoch); + let obj_bits = ctx.block().bitcast_double_to_i64(obj_box); + let cached_obj_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]); + let cached_obj = ctx.block().load(I64, &cached_obj_ptr); + let obj_matches = ctx.block().icmp_eq(I64, &obj_bits, &cached_obj); + let sym_bits = ctx.block().bitcast_double_to_i64(sym_box); + let cached_sym_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "2")]); + let cached_sym = ctx.block().load(I64, &cached_sym_ptr); + let sym_matches = ctx.block().icmp_eq(I64, &sym_bits, &cached_sym); + let identity_matches = ctx.block().and(I1, &obj_matches, &sym_matches); + let hit = ctx.block().and(I1, &epoch_matches, &identity_matches); + ctx.block().cond_br(&hit, &hit_label, &miss_label); + + ctx.current_block = hit_idx; + let cached_value_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "3")]); + let cached_value_bits = ctx.block().load(I64, &cached_value_ptr); + let cached_value = ctx.block().bitcast_i64_to_double(&cached_value_bits); + let hit_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = miss_idx; + let miss_value = ctx.block().call( + DOUBLE, + "js_object_get_symbol_property_ic_miss", + &[(DOUBLE, obj_box), (DOUBLE, sym_box), (PTR, &cache_ref)], + ); + let miss_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + ctx.block().phi( + DOUBLE, + &[(&cached_value, &hit_end), (&miss_value, &miss_end)], + ) +} + /// #7494: deliberately `static_type_of`, not `receiver_class_name`. /// /// `receiver_class_name` returns `None` for any `Expr::LocalGet(id)` with @@ -656,7 +720,8 @@ pub(crate) fn lower_unknown_local_index_get_for_number_context( let index_is_static_string_or_symbol = matches!( index.as_ref(), Expr::String(_) | Expr::WtfString(_) | Expr::SymbolFor(_) - ) || is_string_expr(ctx, index); + ) || is_string_expr(ctx, index) + || super::compare::is_proven_symbol_expr(ctx, index); if index_is_static_string_or_symbol { return Ok(None); } @@ -932,18 +997,14 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // symbol keys to the symbol-property resolver (mirrors the array // path), which exposes `@@toStringTag` (`safe-stable-stringify`) // and `@@iterator`. - if matches!(index.as_ref(), Expr::SymbolFor(_)) { + if super::compare::is_proven_symbol_expr(ctx, index) { // #7640 section B (MEDIUM): `Expr::SymbolFor` lowers to a // real `js_symbol_for` call, which INTERNS — it allocates a // SymbolHeader on first use — so the receiver was live // across an allocation. return rooting::with_operands_rooted(ctx, &[object, index], |ctx, vals| { let (obj_box, key_box) = (vals[0].clone(), vals[1].clone()); - Ok(ctx.block().call( - DOUBLE, - "js_object_get_symbol_property", - &[(DOUBLE, &obj_box), (DOUBLE, &key_box)], - )) + Ok(lower_symbol_property_get_ic(ctx, &obj_box, &key_box)) }); } // #2063 / fractional numeric keys: only proven integer element @@ -1245,7 +1306,8 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { let index_is_static_string_or_symbol = matches!( index.as_ref(), Expr::String(_) | Expr::WtfString(_) | Expr::SymbolFor(_) - ) || is_string_expr(ctx, index); + ) || is_string_expr(ctx, index) + || super::compare::is_proven_symbol_expr(ctx, index); // #7854 recovered a receiver's declared array type for a LOCAL // (`const names = e.names`), never for the read used directly as a // receiver (`e.vals[i]`, `p.toks[p.pos]`) — the HIR types a @@ -1298,18 +1360,14 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // value yields a garbage index (returned a number). Route symbol // keys to the symbol-property resolver, which exposes the array // iterator for `Symbol.iterator`. - if matches!(index.as_ref(), Expr::SymbolFor(_)) { + if super::compare::is_proven_symbol_expr(ctx, index) { // #7640 section B (MEDIUM): `Expr::SymbolFor` lowers to a // real `js_symbol_for` call, which INTERNS — it allocates a // SymbolHeader on first use — so the receiver was live // across an allocation. return rooting::with_operands_rooted(ctx, &[object, index], |ctx, vals| { let (obj_box, key_box) = (vals[0].clone(), vals[1].clone()); - Ok(ctx.block().call( - DOUBLE, - "js_object_get_symbol_property", - &[(DOUBLE, &obj_box), (DOUBLE, &key_box)], - )) + Ok(lower_symbol_property_get_ic(ctx, &obj_box, &key_box)) }); } if !is_numeric_expr(ctx, index) { @@ -1560,6 +1618,9 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { let obj_box = ctx.block() .call(DOUBLE, "js_require_object_coercible", &[(DOUBLE, &obj_box)]); + if super::compare::is_proven_symbol_expr(ctx, index) { + return Ok(lower_symbol_property_get_ic(ctx, &obj_box, &idx_box)); + } let blk = ctx.block(); let obj_bits = blk.bitcast_double_to_i64(&obj_box); let obj_handle = @@ -1579,11 +1640,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { ctx.block().cond_br(&is_sym_bit, &sym_lbl, &nonsym_lbl); // Symbol key → side-table get. ctx.current_block = sym_idx; - let v_sym = ctx.block().call( - DOUBLE, - "js_object_get_symbol_property", - &[(DOUBLE, &obj_box), (DOUBLE, &idx_box)], - ); + let v_sym = lower_symbol_property_get_ic(ctx, &obj_box, &idx_box); let sym_end_lbl = ctx.block().label.clone(); ctx.block().br(&merge_lbl); // Not a symbol → recompute idx_bits in this block. diff --git a/crates/perry-codegen/src/expr/index_get/guarded_array.rs b/crates/perry-codegen/src/expr/index_get/guarded_array.rs index 6fcf6aac67..c54878c435 100644 --- a/crates/perry-codegen/src/expr/index_get/guarded_array.rs +++ b/crates/perry-codegen/src/expr/index_get/guarded_array.rs @@ -74,6 +74,18 @@ pub(super) fn lower_guarded_array_index_get( ); let fast_idx = ctx.new_block(&format!("{}.fast", block_prefix)); let fallback_idx = ctx.new_block(&format!("{}.fallback", block_prefix)); + // A non-negative ordinary-array index at or above `length` has no own + // element: defining an array-index property would have raised `length`. + // Once the same structural checks used by the raw-load tier have also + // proved that there are no indexed descriptors and no indexed prototype + // properties, that result is `undefined` without consulting the generic + // polymorphic getter. Sparse-set membership tests hit exactly this arm for + // absent ids, so keep it separate from the in-bounds raw-load block. + let inline_oob_idx = if !typed_feedback_emission_enabled() { + Some(ctx.new_block(&format!("{}.guard.oob", block_prefix))) + } else { + None + }; let merge_idx = ctx.new_block(&format!("{}.merge", block_prefix)); let fast_label = ctx.block_label(fast_idx); let fallback_label = ctx.block_label(fallback_idx); @@ -114,6 +126,8 @@ pub(super) fn lower_guarded_array_index_get( Some(idx) => ctx.block_label(idx), None => fallback_label.clone(), }; + let range_idx = ctx.new_block(&format!("{}.guard.range", block_prefix)); + let range_label = ctx.block_label(range_idx); { let blk = ctx.block(); let arr_bits = blk.bitcast_double_to_i64(arr_box); @@ -167,7 +181,7 @@ pub(super) fn lower_guarded_array_index_get( }; ctx.current_block = live_deref_idx; - { + let (index_in_bounds, reserved) = { let blk = ctx.block(); let live_gc_type_addr = blk.sub(I64, &live_handle, "8"); let live_gc_type_ptr = blk.inttoptr(I64, &live_gc_type_addr); @@ -200,14 +214,29 @@ pub(super) fn lower_guarded_array_index_get( let capacity_sane = blk.icmp_ule(I32, &capacity, "16000000"); let length_within_capacity = blk.icmp_ule(I32, &length, &capacity); - let mut guard_ok = blk.and(I1, &is_array, ¬_forwarded); - guard_ok = blk.and(I1, &guard_ok, &no_descriptors); - guard_ok = blk.and(I1, &guard_ok, &default_prototype_chain); - guard_ok = blk.and(I1, &guard_ok, &index_nonnegative); - guard_ok = blk.and(I1, &guard_ok, &index_in_bounds); - guard_ok = blk.and(I1, &guard_ok, &length_sane); - guard_ok = blk.and(I1, &guard_ok, &capacity_sane); - guard_ok = blk.and(I1, &guard_ok, &length_within_capacity); + let mut structural_ok = blk.and(I1, &is_array, ¬_forwarded); + structural_ok = blk.and(I1, &structural_ok, &no_descriptors); + structural_ok = blk.and(I1, &structural_ok, &default_prototype_chain); + structural_ok = blk.and(I1, &structural_ok, &index_nonnegative); + structural_ok = blk.and(I1, &structural_ok, &length_sane); + structural_ok = blk.and(I1, &structural_ok, &capacity_sane); + structural_ok = blk.and(I1, &structural_ok, &length_within_capacity); + blk.cond_br(&structural_ok, &range_label, &guard_fail_label); + + // `index_in_bounds` and `reserved` dominate the range block. The + // former selects raw load versus the proven-absent result; the + // latter carries the optional numeric-layout proof below. + (index_in_bounds, reserved) + }; + + let numeric_in_bounds_idx = require_numeric_layout + .then(|| ctx.new_block(&format!("{}.guard.numeric_in_bounds", block_prefix))); + let numeric_in_bounds_label = numeric_in_bounds_idx.map(|idx| ctx.block_label(idx)); + let oob_label = ctx.block_label(inline_oob_idx.expect("normal-build OOB block")); + ctx.current_block = range_idx; + { + let blk = ctx.block(); + let mut in_bounds_ok = index_in_bounds.clone(); if require_numeric_layout { // Dense raw-f64 proof: every slot in [0, length) holds // canonical raw f64 bits (GC_ARRAY_RAW_F64_LAYOUT, 0x80). @@ -229,10 +258,26 @@ pub(super) fn lower_guarded_array_index_get( }; let raw_bits = blk.and(I16, &reserved, raw_mask); let is_raw = blk.icmp_ne(I16, &raw_bits, "0"); - guard_ok = blk.and(I1, &guard_ok, &is_raw); + in_bounds_ok = blk.and(I1, &in_bounds_ok, &is_raw); + } + if require_numeric_layout { + // An in-bounds array without the requested numeric layout must + // still visit the cold rebuilding guard. OOB needs no element + // layout at all and can return directly. + let in_bounds_idx = numeric_in_bounds_idx.expect("numeric in-bounds block"); + let in_bounds_label = numeric_in_bounds_label + .as_deref() + .expect("numeric in-bounds label"); + blk.cond_br(&index_in_bounds, &in_bounds_label, &oob_label); + + ctx.current_block = in_bounds_idx; + ctx.block() + .cond_br(&in_bounds_ok, &fast_label, &guard_fail_label); + inline_fast_handle = Some((live_handle, ctx.block().label.clone())); + } else { + inline_fast_handle = Some((live_handle, blk.label.clone())); + blk.cond_br(&in_bounds_ok, &fast_label, &oob_label); } - inline_fast_handle = Some((live_handle, blk.label.clone())); - blk.cond_br(&guard_ok, &fast_label, &guard_fail_label); } if let Some(cold_idx) = cold_guard_idx { @@ -296,6 +341,20 @@ pub(super) fn lower_guarded_array_index_get( ctx.block().cond_br(&guard_ok, &fast_label, &fallback_label); } + let inline_oob = inline_oob_idx.map(|oob_idx| { + ctx.current_block = oob_idx; + let value = if require_numeric_layout && coerce_numeric_fallback { + // This is ToNumber(undefined), matching the boxed fallback. + "0x7FF8000000000000".to_string() + } else { + ctx.block() + .bitcast_i64_to_double(crate::nanbox::TAG_UNDEFINED_I64) + }; + let end_label = ctx.block().label.clone(); + ctx.block().br(&merge_label); + (value, end_label) + }); + ctx.current_block = fallback_idx; // Materialize the f64 index only here (cold path) so the int→fp conversion // stays out of the numeric loop's hot region. @@ -443,13 +502,14 @@ pub(super) fn lower_guarded_array_index_get( } ctx.current_block = merge_idx; - Ok(ctx.block().phi( - DOUBLE, - &[ - (&fast_val, &fast_end_label), - (&fallback_val, &fallback_end_label), - ], - )) + let mut incoming: Vec<(&str, &str)> = vec![ + (fast_val.as_str(), fast_end_label.as_str()), + (fallback_val.as_str(), fallback_end_label.as_str()), + ]; + if let Some((oob_value, oob_end_label)) = inline_oob.as_ref() { + incoming.push((oob_value.as_str(), oob_end_label.as_str())); + } + Ok(ctx.block().phi(DOUBLE, &incoming)) } pub(super) fn packed_f64_loop_fact( diff --git a/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs b/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs index f911454ef6..89b38a4272 100644 --- a/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs +++ b/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs @@ -325,19 +325,53 @@ pub(super) fn lower_inline_dyn_typed_array_get( let cache_ref = format!("@{cache_name}"); let object_header_idx = ctx.new_block("arrlike.ic.header"); + let object_brand_idx = ctx.new_block("arrlike.ic.brand"); + let object_array_guard_idx = ctx.new_block("arrlike.ic.array_guard"); + let object_array_load_idx = ctx.new_block("arrlike.ic.array_load"); + let object_shape_idx = ctx.new_block("arrlike.ic.shape"); + let object_identity_idx = ctx.new_block("arrlike.ic.identity"); + let object_exact_idx = ctx.new_block("arrlike.ic.exact"); + let object_family_meta_idx = ctx.new_block("arrlike.ic.family_meta"); + let object_family_token_idx = ctx.new_block("arrlike.ic.family_token"); let object_bounds_idx = ctx.new_block("arrlike.ic.bounds"); + let object_length_inline_idx = ctx.new_block("arrlike.ic.length_inline"); + let object_length_spill_meta_idx = ctx.new_block("arrlike.ic.length_spill_meta"); + let object_length_spill_ptr_idx = ctx.new_block("arrlike.ic.length_spill_ptr"); + let object_length_spill_load_idx = ctx.new_block("arrlike.ic.length_spill_load"); + let object_range_idx = ctx.new_block("arrlike.ic.range"); let object_inline_idx = ctx.new_block("arrlike.ic.inline"); let object_spill_idx = ctx.new_block("arrlike.ic.spill"); let object_spill_ptr_idx = ctx.new_block("arrlike.ic.spill_ptr"); let object_spill_load_idx = ctx.new_block("arrlike.ic.spill_load"); let object_miss_idx = ctx.new_block("arrlike.ic.miss"); let object_header_label = ctx.block_label(object_header_idx); + let object_brand_label = ctx.block_label(object_brand_idx); + let object_array_guard_label = ctx.block_label(object_array_guard_idx); + let object_array_load_label = ctx.block_label(object_array_load_idx); + let object_shape_label = ctx.block_label(object_shape_idx); + let object_identity_label = ctx.block_label(object_identity_idx); + let object_exact_label = ctx.block_label(object_exact_idx); + let object_family_meta_label = ctx.block_label(object_family_meta_idx); + let object_family_token_label = ctx.block_label(object_family_token_idx); let object_bounds_label = ctx.block_label(object_bounds_idx); + let object_length_inline_label = ctx.block_label(object_length_inline_idx); + let object_length_spill_meta_label = ctx.block_label(object_length_spill_meta_idx); + let object_length_spill_ptr_label = ctx.block_label(object_length_spill_ptr_idx); + let object_length_spill_load_label = ctx.block_label(object_length_spill_load_idx); + let object_range_label = ctx.block_label(object_range_idx); let object_inline_label = ctx.block_label(object_inline_idx); let object_spill_label = ctx.block_label(object_spill_idx); let object_spill_ptr_label = ctx.block_label(object_spill_ptr_idx); let object_spill_load_label = ctx.block_label(object_spill_load_idx); let object_miss_label = ctx.block_label(object_miss_idx); + let meta_ptr_size: u64 = if crate::target_layout::target_is_ilp32(ctx.target_triple) { + 4 + } else { + 8 + }; + let meta_offset = (crate::target_layout::object_header_size_bytes(ctx.target_triple) + - meta_ptr_size) + .to_string(); // Reject every non-pointer / handle-band / noncanonical-index case before // touching a managed header. The miss helper retains full ToPropertyKey, @@ -364,9 +398,11 @@ pub(super) fn lower_inline_dyn_typed_array_get( ctx.block() .cond_br(&object_entry_ok, &object_header_label, &object_miss_label); - // Exact class + semantic ShapeId identity. The runtime primes only a - // prototype-unmodified dense Array-subclass shape with no relevant - // accessors, and publishes no heap pointer in this cache. + // One validated managed header feeds two tiers: a direct ordinary-Array + // load and the Array-subclass shape/family IC. The old miss path handled + // only the latter, so every unknown-receiver plain Array read immediately + // called the full polymorphic dispatcher despite having all guard inputs + // available here. ctx.current_block = object_header_idx; let object_idx_i64 = ctx.block().fptosi(DOUBLE, idx_d, I64); let object_idx_back = ctx.block().sitofp(I64, &object_idx_i64, DOUBLE); @@ -374,12 +410,96 @@ pub(super) fn lower_inline_dyn_typed_array_get( let gc_type_addr = ctx.block().sub(I64, &object_raw, "8"); let gc_type_ptr = ctx.block().inttoptr(I64, &gc_type_addr); let gc_type = ctx.block().load(I8, &gc_type_ptr); - let is_object = ctx.block().icmp_eq(I8, &gc_type, "2"); + let is_array = ctx.block().icmp_eq(I8, &gc_type, "1"); let gc_flags_addr = ctx.block().sub(I64, &object_raw, "7"); let gc_flags_ptr = ctx.block().inttoptr(I64, &gc_flags_addr); let gc_flags = ctx.block().load(I8, &gc_flags_ptr); - let forwarded = ctx.block().and(I8, &gc_flags, "1"); + let forwarded = ctx.block().and(I8, &gc_flags, "128"); let not_forwarded = ctx.block().icmp_eq(I8, &forwarded, "0"); + let header_ok = ctx.block().and(I1, &object_idx_is_int, ¬_forwarded); + ctx.block() + .cond_br(&header_ok, &object_brand_label, &object_miss_label); + + ctx.current_block = object_brand_idx; + ctx.block() + .cond_br(&is_array, &object_array_guard_label, &object_shape_label); + + // Ordinary Array: the receiver tag and forwarding state were checked in + // the predecessor. Reject descriptors or any process-wide prototype + // invalidation, then prove a dense in-capacity index before loading the + // raw JSValue. A hole is exposed as `undefined`, exactly like the guarded + // statically-Array tier. Every exotic/OOB case retains the unchanged + // boxed dispatcher. + ctx.current_block = object_array_guard_idx; + let array_reserved_addr = ctx.block().sub(I64, &object_raw, "6"); + let array_reserved_ptr = ctx.block().inttoptr(I64, &array_reserved_addr); + let array_reserved = ctx.block().load(I16, &array_reserved_ptr); + let array_descriptor_bits = ctx.block().and(I16, &array_reserved, "1024"); + let array_no_descriptors = ctx.block().icmp_eq(I16, &array_descriptor_bits, "0"); + let array_invalidated = ctx + .block() + .load_volatile(I8, "@PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED"); + let array_default_prototypes = ctx.block().icmp_eq(I8, &array_invalidated, "0"); + let array_ptr = ctx.block().inttoptr(I64, &object_raw); + let array_length = ctx.block().load(I32, &array_ptr); + let array_capacity_addr = ctx.block().add(I64, &object_raw, "4"); + let array_capacity_ptr = ctx.block().inttoptr(I64, &array_capacity_addr); + let array_capacity = ctx.block().load(I32, &array_capacity_ptr); + let array_length_i64 = ctx.block().zext(I32, &array_length, I64); + let array_capacity_i64 = ctx.block().zext(I32, &array_capacity, I64); + let array_index_in_bounds = ctx + .block() + .icmp_ult(I64, &object_idx_i64, &array_length_i64); + let array_length_within_capacity = + ctx.block() + .icmp_ule(I64, &array_length_i64, &array_capacity_i64); + let array_guard_ok = ctx + .block() + .and(I1, &array_no_descriptors, &array_default_prototypes); + let array_guard_ok = ctx.block().and(I1, &array_guard_ok, &array_index_in_bounds); + let array_guard_ok = ctx + .block() + .and(I1, &array_guard_ok, &array_length_within_capacity); + ctx.block().cond_br( + &array_guard_ok, + &object_array_load_label, + &object_miss_label, + ); + + ctx.current_block = object_array_load_idx; + let array_element_word = ctx.block().add(I64, &object_idx_i64, "1"); + let array_element_ptr = + ctx.block() + .gep_inbounds(I64, &array_ptr, &[(I64, &array_element_word)]); + let array_raw = ctx.block().load(DOUBLE, &array_element_ptr); + let array_raw_bits = ctx.block().bitcast_double_to_i64(&array_raw); + let array_is_hole = ctx + .block() + .icmp_eq(I64, &array_raw_bits, crate::nanbox::TAG_HOLE_I64); + let array_undefined = ctx + .block() + .bitcast_i64_to_double(crate::nanbox::TAG_UNDEFINED_I64); + let array_value = ctx + .block() + .select(I1, &array_is_hole, DOUBLE, &array_undefined, &array_raw); + let array_value = if coerce_slow_to_number { + ctx.block() + .call(DOUBLE, "js_number_coerce", &[(DOUBLE, &array_value)]) + } else { + array_value + }; + let array_end_label = ctx.block().label.clone(); + ctx.block().br(&merge_label); + kind_incoming.push((array_value, array_end_label)); + + // The runtime publishes either an exact `(class, ShapeId)` identity or a + // high-bit Array-subclass dense-tail family token. The latter lives in + // ObjectMeta and survives only the exact learned numeric push/pop edges; + // every generic structural or descriptor mutation retires it before the + // mutation is observable. This lets lifecycle-heavy subclasses traverse + // a thousand historical tail shapes without thrashing a monomorphic IC. + ctx.current_block = object_shape_idx; + let is_object = ctx.block().icmp_eq(I8, &gc_type, "2"); let object_ptr = ctx.block().inttoptr(I64, &object_raw); let class_id = ctx.block().load(I32, &object_ptr); let shape_addr = ctx.block().add(I64, &object_raw, "4"); @@ -391,18 +511,59 @@ pub(super) fn lower_inline_dyn_typed_array_get( let live_key = ctx.block().or(I64, &class_high, &shape64); let cached_key_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]); let cached_key = ctx.block().load(I64, &cached_key_ptr); - let key_matches = ctx.block().icmp_eq(I64, &live_key, &cached_key); let key_nonzero = ctx.block().icmp_ne(I64, &cached_key, "0"); - let object_ok = ctx.block().and(I1, &object_idx_is_int, &is_object); - let object_ok = ctx.block().and(I1, &object_ok, ¬_forwarded); - let object_ok = ctx.block().and(I1, &object_ok, &key_matches); - let object_ok = ctx.block().and(I1, &object_ok, &key_nonzero); + let object_ok = ctx.block().and(I1, &is_object, &key_nonzero); + ctx.block() + .cond_br(&object_ok, &object_identity_label, &object_miss_label); + + ctx.current_block = object_identity_idx; + let family_token_bit = crate::nanbox::i64_literal(1u64 << 63); + let family_bits = ctx.block().and(I64, &cached_key, &family_token_bit); + let is_family = ctx.block().icmp_ne(I64, &family_bits, "0"); ctx.block() - .cond_br(&object_ok, &object_bounds_label, &object_miss_label); + .cond_br(&is_family, &object_family_meta_label, &object_exact_label); - // The exact shape proves the cached length slot is live and inline. Check - // its current value and the proved dense prefix on every hit; growing - // `length` without creating properties therefore cannot expose holes. + ctx.current_block = object_exact_idx; + let key_matches = ctx.block().icmp_eq(I64, &live_key, &cached_key); + ctx.block() + .cond_br(&key_matches, &object_bounds_label, &object_miss_label); + + ctx.current_block = object_family_meta_idx; + let family_meta_addr = ctx.block().add(I64, &object_raw, &meta_offset); + let family_meta_slot_ptr = ctx.block().inttoptr(I64, &family_meta_addr); + let family_meta_loaded = ctx.block().load( + if meta_ptr_size == 4 { I32 } else { I64 }, + &family_meta_slot_ptr, + ); + let family_meta_i64 = if meta_ptr_size == 4 { + ctx.block().zext(I32, &family_meta_loaded, I64) + } else { + family_meta_loaded + }; + let family_has_meta = ctx.block().icmp_ne(I64, &family_meta_i64, "0"); + ctx.block().cond_br( + &family_has_meta, + &object_family_token_label, + &object_miss_label, + ); + + ctx.current_block = object_family_token_idx; + let family_meta_ptr = ctx.block().inttoptr(I64, &family_meta_i64); + // repr(C) ObjectMeta word 6 is the move-stable Array-subclass named-prefix + // token. The dense-tail miss helper only publishes it after proving that + // the canonical numeric suffix immediately follows that prefix. + let family_token_ptr = ctx.block().gep(I64, &family_meta_ptr, &[(I64, "6")]); + let live_family_token = ctx.block().load(I64, &family_token_ptr); + let family_matches = ctx.block().icmp_eq(I64, &live_family_token, &cached_key); + ctx.block() + .cond_br(&family_matches, &object_bounds_label, &object_miss_label); + + // The exact shape or family token proves the cached slots. `length` may + // itself be in ObjectMeta::spill (wolf-ecs Archetype has four declared + // fields before Array-subclass init installs it), so split its load just + // like the element load below. Check the live value against the admitted + // dense-prefix high-water mark on every hit; a generic length-only grow + // therefore cannot expose holes through this tier. ctx.current_block = object_bounds_idx; let length_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]); let length_slot = ctx.block().load(I64, &length_slot_ptr); @@ -414,11 +575,83 @@ pub(super) fn lower_inline_dyn_typed_array_get( let inline_bound = ctx.block().load(I64, &inline_bound_ptr); let object_header_size = crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string(); + let length_is_inline = ctx.block().icmp_ult(I64, &length_slot, &inline_bound); + ctx.block().cond_br( + &length_is_inline, + &object_length_inline_label, + &object_length_spill_meta_label, + ); + + ctx.current_block = object_length_inline_idx; let length_bytes = ctx.block().shl(I64, &length_slot, "3"); let length_offset = ctx.block().add(I64, &length_bytes, &object_header_size); let length_addr = ctx.block().add(I64, &object_raw, &length_offset); let length_ptr = ctx.block().inttoptr(I64, &length_addr); - let live_length = ctx.block().load(DOUBLE, &length_ptr); + let inline_length = ctx.block().load(DOUBLE, &length_ptr); + let inline_length_end = ctx.block().label.clone(); + ctx.block().br(&object_range_label); + + ctx.current_block = object_length_spill_meta_idx; + let length_meta_addr = ctx.block().add(I64, &object_raw, &meta_offset); + let length_meta_slot_ptr = ctx.block().inttoptr(I64, &length_meta_addr); + let length_meta_loaded = ctx.block().load( + if meta_ptr_size == 4 { I32 } else { I64 }, + &length_meta_slot_ptr, + ); + let length_meta_i64 = if meta_ptr_size == 4 { + ctx.block().zext(I32, &length_meta_loaded, I64) + } else { + length_meta_loaded + }; + let length_has_meta = ctx.block().icmp_ne(I64, &length_meta_i64, "0"); + ctx.block().cond_br( + &length_has_meta, + &object_length_spill_ptr_label, + &object_miss_label, + ); + + ctx.current_block = object_length_spill_ptr_idx; + let length_meta_ptr = ctx.block().inttoptr(I64, &length_meta_i64); + let length_spill_slot_ptr = ctx.block().gep(I64, &length_meta_ptr, &[(I64, "4")]); + let length_spill_i64 = ctx.block().load(I64, &length_spill_slot_ptr); + let length_has_spill = ctx.block().icmp_ne(I64, &length_spill_i64, "0"); + let safe_length_spill_i64 = ctx.block().select( + I1, + &length_has_spill, + I64, + &length_spill_i64, + &length_meta_i64, + ); + let length_spill_ptr = ctx.block().inttoptr(I64, &safe_length_spill_i64); + let length_spill_len = ctx.block().load(I32, &length_spill_ptr); + let length_spill_len_i64 = ctx.block().zext(I32, &length_spill_len, I64); + let length_in_spill = ctx + .block() + .icmp_ult(I64, &length_slot, &length_spill_len_i64); + let length_spill_ok = ctx.block().and(I1, &length_has_spill, &length_in_spill); + ctx.block().cond_br( + &length_spill_ok, + &object_length_spill_load_label, + &object_miss_label, + ); + + ctx.current_block = object_length_spill_load_idx; + let length_element_word = ctx.block().add(I64, &length_slot, "1"); + let length_element_ptr = + ctx.block() + .gep_inbounds(I64, &length_spill_ptr, &[(I64, &length_element_word)]); + let spilled_length = ctx.block().load(DOUBLE, &length_element_ptr); + let spilled_length_end = ctx.block().label.clone(); + ctx.block().br(&object_range_label); + + ctx.current_block = object_range_idx; + let live_length = ctx.block().phi( + DOUBLE, + &[ + (&inline_length, &inline_length_end), + (&spilled_length, &spilled_length_end), + ], + ); let below_length = ctx.block().fcmp("olt", idx_d, &live_length); let below_prefix = ctx.block().icmp_ult(I64, &object_idx_i64, &dense_prefix); let in_dense_range = ctx.block().and(I1, &below_length, &below_prefix); @@ -454,14 +687,6 @@ pub(super) fn lower_inline_dyn_typed_array_get( // spill Array. Reload both moving pointers from the live receiver; the IC // itself contains only scalar offsets. ctx.current_block = object_spill_idx; - let meta_ptr_size: u64 = if crate::target_layout::target_is_ilp32(ctx.target_triple) { - 4 - } else { - 8 - }; - let meta_offset = (crate::target_layout::object_header_size_bytes(ctx.target_triple) - - meta_ptr_size) - .to_string(); let meta_addr = ctx.block().add(I64, &object_raw, &meta_offset); let meta_slot_ptr = ctx.block().inttoptr(I64, &meta_addr); let meta_loaded = ctx diff --git a/crates/perry-codegen/src/expr/index_get_claim_tests.rs b/crates/perry-codegen/src/expr/index_get_claim_tests.rs index fe25b8dad5..2bbfc20396 100644 --- a/crates/perry-codegen/src/expr/index_get_claim_tests.rs +++ b/crates/perry-codegen/src/expr/index_get_claim_tests.rs @@ -6,10 +6,11 @@ use crate::temp_root_coverage::main_ir_for as ir_for; use perry_hir::types::Type; -use perry_hir::{Expr, Stmt}; +use perry_hir::{BinaryOp, Expr, Stmt}; const ITEMS: u32 = 1; const RESULT: u32 = 2; +const SYMBOL: u32 = 3; fn declared_array_read_ir(name: &str, index: Expr) -> String { ir_for( @@ -68,8 +69,212 @@ fn numeric_key_on_a_declared_array_keeps_the_guarded_array_tier() { ir.contains("arr.guard.deref"), "the numeric receiver-validation tier was not emitted:\n{ir}" ); + assert!( + ir.contains("arr.guard.oob") && ir.contains("9222246136947933185"), + "a structurally-proven OOB ordinary-array read must return the undefined tag inline:\n{ir}" + ); assert!( !ir.contains("aidxkey.sso") && !ir.contains("call double @js_string_index_get_boxed("), "the SSO receiver guard widened onto the numeric array path:\n{ir}" ); } + +#[test] +fn numeric_layout_oob_array_read_returns_undefined_inline() { + let ir = ir_for( + "numeric_layout_oob_array_read", + vec![ + Stmt::Let { + id: ITEMS, + name: "items".to_string(), + ty: Type::Array(Box::new(Type::Number)), + mutable: false, + init: Some(Expr::PropertyGet { + object: Box::new(Expr::Object(vec![( + "value".to_string(), + Expr::Array(vec![]), + )])), + property: "value".to_string(), + byte_offset: 0, + }), + }, + Stmt::Let { + id: RESULT, + name: "result".to_string(), + ty: Type::Number, + mutable: false, + init: Some(Expr::Binary { + op: BinaryOp::Sub, + left: Box::new(Expr::IndexGet { + object: Box::new(Expr::LocalGet(ITEMS)), + index: Box::new(Expr::Integer(7)), + }), + right: Box::new(Expr::Number(1.0)), + }), + }, + ], + ); + assert!( + ir.contains("arr.guard.oob") && ir.contains("9222246136947933185"), + "a numeric-layout OOB read must inline the undefined tag:\n{ir}" + ); + assert!( + ir.contains("arr.guard.numeric_in_bounds"), + "only the in-bounds arm may require the numeric element-layout proof:\n{ir}" + ); +} + +#[test] +fn unknown_numeric_read_guards_dense_subclass_families_and_spilled_length() { + let ir = ir_for( + "unknown_dense_subclass_read", + vec![ + Stmt::Let { + id: ITEMS, + name: "items".to_string(), + ty: Type::Any, + mutable: false, + // Hide the representation behind an ordinary property read + // so scalar replacement cannot fold the indexed access. + init: Some(Expr::PropertyGet { + object: Box::new(Expr::Object(vec![( + "value".to_string(), + Expr::Array(vec![Expr::Number(7.0)]), + )])), + property: "value".to_string(), + byte_offset: 0, + }), + }, + Stmt::Let { + id: RESULT, + name: "result".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::IndexGet { + object: Box::new(Expr::LocalGet(ITEMS)), + index: Box::new(Expr::Integer(0)), + }), + }, + ], + ); + assert!( + ir.contains("arrlike.ic.family_token"), + "the generated IC must compare the move-stable dense-tail family token:\n{ir}" + ); + assert!( + ir.contains("arrlike.ic.array_guard") && ir.contains("arrlike.ic.array_load"), + "an ordinary Array behind the erased receiver must retain a direct guarded load:\n{ir}" + ); + assert!( + ir.contains("arrlike.ic.length_spill_load"), + "an Array-subclass whose length slot spilled must retain an inline IC tier:\n{ir}" + ); + assert!( + ir.contains("arrlike.ic.range") && ir.contains("arrlike.ic.miss"), + "the live length and cached dense-prefix bound must retain a semantic side exit:\n{ir}" + ); +} + +fn dynamic_symbol_access_ir(symbol_init: Expr, field: Option<&str>) -> String { + let symbol_read = Expr::IndexGet { + object: Box::new(Expr::LocalGet(ITEMS)), + index: Box::new(Expr::LocalGet(SYMBOL)), + }; + let result = match field { + Some(property) => Expr::PropertyGet { + object: Box::new(symbol_read), + property: property.to_string(), + byte_offset: 0, + }, + None => symbol_read, + }; + ir_for( + "dynamic_symbol_read", + vec![ + Stmt::Let { + id: ITEMS, + name: "items".to_string(), + ty: Type::Any, + mutable: false, + // Hide the receiver behind a generic read so this exercises + // the erased-receiver IndexGet dispatcher used by wolf-ecs. + init: Some(Expr::PropertyGet { + object: Box::new(Expr::Object(vec![( + "value".to_string(), + Expr::Object(vec![]), + )])), + property: "value".to_string(), + byte_offset: 0, + }), + }, + Stmt::Let { + id: SYMBOL, + name: "componentData".to_string(), + ty: Type::Symbol, + mutable: false, + init: Some(symbol_init), + }, + Stmt::Let { + id: RESULT, + name: "result".to_string(), + ty: Type::Any, + mutable: false, + init: Some(result), + }, + ], + ) +} + +fn dynamic_symbol_read_ir(symbol_init: Expr) -> String { + dynamic_symbol_access_ir(symbol_init, None) +} + +#[test] +fn proven_symbol_key_skips_registry_probe_and_uses_weak_own_property_ic() { + let ir = dynamic_symbol_read_ir(Expr::SymbolNew(None)); + assert!( + ir.contains("symic.hit") + && ir.contains("load atomic i64, ptr @PERRY_SYMBOL_PROPERTY_IC_EPOCH acquire") + && ir.contains("call double @js_object_get_symbol_property_ic_miss("), + "the weak epoch-guarded Symbol property IC was not emitted:\n{ir}" + ); + assert!( + !ir.contains("call i32 @js_is_symbol("), + "compiler-owned Symbol provenance must remove the registry probe:\n{ir}" + ); +} + +#[test] +fn proven_symbol_then_named_field_composes_identity_and_shape_caches() { + let ir = dynamic_symbol_access_ir(Expr::SymbolNew(None), Some("id")); + assert!( + ir.contains("symfield.identity") + && ir.contains("symfield.hit") + && ir.contains("load atomic i64, ptr @PERRY_SYMBOL_PROPERTY_IC_EPOCH acquire") + && ir.contains("call double @js_object_get_symbol_then_field_ic_miss(") + && ir.contains("4611686018427387904"), + "the weak Symbol identity and exact ShapeId field caches were not composed:\n{ir}" + ); + assert!( + ir.contains("and i16") && ir.contains("2048"), + "the composed hit must reject descriptor-bearing metadata objects:\n{ir}" + ); + assert!( + !ir.contains("call i32 @js_is_symbol("), + "compiler-owned Symbol provenance must retain its registry-free route:\n{ir}" + ); +} + +#[test] +fn erased_symbol_annotation_does_not_bypass_runtime_validation() { + let ir = dynamic_symbol_read_ir(Expr::PropertyGet { + object: Box::new(Expr::Object(vec![("value".to_string(), Expr::Number(7.0))])), + property: "value".to_string(), + byte_offset: 0, + }); + assert!( + !ir.contains("symic.hit") + && !ir.contains("call double @js_object_get_symbol_property_ic_miss("), + "a TypeScript Symbol annotation without initializer provenance must not enter the exact-Symbol IC:\n{ir}" + ); +} diff --git a/crates/perry-codegen/src/expr/index_set.rs b/crates/perry-codegen/src/expr/index_set.rs index ec4d963473..deee2e1566 100644 --- a/crates/perry-codegen/src/expr/index_set.rs +++ b/crates/perry-codegen/src/expr/index_set.rs @@ -1159,11 +1159,8 @@ pub(crate) fn lower( let g_ref = format!("@{}", global_name); // GC_STORE_AUDIT(ROOT): module global array slot is a registered mutable GC root. emit_root_nanbox_store_on_block(ctx.block(), &new_box, &g_ref); - // Gen-GC Phase C2: write barrier on array element store. - if write_barrier_needed { - let val_bits = ctx.block().bitcast_double_to_i64(&val_double); - emit_write_barrier(ctx, &arr_bits, &val_bits); - } + // The extending runtime setter barriers the actual + // destination slot on every pointer-bearing store. } else { // Closure-captured array, or local without a // stack slot (rare). Issue #637 followup / hono r2: @@ -1194,11 +1191,8 @@ pub(crate) fn lower( (DOUBLE, &val_double), ], ); - // Gen-GC Phase C2: write barrier on array element store. - if write_barrier_needed { - let val_bits = ctx.block().bitcast_double_to_i64(&val_double); - emit_write_barrier(ctx, &arr_bits, &val_bits); - } + // The extending runtime setter barriers the actual + // destination slot on every pointer-bearing store. } } else { let idx_i32 = { @@ -1241,11 +1235,7 @@ pub(crate) fn lower( (DOUBLE, &val_double), ], ); - if write_barrier_needed { - let val_bits = - ctx.block().bitcast_double_to_i64(&val_double); - emit_write_barrier(ctx, &arr_bits, &val_bits); - } + // The helper owns the precise slot barrier. Ok(()) }, )?; @@ -1275,11 +1265,9 @@ pub(crate) fn lower( (DOUBLE, &val_double), ], ); - // Gen-GC Phase C2: write barrier on array element store. - if write_barrier_needed { - let val_bits = ctx.block().bitcast_double_to_i64(&val_double); - emit_write_barrier(ctx, &arr_bits, &val_bits); - } + // The extending runtime setter owns the precise slot + // barrier; a second opaque parent/child barrier here + // would repeat both pointer decodes. } // The group is released after `body` returns, never before: // every branch above ends in a helper that can itself allocate diff --git a/crates/perry-codegen/src/expr/index_set_barrier_tests.rs b/crates/perry-codegen/src/expr/index_set_barrier_tests.rs index e895997678..920d96e0e1 100644 --- a/crates/perry-codegen/src/expr/index_set_barrier_tests.rs +++ b/crates/perry-codegen/src/expr/index_set_barrier_tests.rs @@ -377,3 +377,36 @@ fn the_gated_element_store_still_reaches_the_barrier_call() { exist — the remembered set's arming protocol reads this:\n{ir}" ); } + +/// The guarded diamond has two mutually exclusive store owners: +/// +/// - its inline arm writes the slot itself and must emit the precise slot +/// barrier asserted above; +/// - its slow arm calls `js_typed_feedback_array_set_f64_extend`, whose every +/// successful pointer-bearing destination is barriered inside the runtime. +/// +/// Re-adding a bare parent/child barrier after the helper is correct but +/// duplicates receiver/child decoding on every slow pointer overwrite. This +/// pins that ownership boundary while also proving the fixture reaches both +/// tiers rather than passing because it stopped emitting an array setter. +#[test] +fn runtime_array_setter_is_not_followed_by_a_duplicate_opaque_barrier() { + assert_default_barrier_env_not_disabled(); + let ir = ir(); + assert!( + ir.contains("call i64 @js_typed_feedback_array_set_f64_extend("), + "fixture no longer reaches the runtime array-set fallback:\n{ir}" + ); + assert!( + !ir.contains("call void @js_write_barrier("), + "the runtime array setter already barriers its destination slot; the \ + generated opaque wrapper repeats that work:\n{ir}" + ); + let barrier_body = numbered_barrier_block_body(&ir) + .unwrap_or_else(|| panic!("inline store lost `{BARRIER_BLOCK}`:\n{ir}")); + assert!( + barrier_body.contains(BARRIER_CALL), + "removing the slow helper's duplicate barrier must not remove the \ + inline store's precise slot barrier:\n{barrier_body}" + ); +} diff --git a/crates/perry-codegen/src/expr/logical_collections.rs b/crates/perry-codegen/src/expr/logical_collections.rs index 1ba49da27a..0f7e0bd312 100644 --- a/crates/perry-codegen/src/expr/logical_collections.rs +++ b/crates/perry-codegen/src/expr/logical_collections.rs @@ -813,10 +813,8 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // a NaN-tagged TAG_TRUE/TAG_FALSE so console.log prints // "true"/"false" via the runtime's NaN-tag dispatch. Expr::BooleanCoerce(operand) => { - let v = lower_expr(ctx, operand)?; + let (_v, bit) = crate::lower_conditional::lower_expr_with_truthy(ctx, operand)?; let blk = ctx.block(); - let i32_v = blk.call(I32, "js_is_truthy", &[(DOUBLE, &v)]); - let bit = blk.icmp_ne(I32, &i32_v, "0"); let tagged = blk.select( crate::types::I1, &bit, diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 2cc7baf037..e3ad275e8b 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -32,6 +32,8 @@ use crate::types::{DOUBLE, F32, I1, I16, I32, I64, I8, PTR}; // remain here. `pub(crate) use` keeps the public surface stable so // existing `crate::expr::X` paths resolve unchanged. mod array_literal; +mod bitset_test; +pub(crate) use bitset_test::is_u32_bitset_test; mod buffer_access; mod buffer_views; mod channel; @@ -224,6 +226,20 @@ pub(crate) struct InlineCtorReturn { pub is_derived: bool, } +/// One statement-region-owned property IC shared by equivalent reads. +/// +/// The owner emits a speculative, side-effect-free cache probe before the +/// original statements, then leaves the original generic property reads in +/// place as the semantic fallback. Those reads must prime the *same* cache or +/// the speculative probe would remain cold forever. Sharing is safe only for +/// the exact `(base local, static property name)` pair recorded here. +#[derive(Clone)] +pub(crate) struct PropertyGetIcOverride { + pub base_local_id: u32, + pub property: String, + pub cache_name: String, +} + /// Per-function codegen context. Held briefly during lowering, never stored. /// #8122: where an inline-`new` site gets its `<2 x i64>` header image from. #[derive(Clone, Debug)] @@ -315,6 +331,18 @@ pub(crate) struct FnCtx<'a> { /// reading the field, because they consult it *after* lowering their /// operands, by which point the field has been taken again. pub discard_this_expr: bool, + /// A condition consumer is lowering a call and can consume an `i1` + /// truthiness result directly. Guarded user-method dispatch uses this to + /// keep the statically-resolved arm's constructively-Boolean result native + /// while applying full `js_is_truthy` semantics to the dynamic override + /// arm. The ordinary JSValue result remains available for every other use. + pub truthy_call_result_requested: bool, + /// `(canonical boxed result, native truthiness)` published by the + /// outermost call lowering that honored `truthy_call_result_requested`. + /// The consumer compares the boxed SSA name with the expression result, + /// so a nested argument/receiver call can never be mistaken for the call + /// whose truthiness was requested. + pub pending_truthy_call_result: Option<(String, String)>, /// HIR FuncId → LLVM function name. Resolved at the top of /// `compile_module` so `FuncRef(id)` calls know what to emit. pub func_names: &'a std::collections::HashMap, @@ -1458,6 +1486,11 @@ pub(crate) struct FnCtx<'a> { /// global [2 x i64] zeroinitializer` for each entry. pub ic_globals: Vec, + /// Region-scoped cache selected by a guarded statement fusion. Generic + /// property reads matching the exact base local and key reuse it instead + /// of allocating independent per-expression caches. + pub property_get_ic_override: Option, + /// Issue #179 typed-parse: raw rodata globals emitted by /// `JsonParseTyped` codegen. Each entry is the full LLVM IR line /// `@ = private unnamed_addr constant [N x i8] c"..."` to @@ -2486,8 +2519,11 @@ mod index_get_claim_tests; mod masked_window; #[cfg(test)] mod null_default_numeric_add_tests; + mod ptr_numarray_access; mod ta_param_f64_read; +#[cfg(test)] +mod unary_bitnot_tests; pub(crate) use index_get::{ numeric_index_has_integer_array_index_proof, packed_f64_loop_index_parts, }; @@ -2976,16 +3012,8 @@ pub(crate) fn lower_i32_control_store_value(ctx: &mut FnCtx<'_>, value: &Expr) - } pub(crate) fn lower_i1_control_store_value(ctx: &mut FnCtx<'_>, value: &Expr) -> Result { - if let Some(lowered) = lower_expr_value(ctx, value)? { - if matches!(lowered.rep, NativeRep::I1) { - return Ok(lowered.value); - } - let boxed = materialize_js_value(ctx, lowered, MaterializationReason::RuntimeApi); - let truthy = crate::lower_conditional::lower_truthy(ctx, &boxed, value); - return Ok(truthy); - } - let boxed = lower_expr(ctx, value)?; - Ok(crate::lower_conditional::lower_truthy(ctx, &boxed, value)) + let (_boxed, truthy) = crate::lower_conditional::lower_expr_with_truthy(ctx, value)?; + Ok(truthy) } fn lower_async_i32_control_const_compare( diff --git a/crates/perry-codegen/src/expr/property_get.rs b/crates/perry-codegen/src/expr/property_get.rs index 8229d2f0c4..7fe08a8f3c 100644 --- a/crates/perry-codegen/src/expr/property_get.rs +++ b/crates/perry-codegen/src/expr/property_get.rs @@ -31,12 +31,13 @@ use crate::nanbox::{double_literal, POINTER_MASK_I64}; use crate::native_value::{ BoundsState, BufferAccessMode, LoweredValue, MaterializationReason, NativeRep, SemanticKind, }; +use crate::rooting; use crate::type_analysis::{ is_array_expr, is_map_expr, is_numeric_typed_array_class, is_set_expr, is_string_expr, is_url_search_params_expr, is_url_search_params_subclass_expr, receiver_class_name, receiver_is_error_type, }; -use crate::types::{DOUBLE, I1, I32, I64, I8, PTR}; +use crate::types::{DOUBLE, I1, I16, I32, I64, I8, PTR}; use super::property_get_names::{ is_headers_method_name, is_http_agent_method_name, is_http_client_request_method_name, @@ -65,6 +66,148 @@ use super::{ TypedFeedbackContract, TypedFeedbackKind, }; +/// Fuse `base[provenSymbol].field` into one weak identity/epoch guard followed +/// by one exact ShapeId guard and direct slot load. +/// +/// The ordinary Symbol IC already proves that its cached intermediate value is +/// the current own Symbol data property and invalidates on every Symbol write +/// or completed GC. A second ordinary property PIC currently throws that fact +/// away and repeats receiver tag, GC-header, descriptor, ShapeId, and dispatch +/// classification. This composed site keeps two normal cache records: the +/// Symbol identity/value cache and an ordinary property cache primed by the +/// shared runtime miss handler. A hit still reloads the named field's current +/// bits; it never caches the final value, so `metadata.id = next` is observed +/// without an epoch bump. +fn lower_symbol_then_named_property_ic( + ctx: &mut FnCtx<'_>, + base: &Expr, + symbol: &Expr, + property: &str, + byte_offset: u32, +) -> Result { + rooting::with_operands_rooted(ctx, &[base, symbol], |ctx, values| { + let base_box = ctx.block().call( + DOUBLE, + "js_require_object_coercible", + &[(DOUBLE, values[0].as_str())], + ); + let symbol_box = values[1].clone(); + crate::expr::calls::emit_call_location_at(ctx, byte_offset); + let feedback_site_id = emit_typed_feedback_register_site( + ctx, + TypedFeedbackKind::PropertyGet, + property, + TypedFeedbackContract::object_get_by_name(), + ); + + let symbol_site = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let symbol_cache = super::inline_cache_global_name(ctx, symbol_site); + ctx.ic_globals.push(symbol_cache.clone()); + let symbol_cache = format!("@{symbol_cache}"); + + let field_site = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let field_cache = super::inline_cache_global_name(ctx, field_site); + ctx.ic_globals.push(field_cache.clone()); + let field_cache = format!("@{field_cache}"); + + let identity_idx = ctx.new_block("symfield.identity"); + let hit_idx = ctx.new_block("symfield.hit"); + let miss_idx = ctx.new_block("symfield.miss"); + let merge_idx = ctx.new_block("symfield.merge"); + let identity_label = ctx.block_label(identity_idx); + let hit_label = ctx.block_label(hit_idx); + let miss_label = ctx.block_label(miss_idx); + let merge_label = ctx.block_label(merge_idx); + + let epoch = ctx + .block() + .load_atomic_acquire(I64, "@PERRY_SYMBOL_PROPERTY_IC_EPOCH", 8); + let cached_epoch_ptr = ctx.block().gep(I64, &symbol_cache, &[(I64, "0")]); + let cached_epoch = ctx.block().load_atomic_acquire(I64, &cached_epoch_ptr, 8); + let epoch_matches = ctx.block().icmp_eq(I64, &epoch, &cached_epoch); + let base_bits = ctx.block().bitcast_double_to_i64(&base_box); + let cached_base_ptr = ctx.block().gep(I64, &symbol_cache, &[(I64, "1")]); + let cached_base = ctx.block().load(I64, &cached_base_ptr); + let base_matches = ctx.block().icmp_eq(I64, &base_bits, &cached_base); + let symbol_bits = ctx.block().bitcast_double_to_i64(&symbol_box); + let cached_symbol_ptr = ctx.block().gep(I64, &symbol_cache, &[(I64, "2")]); + let cached_symbol = ctx.block().load(I64, &cached_symbol_ptr); + let symbol_matches = ctx.block().icmp_eq(I64, &symbol_bits, &cached_symbol); + let identity_matches = ctx.block().and(I1, &base_matches, &symbol_matches); + let identity_matches = ctx.block().and(I1, &epoch_matches, &identity_matches); + ctx.block() + .cond_br(&identity_matches, &identity_label, &miss_label); + + // The epoch/identity edge is what makes dereferencing cache[3] safe: + // any collection that could relocate this weak value changes the epoch + // first. Named-property mutations do not, so independently validate + // the intermediate object's live ShapeId and descriptor latch. + ctx.current_block = identity_idx; + let intermediate_ptr = ctx.block().gep(I64, &symbol_cache, &[(I64, "3")]); + let intermediate_bits = ctx.block().load(I64, &intermediate_ptr); + let intermediate_handle = ctx.block().and(I64, &intermediate_bits, POINTER_MASK_I64); + let descriptor_addr = ctx.block().sub(I64, &intermediate_handle, "6"); + let descriptor_ptr = ctx.block().inttoptr(I64, &descriptor_addr); + let gc_flags = ctx.block().load(I16, &descriptor_ptr); + let descriptor_bits = ctx.block().and(I16, &gc_flags, "2048"); + let data_only = ctx.block().icmp_eq(I16, &descriptor_bits, "0"); + let shape_addr = ctx.block().add(I64, &intermediate_handle, "4"); + let shape_ptr = ctx.block().inttoptr(I64, &shape_addr); + let shape_id = ctx.block().load(I32, &shape_ptr); + let shape_token = ctx.block().zext(I32, &shape_id, I64); + let shape_token = ctx.block().or(I64, &shape_token, "4611686018427387904"); + let cached_token_ptr = ctx.block().gep(I64, &field_cache, &[(I64, "0")]); + let cached_token = ctx.block().load(I64, &cached_token_ptr); + let shape_matches = ctx.block().icmp_eq(I64, &shape_token, &cached_token); + let hit = ctx.block().and(I1, &data_only, &shape_matches); + ctx.block().cond_br(&hit, &hit_label, &miss_label); + + ctx.current_block = hit_idx; + crate::expr::emit_typed_feedback_record_call( + ctx.block(), + "js_typed_feedback_record_guard_pass", + &[(I64, &feedback_site_id)], + ); + let cached_slot_ptr = ctx.block().gep(I64, &field_cache, &[(I64, "1")]); + let cached_slot = ctx.block().load(I64, &cached_slot_ptr); + let slot_bytes = ctx.block().shl(I64, &cached_slot, "3"); + let fields_base = ctx.block().add(I64, &intermediate_handle, "16"); + let field_addr = ctx.block().add(I64, &fields_base, &slot_bytes); + let field_ptr = ctx.block().inttoptr(I64, &field_addr); + let hit_value = ctx.block().load(DOUBLE, &field_ptr); + let hit_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = miss_idx; + let key_index = ctx.strings.intern(property); + let key_global = format!("@{}", ctx.strings.entry(key_index).handle_global); + let key_box = ctx.block().load(DOUBLE, &key_global); + let key_bits = ctx.block().bitcast_double_to_i64(&key_box); + let key_handle = ctx.block().and(I64, &key_bits, POINTER_MASK_I64); + let miss_value = ctx.block().call( + DOUBLE, + "js_object_get_symbol_then_field_ic_miss", + &[ + (DOUBLE, &base_box), + (DOUBLE, &symbol_box), + (I64, &key_handle), + (I64, &feedback_site_id), + (PTR, &symbol_cache), + (PTR, &field_cache), + ], + ); + let miss_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + Ok(ctx + .block() + .phi(DOUBLE, &[(&hit_value, &hit_end), (&miss_value, &miss_end)])) + }) +} + /// A declared class may nominate the guarded field/method route, but never a /// raw load by itself. Every field consumer below checks the live receiver's /// class id and keys token before dereferencing; method-value/runtime-member @@ -79,6 +222,225 @@ fn guarded_declared_class_get_candidate(ctx: &FnCtx<'_>, object: &Expr) -> Optio ctx.classes.contains_key(name).then(|| name.clone()) } +/// Emit the object-backed Array-subclass tier for a guarded `.length` miss. +/// +/// Cache words are scalar facts only: exact `(class, ShapeId)` or the stable +/// named-prefix token, the `length` slot, and the live inline-slot bound. A +/// hit reloads every object/meta/spill pointer from the current receiver, so a +/// moving collection never needs to visit the per-site global. +fn emit_array_subclass_length_ic( + ctx: &mut FnCtx<'_>, + recv_box: &str, + recv_bits: &str, + recv_handle: &str, + outer_merge_label: &str, +) -> (String, String) { + let site_id = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let cache_name = super::inline_cache_global_name(ctx, site_id); + ctx.ic_globals.push(cache_name.clone()); + let cache_ref = format!("@{cache_name}"); + + let header_idx = ctx.new_block("plen.ic.header"); + let shape_idx = ctx.new_block("plen.ic.shape"); + let identity_idx = ctx.new_block("plen.ic.identity"); + let exact_idx = ctx.new_block("plen.ic.exact"); + let family_meta_idx = ctx.new_block("plen.ic.family_meta"); + let family_token_idx = ctx.new_block("plen.ic.family_token"); + let slot_idx = ctx.new_block("plen.ic.slot"); + let inline_idx = ctx.new_block("plen.ic.inline"); + let spill_meta_idx = ctx.new_block("plen.ic.spill_meta"); + let spill_ptr_idx = ctx.new_block("plen.ic.spill_ptr"); + let spill_load_idx = ctx.new_block("plen.ic.spill_load"); + let miss_idx = ctx.new_block("plen.ic.miss"); + let merge_idx = ctx.new_block("plen.ic.merge"); + let header_label = ctx.block_label(header_idx); + let shape_label = ctx.block_label(shape_idx); + let identity_label = ctx.block_label(identity_idx); + let exact_label = ctx.block_label(exact_idx); + let family_meta_label = ctx.block_label(family_meta_idx); + let family_token_label = ctx.block_label(family_token_idx); + let slot_label = ctx.block_label(slot_idx); + let inline_label = ctx.block_label(inline_idx); + let spill_meta_label = ctx.block_label(spill_meta_idx); + let spill_ptr_label = ctx.block_label(spill_ptr_idx); + let spill_load_label = ctx.block_label(spill_load_idx); + let miss_label = ctx.block_label(miss_idx); + let merge_label = ctx.block_label(merge_idx); + + // This block is reachable for every failed ordinary Array/String guard, + // including primitives and native handle ids. Validate the exact pointer + // tag and target heap window before reading a managed header. + let recv_top16 = ctx.block().lshr(I64, recv_bits, "48"); + let pointer_tag = ctx + .block() + .icmp_eq(I64, &recv_top16, crate::nanbox::POINTER_TAG_TOP16_I64); + let heap_floor = + crate::target_layout::heap_addr_lower_bound_inclusive(ctx.target_triple).to_string(); + let heap_ceiling = + crate::target_layout::heap_addr_upper_bound_exclusive(ctx.target_triple).to_string(); + let above_floor = ctx.block().icmp_uge(I64, recv_handle, &heap_floor); + let below_ceiling = ctx.block().icmp_ult(I64, recv_handle, &heap_ceiling); + let in_heap = ctx.block().and(I1, &pointer_tag, &above_floor); + let in_heap = ctx.block().and(I1, &in_heap, &below_ceiling); + ctx.block().cond_br(&in_heap, &header_label, &miss_label); + + ctx.current_block = header_idx; + let gc_type_addr = ctx.block().sub(I64, recv_handle, "8"); + let gc_type_ptr = ctx.block().inttoptr(I64, &gc_type_addr); + let gc_type = ctx.block().load(I8, &gc_type_ptr); + let is_object = ctx.block().icmp_eq(I8, &gc_type, "2"); + let gc_flags_addr = ctx.block().sub(I64, recv_handle, "7"); + let gc_flags_ptr = ctx.block().inttoptr(I64, &gc_flags_addr); + let gc_flags = ctx.block().load(I8, &gc_flags_ptr); + let forwarded = ctx.block().and(I8, &gc_flags, "128"); + let not_forwarded = ctx.block().icmp_eq(I8, &forwarded, "0"); + let header_ok = ctx.block().and(I1, &is_object, ¬_forwarded); + ctx.block().cond_br(&header_ok, &shape_label, &miss_label); + + ctx.current_block = shape_idx; + let object_ptr = ctx.block().inttoptr(I64, recv_handle); + let class_id = ctx.block().load(I32, &object_ptr); + let shape_addr = ctx.block().add(I64, recv_handle, "4"); + let shape_ptr = ctx.block().inttoptr(I64, &shape_addr); + let shape_id = ctx.block().load(I32, &shape_ptr); + let class64 = ctx.block().zext(I32, &class_id, I64); + let shape64 = ctx.block().zext(I32, &shape_id, I64); + let class_high = ctx.block().shl(I64, &class64, "32"); + let live_key = ctx.block().or(I64, &class_high, &shape64); + let cached_key_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]); + let cached_key = ctx.block().load(I64, &cached_key_ptr); + let key_nonzero = ctx.block().icmp_ne(I64, &cached_key, "0"); + ctx.block() + .cond_br(&key_nonzero, &identity_label, &miss_label); + + ctx.current_block = identity_idx; + let family_token_bit = crate::nanbox::i64_literal(1u64 << 63); + let family_bits = ctx.block().and(I64, &cached_key, &family_token_bit); + let is_family = ctx.block().icmp_ne(I64, &family_bits, "0"); + ctx.block() + .cond_br(&is_family, &family_meta_label, &exact_label); + + ctx.current_block = exact_idx; + let exact_match = ctx.block().icmp_eq(I64, &live_key, &cached_key); + ctx.block().cond_br(&exact_match, &slot_label, &miss_label); + + let meta_ptr_size: u64 = if crate::target_layout::target_is_ilp32(ctx.target_triple) { + 4 + } else { + 8 + }; + let meta_offset = (crate::target_layout::object_header_size_bytes(ctx.target_triple) + - meta_ptr_size) + .to_string(); + + ctx.current_block = family_meta_idx; + let family_meta_addr = ctx.block().add(I64, recv_handle, &meta_offset); + let family_meta_slot_ptr = ctx.block().inttoptr(I64, &family_meta_addr); + let family_meta_loaded = ctx.block().load( + if meta_ptr_size == 4 { I32 } else { I64 }, + &family_meta_slot_ptr, + ); + let family_meta_i64 = if meta_ptr_size == 4 { + ctx.block().zext(I32, &family_meta_loaded, I64) + } else { + family_meta_loaded + }; + let family_has_meta = ctx.block().icmp_ne(I64, &family_meta_i64, "0"); + ctx.block() + .cond_br(&family_has_meta, &family_token_label, &miss_label); + + ctx.current_block = family_token_idx; + let family_meta_ptr = ctx.block().inttoptr(I64, &family_meta_i64); + let family_token_ptr = ctx.block().gep(I64, &family_meta_ptr, &[(I64, "6")]); + let live_family_token = ctx.block().load(I64, &family_token_ptr); + let family_match = ctx.block().icmp_eq(I64, &live_family_token, &cached_key); + ctx.block().cond_br(&family_match, &slot_label, &miss_label); + + ctx.current_block = slot_idx; + let length_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]); + let length_slot = ctx.block().load(I64, &length_slot_ptr); + let inline_bound_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "2")]); + let inline_bound = ctx.block().load(I64, &inline_bound_ptr); + let length_is_inline = ctx.block().icmp_ult(I64, &length_slot, &inline_bound); + ctx.block() + .cond_br(&length_is_inline, &inline_label, &spill_meta_label); + + ctx.current_block = inline_idx; + let object_header_size = + crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string(); + let length_bytes = ctx.block().shl(I64, &length_slot, "3"); + let length_offset = ctx.block().add(I64, &length_bytes, &object_header_size); + let length_addr = ctx.block().add(I64, recv_handle, &length_offset); + let length_ptr = ctx.block().inttoptr(I64, &length_addr); + let inline_length = ctx.block().load(DOUBLE, &length_ptr); + let inline_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = spill_meta_idx; + let spill_meta_addr = ctx.block().add(I64, recv_handle, &meta_offset); + let spill_meta_slot_ptr = ctx.block().inttoptr(I64, &spill_meta_addr); + let spill_meta_loaded = ctx.block().load( + if meta_ptr_size == 4 { I32 } else { I64 }, + &spill_meta_slot_ptr, + ); + let spill_meta_i64 = if meta_ptr_size == 4 { + ctx.block().zext(I32, &spill_meta_loaded, I64) + } else { + spill_meta_loaded + }; + let has_meta = ctx.block().icmp_ne(I64, &spill_meta_i64, "0"); + ctx.block() + .cond_br(&has_meta, &spill_ptr_label, &miss_label); + + ctx.current_block = spill_ptr_idx; + let spill_meta_ptr = ctx.block().inttoptr(I64, &spill_meta_i64); + let spill_slot_ptr = ctx.block().gep(I64, &spill_meta_ptr, &[(I64, "4")]); + let spill_i64 = ctx.block().load(I64, &spill_slot_ptr); + let has_spill = ctx.block().icmp_ne(I64, &spill_i64, "0"); + let safe_spill_i64 = ctx + .block() + .select(I1, &has_spill, I64, &spill_i64, &spill_meta_i64); + let spill_ptr = ctx.block().inttoptr(I64, &safe_spill_i64); + let spill_len = ctx.block().load(I32, &spill_ptr); + let spill_len_i64 = ctx.block().zext(I32, &spill_len, I64); + let length_in_spill = ctx.block().icmp_ult(I64, &length_slot, &spill_len_i64); + let spill_ok = ctx.block().and(I1, &has_spill, &length_in_spill); + ctx.block() + .cond_br(&spill_ok, &spill_load_label, &miss_label); + + ctx.current_block = spill_load_idx; + let spill_element_word = ctx.block().add(I64, &length_slot, "1"); + let spill_element_ptr = + ctx.block() + .gep_inbounds(I64, &spill_ptr, &[(I64, &spill_element_word)]); + let spilled_length = ctx.block().load(DOUBLE, &spill_element_ptr); + let spill_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = miss_idx; + let miss_length = ctx.block().call( + DOUBLE, + "js_value_length_property_ic_f64", + &[(DOUBLE, recv_box), (PTR, &cache_ref)], + ); + let miss_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + let length = ctx.block().phi( + DOUBLE, + &[ + (&inline_length, &inline_end), + (&spilled_length, &spill_end), + (&miss_length, &miss_end), + ], + ); + let end = ctx.block().label.clone(); + ctx.block().br(outer_merge_label); + (length, end) +} + pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // #7219: reading `.buffer` on a tracked typed-array view HANDS OUT ITS // STORAGE, so the local's inline-storage proof stops holding from here on. @@ -103,9 +465,26 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // to be recorded where the alias is created rather than where it is used. // `MutableAlias` is exactly what this is. if let Expr::PropertyGet { - object, property, .. + object, + property, + byte_offset, } = expr { + if let Expr::IndexGet { + object: base, + index: symbol, + } = object.as_ref() + { + if super::compare::is_proven_symbol_expr(ctx, symbol) { + return lower_symbol_then_named_property_ic( + ctx, + base, + symbol, + property, + *byte_offset, + ); + } + } if property == "buffer" { if let Expr::LocalGet(id) = object.as_ref() { if ctx.buffer_view_slots.contains_key(id) { @@ -580,13 +959,13 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // a missing property, preserves a non-numeric property value, and // throws for a nullish receiver. ctx.current_block = slow_idx; - let slow_len = ctx.block().call( - DOUBLE, - "js_value_length_property_f64", - &[(DOUBLE, &recv_box)], + let (slow_len, slow_pred_label) = emit_array_subclass_length_ic( + ctx, + &recv_box, + &recv_bits, + &recv_handle, + &merge_label, ); - let slow_pred_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); ctx.current_block = merge_idx; Ok(ctx.block().phi( diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 28bdc9c7f4..f38994896c 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -33,6 +33,10 @@ pub(crate) const PIC_WAYS: usize = 4; /// are worth running; `0` (fresh) and a negative megamorphic countdown /// both skip them. Mirrors the runtime's `PIC_WAY_STATE`. pub(crate) const PIC_WAY_STATE: usize = 3; +/// Optional Array-subclass class-declared named-prefix token. A nonzero value +/// proves the cached slot survives exact numeric-tail ShapeId transitions. +/// Mirrors runtime `PicCache` word 2. +pub(crate) const PIC_NAMED_PREFIX_TOKEN: usize = 2; /// Materialise the pooled property-key `StringHeader*` in the CURRENT block. /// @@ -55,6 +59,28 @@ fn emit_key_handle(ctx: &mut FnCtx<'_>, key_handle_global: &str) -> String { blk.and(I64, &key_bits, POINTER_MASK_I64) } +fn overridden_cache_name(ctx: &FnCtx<'_>, object: &Expr, property: &str) -> Option { + let Expr::LocalGet(base_local_id) = object else { + return None; + }; + ctx.property_get_ic_override + .as_ref() + .filter(|shared| { + shared.base_local_id == *base_local_id && shared.property.as_str() == property + }) + .map(|shared| shared.cache_name.clone()) +} + +fn allocate_property_cache(ctx: &mut FnCtx<'_>) -> String { + let cache_site = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let cache_name = super::super::inline_cache_global_name(ctx, cache_site); + ctx.pending_declares + .push((format!("__ic_decl_{cache_site}"), DOUBLE, vec![])); + ctx.ic_globals.push(cache_name.clone()); + cache_name +} + /// The generic per-site monomorphic inline-cache dispatch for `obj.property`. /// This is the fall-through tail of the general catch-all arm: all earlier /// specializations have been ruled out. @@ -108,12 +134,8 @@ pub(crate) fn lower_generic_property_get( // Per-site monomorphic IC cache, allocated identically to the inline path // (below) so the helper's `js_object_get_field_ic_miss` cache-priming is // unchanged. - let cache_site = ctx.ic_site_counter; - ctx.ic_site_counter += 1; - let cache_name = super::super::inline_cache_global_name(ctx, cache_site); - ctx.pending_declares - .push((format!("__ic_decl_{}", cache_site), DOUBLE, vec![])); - ctx.ic_globals.push(cache_name.clone()); + let cache_name = overridden_cache_name(ctx, object, property) + .unwrap_or_else(|| allocate_property_cache(ctx)); let cache_ref = format!("@{}", cache_name); let key_handle = emit_key_handle(ctx, &key_handle_global); let val = ctx.block().call( @@ -281,18 +303,15 @@ pub(crate) fn lower_generic_property_get( } // Monomorphic inline cache. The per-site global holds an authoritative - // ShapeId token and its cached slot; word 2 is non-identity scratch. + // ShapeId token and its cached slot; word 2 optionally carries the proved + // Array-subclass named-prefix family token. // The fast path compares the receiver's discriminated ShapeId token to // cache[0] and, on match, loads // the field directly at obj+ObjectHeader::SIZE+slot*8: no function call, no hash, // no linear scan. On miss, calls the slow helper which does the // full lookup and primes the cache for next time. - let site_id = ctx.ic_site_counter; - ctx.ic_site_counter += 1; - let cache_name = super::super::inline_cache_global_name(ctx, site_id); - ctx.pending_declares - .push((format!("__ic_decl_{}", site_id), DOUBLE, vec![])); - ctx.ic_globals.push(cache_name.clone()); + let cache_name = overridden_cache_name(ctx, object, property) + .unwrap_or_else(|| allocate_property_cache(ctx)); // Issue #72: validate the receiver is actually a GC_TYPE_OBJECT // before reading its ShapeId. The receiver @@ -335,6 +354,15 @@ pub(crate) fn lower_generic_property_get( // through phis (`false`/`0` on the early-exit edges, which is exactly // what the flat predicate computed there). let hit_idx = ctx.new_block("pic.hit"); + let prefix_guard_idx = ctx.new_block("pic.prefix.guard"); + let prefix_meta_idx = ctx.new_block("pic.prefix.meta"); + let prefix_token_idx = ctx.new_block("pic.prefix.token"); + let prefix_hit_idx = ctx.new_block("pic.prefix.hit"); + let desc_classify_idx = ctx.new_block("pic.desc.classify"); + let desc_prefix_guard_idx = ctx.new_block("pic.desc.prefix.guard"); + let desc_prefix_meta_idx = ctx.new_block("pic.desc.prefix.meta"); + let desc_prefix_token_idx = ctx.new_block("pic.desc.prefix.token"); + let desc_prefix_hit_idx = ctx.new_block("pic.desc.prefix.hit"); let miss_idx = ctx.new_block("pic.miss"); // #7907: the two receiver-validation failures get their own landing block // so `pic.miss` is dominated by `pic.token`. See the comment on @@ -343,6 +371,15 @@ pub(crate) fn lower_generic_property_get( let call_idx = ctx.new_block("pic.miss.call"); let merge_idx = ctx.new_block("pic.merge"); let hit_label = ctx.block_label(hit_idx); + let prefix_guard_label = ctx.block_label(prefix_guard_idx); + let prefix_meta_label = ctx.block_label(prefix_meta_idx); + let prefix_token_label = ctx.block_label(prefix_token_idx); + let prefix_hit_label = ctx.block_label(prefix_hit_idx); + let desc_classify_label = ctx.block_label(desc_classify_idx); + let desc_prefix_guard_label = ctx.block_label(desc_prefix_guard_idx); + let desc_prefix_meta_label = ctx.block_label(desc_prefix_meta_idx); + let desc_prefix_token_label = ctx.block_label(desc_prefix_token_idx); + let desc_prefix_hit_label = ctx.block_label(desc_prefix_hit_idx); let miss_label = ctx.block_label(miss_idx); let cold_label = ctx.block_label(cold_idx); let call_label = ctx.block_label(call_idx); @@ -370,7 +407,7 @@ pub(crate) fn lower_generic_property_get( let gc_type_addr = ctx.block().sub(I64, &obj_handle, "8"); let gc_type_ptr = ctx.block().inttoptr(I64, &gc_type_addr); let gc_type = ctx.block().load(I8, &gc_type_ptr); - let is_object = ctx.block().icmp_eq(I8, &gc_type, "2"); + let is_object_kind = ctx.block().icmp_eq(I8, &gc_type, "2"); // Closures and RegExp values have distinct GC kinds. Every // `GC_TYPE_OBJECT` payload is therefore an ObjectHeader and its ShapeId is @@ -393,7 +430,7 @@ pub(crate) fn lower_generic_property_get( let reserved = ctx.block().load(crate::types::I16, &reserved_ptr); let has_desc = ctx.block().and(crate::types::I16, &reserved, "2048"); // OBJ_FLAG_HAS_DESCRIPTORS (0x800) let no_desc = ctx.block().icmp_eq(crate::types::I16, &has_desc, "0"); - let is_object = ctx.block().and(I1, &is_object, &no_desc); + let is_plain_object = ctx.block().and(I1, &is_object_kind, &no_desc); // #7883: first exit. The header predicates above are kept as one flat // `and` on purpose — they are loads from the same cache line and LLVM @@ -408,7 +445,16 @@ pub(crate) fn lower_generic_property_get( // `way_hit` by construction, so consulting the ways for it was always dead // work, and keeping it out is what lets `pic.miss` reuse this block's // values instead of re-deriving them. - ctx.block().cond_br(&is_object, &tok_label, &cold_label); + ctx.block() + .cond_br(&is_plain_object, &tok_label, &desc_classify_label); + + // A descriptor-bearing GC_TYPE_OBJECT normally goes cold. Array-subclass + // `length` is the important exception: runtime can prove that descriptor + // is unrelated to all class-declared named fields and arm word 2. Keep + // this classification off the ordinary descriptor-free hit path. + ctx.current_block = desc_classify_idx; + ctx.block() + .cond_br(&is_object_kind, &desc_prefix_guard_label, &cold_label); ctx.current_block = tok_idx; // The receiver token is derived solely from its authoritative ShapeId. @@ -463,7 +509,7 @@ pub(crate) fn lower_generic_property_get( let token_eq = ctx.block().icmp_eq(I64, &token, &cached_token); let hit = ctx.block().and(I1, &token_eq, &token_nonnull); - ctx.block().cond_br(&hit, &hit_label, &miss_label); + ctx.block().cond_br(&hit, &hit_label, &prefix_guard_label); // `js_object_get_field_ic_miss` primes only slots below the descriptor's // exact `live_inline_slot_count`. ShapeIds are never reused, so an exact @@ -490,6 +536,148 @@ pub(crate) fn lower_generic_property_get( let hit_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); + // An object-backed Array subclass changes exact ShapeId on every numeric + // push/pop because its elements live in ordinary object slots. Its class- + // declared named prefix does not move. Runtime miss handling proves the + // complete registered prefix plus dense numeric suffix once and publishes + // a nonzero token in cache word 2 and ObjectMeta. Generic structural or + // descriptor transitions clear the object token; only the exact learned + // numeric-tail installer preserves it. + // + // Keep the ordinary miss path cheap: test the cache word first. Every + // non-Array-subclass site reads zero and leaves without touching the + // receiver's meta pointer. + ctx.current_block = prefix_guard_idx; + let cached_prefix_ptr = ctx.block().gep( + I64, + &cache_ref, + &[(I64, &PIC_NAMED_PREFIX_TOKEN.to_string())], + ); + let cached_prefix = ctx.block().load(I64, &cached_prefix_ptr); + let prefix_armed = ctx.block().icmp_ne(I64, &cached_prefix, "0"); + ctx.block() + .cond_br(&prefix_armed, &prefix_meta_label, &miss_label); + + // ObjectHeader::meta is the final header field: offset 8 on LP64, 12 on + // ILP32. Load it with the target pointer width, then branch before reading + // ObjectMeta so a null metadata pointer remains harmless. + ctx.current_block = prefix_meta_idx; + let meta_ptr_size: u64 = if crate::target_layout::target_is_ilp32(ctx.target_triple) { + 4 + } else { + 8 + }; + let meta_offset = (crate::target_layout::object_header_size_bytes(ctx.target_triple) + - meta_ptr_size) + .to_string(); + let meta_addr = ctx.block().add(I64, &obj_handle, &meta_offset); + let meta_slot = ctx.block().inttoptr(I64, &meta_addr); + let meta_load_ty = if meta_ptr_size == 4 { I32 } else { I64 }; + let meta_raw = ctx.block().load(meta_load_ty, &meta_slot); + let meta = if meta_ptr_size == 4 { + ctx.block().zext(I32, &meta_raw, I64) + } else { + meta_raw + }; + let meta_nonnull = ctx.block().icmp_ne(I64, &meta, "0"); + ctx.block() + .cond_br(&meta_nonnull, &prefix_token_label, &miss_label); + + ctx.current_block = prefix_token_idx; + let meta_ptr = ctx.block().inttoptr(I64, &meta); + // repr(C) ObjectMeta word 6. The first six u64 words are prototype, + // descriptor blooms, flags, spill, and private brand. Runtime has an + // offset assertion paired with the IR test below. + let object_prefix_ptr = ctx.block().gep(I64, &meta_ptr, &[(I64, "6")]); + let object_prefix = ctx.block().load(I64, &object_prefix_ptr); + let prefix_match = ctx.block().icmp_eq(I64, &object_prefix, &cached_prefix); + ctx.block() + .cond_br(&prefix_match, &prefix_hit_label, &miss_label); + + ctx.current_block = prefix_hit_idx; + // The exact ShapeId guard did fail, so preserve typed-feedback accounting + // just like a polymorphic-way hit: the site remains structurally + // polymorphic even though no runtime fallback call is needed. + crate::expr::emit_typed_feedback_record_call( + ctx.block(), + "js_typed_feedback_record_guard_fail", + &[(I64, &feedback_site_id)], + ); + crate::expr::emit_typed_feedback_record_call( + ctx.block(), + "js_typed_feedback_record_fallback_call", + &[(I64, &feedback_site_id)], + ); + let prefix_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]); + let prefix_slot = ctx.block().load(I64, &prefix_slot_ptr); + let prefix_offset = ctx.block().shl(I64, &prefix_slot, "3"); + let prefix_base = ctx.block().add(I64, &obj_handle, &obj_header_size); + let prefix_field_addr = ctx.block().add(I64, &prefix_base, &prefix_offset); + let prefix_field_ptr = ctx.block().inttoptr(I64, &prefix_field_addr); + let val_prefix = ctx.block().load(DOUBLE, &prefix_field_ptr); + let prefix_end_label = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + // Descriptor-bearing Array subclasses reach this duplicate of the family + // guard without ever entering `pic.token`: the exact raw-load PIC remains + // forbidden, but a runtime-proved data-only declared prefix is still safe. + // Every failure goes to the cold handler because the shape token values + // required by `pic.miss` do not dominate this path. + ctx.current_block = desc_prefix_guard_idx; + let desc_cached_prefix_ptr = ctx.block().gep( + I64, + &cache_ref, + &[(I64, &PIC_NAMED_PREFIX_TOKEN.to_string())], + ); + let desc_cached_prefix = ctx.block().load(I64, &desc_cached_prefix_ptr); + let desc_prefix_armed = ctx.block().icmp_ne(I64, &desc_cached_prefix, "0"); + ctx.block() + .cond_br(&desc_prefix_armed, &desc_prefix_meta_label, &cold_label); + + ctx.current_block = desc_prefix_meta_idx; + let desc_meta_addr = ctx.block().add(I64, &obj_handle, &meta_offset); + let desc_meta_slot = ctx.block().inttoptr(I64, &desc_meta_addr); + let desc_meta_raw = ctx.block().load(meta_load_ty, &desc_meta_slot); + let desc_meta = if meta_ptr_size == 4 { + ctx.block().zext(I32, &desc_meta_raw, I64) + } else { + desc_meta_raw + }; + let desc_meta_nonnull = ctx.block().icmp_ne(I64, &desc_meta, "0"); + ctx.block() + .cond_br(&desc_meta_nonnull, &desc_prefix_token_label, &cold_label); + + ctx.current_block = desc_prefix_token_idx; + let desc_meta_ptr = ctx.block().inttoptr(I64, &desc_meta); + let desc_object_prefix_ptr = ctx.block().gep(I64, &desc_meta_ptr, &[(I64, "6")]); + let desc_object_prefix = ctx.block().load(I64, &desc_object_prefix_ptr); + let desc_prefix_match = ctx + .block() + .icmp_eq(I64, &desc_object_prefix, &desc_cached_prefix); + ctx.block() + .cond_br(&desc_prefix_match, &desc_prefix_hit_label, &cold_label); + + ctx.current_block = desc_prefix_hit_idx; + crate::expr::emit_typed_feedback_record_call( + ctx.block(), + "js_typed_feedback_record_guard_fail", + &[(I64, &feedback_site_id)], + ); + crate::expr::emit_typed_feedback_record_call( + ctx.block(), + "js_typed_feedback_record_fallback_call", + &[(I64, &feedback_site_id)], + ); + let desc_prefix_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]); + let desc_prefix_slot = ctx.block().load(I64, &desc_prefix_slot_ptr); + let desc_prefix_offset = ctx.block().shl(I64, &desc_prefix_slot, "3"); + let desc_prefix_base = ctx.block().add(I64, &obj_handle, &obj_header_size); + let desc_prefix_field_addr = ctx.block().add(I64, &desc_prefix_base, &desc_prefix_offset); + let desc_prefix_field_ptr = ctx.block().inttoptr(I64, &desc_prefix_field_addr); + let val_desc_prefix = ctx.block().load(DOUBLE, &desc_prefix_field_ptr); + let desc_prefix_end_label = ctx.block().label.clone(); + ctx.block().br(&merge_label); + // PIC miss on the MRU entry — before paying for the call, try the // polymorphic ways (#7753). // @@ -671,6 +859,8 @@ pub(crate) fn lower_generic_property_get( DOUBLE, &[ (&val_hit, &hit_end_label), + (&val_prefix, &prefix_end_label), + (&val_desc_prefix, &desc_prefix_end_label), (&val_way, &way_end_label), (&val_miss, &miss_end_label), ], diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 8b4fc7e6d7..ed4cd7de06 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -103,6 +103,29 @@ fn emit(debug: bool, source: Option<&str>) -> String { .expect("LLVM IR should be UTF-8") } +fn emit_guarded_length_read() -> String { + let mut module = Module::new("guarded_length_read.ts"); + module.init = vec![ + Stmt::Let { + id: 11, + name: "values".to_string(), + ty: perry_hir::types::Type::Array(Box::new(perry_hir::types::Type::Any)), + mutable: false, + // An uninitialized erased annotation can still hold any runtime + // value once control reaches this site. It also prevents scalar + // replacement from folding the length to a literal. + init: None, + }, + Stmt::Return(Some(Expr::PropertyGet { + object: Box::new(Expr::LocalGet(11)), + property: "length".to_string(), + byte_offset: 0, + })), + ]; + String::from_utf8(compile_module(&module, ir_opts(false, None)).unwrap()) + .expect("LLVM IR should be UTF-8") +} + #[test] fn property_read_emits_call_location_under_debug_symbols() { let ir = emit(true, Some(SRC)); @@ -125,9 +148,9 @@ fn no_call_location_without_debug_symbols() { ); } -/// #8067: property-read PIC identity is the authoritative ShapeId only. The -/// former keys-pointer epoch word is reserved scratch and must not participate -/// in the emitted hit predicate. +/// #8067: the primary property-read PIC identity is the authoritative ShapeId +/// only. Word 2 may carry the independent Array-subclass named-prefix proof, +/// but it is consulted only after this exact ShapeId predicate fails. #[test] fn generic_property_get_hit_path_is_shape_id_only() { let ir = emit(false, None); @@ -140,15 +163,33 @@ fn generic_property_get_hit_path_is_shape_id_only() { "hit path must form a discriminated ShapeId token:\n{ir}" ); assert!( - !ir.contains("@PERRY_IC_EPOCH") - && !ir.lines().any(|line| { - line.contains("getelementptr i64, ptr @perry_ic_") - && line.trim_end().ends_with(", i64 2") - }), + !ir.contains("@PERRY_IC_EPOCH"), "the removed pointer-token epoch must not appear in emitted IR:\n{ir}" ); } +#[test] +fn guarded_length_read_emits_array_subclass_scalar_ic() { + let ir = emit_guarded_length_read(); + for block in [ + "plen.ic.header", + "plen.ic.identity", + "plen.ic.family_token", + "plen.ic.inline", + "plen.ic.spill_load", + ] { + assert!(ir.contains(block), "missing {block} from length IC:\n{ir}"); + } + assert!( + ir.contains("call double @js_value_length_property_ic_f64"), + "the cold arm must prime the scalar cache while retaining property semantics:\n{ir}" + ); + assert!( + ir.contains("getelementptr i64") && ir.contains("i64 6\n"), + "the family hit must validate ObjectMeta's named-prefix token:\n{ir}" + ); +} + #[test] fn fs_parent_promises_property_installs_before_resolution() { let mut module = Module::new("fs_parent_promises_property.ts"); @@ -185,7 +226,9 @@ fn fs_parent_promises_property_installs_before_resolution() { /// itself so the runtime's copy cannot drift. #[test] fn pic_cache_layout_matches_runtime() { - use crate::expr::property_get::generic_dispatch::{PIC_CACHE_WORDS, PIC_WAYS, PIC_WAY_BASE}; + use crate::expr::property_get::generic_dispatch::{ + PIC_CACHE_WORDS, PIC_NAMED_PREFIX_TOKEN, PIC_WAYS, PIC_WAY_BASE, + }; assert_eq!( PIC_CACHE_WORDS, 12, "perry-runtime's PIC_CACHE_WORDS is 12; update both sides together" @@ -195,6 +238,10 @@ fn pic_cache_layout_matches_runtime() { PIC_CACHE_WORDS, "the ways must fill the emitted global exactly" ); + assert_eq!( + PIC_NAMED_PREFIX_TOKEN, 2, + "runtime PicCache word 2 carries the Array-subclass named-prefix token" + ); let ir = emit(false, None); assert!( ir.contains(&format!( @@ -204,6 +251,43 @@ fn pic_cache_layout_matches_runtime() { ); } +/// Object-backed Array subclasses mint one ShapeId per numeric tail length. +/// A named-field site on such a receiver must try the independently proved +/// class prefix before falling into the bounded shape ways / runtime miss. +#[test] +fn generic_property_get_emits_array_subclass_named_prefix_guard() { + use crate::expr::property_get::generic_dispatch::PIC_NAMED_PREFIX_TOKEN; + + let ir = emit(false, None); + let guard = ir + .find("\npic.prefix.guard") + .unwrap_or_else(|| panic!("expected a named-prefix guard block:\n{ir}")); + let token = ir + .find("\npic.prefix.token") + .unwrap_or_else(|| panic!("expected a named-prefix token block:\n{ir}")); + let hit = ir + .find("\npic.prefix.hit") + .unwrap_or_else(|| panic!("expected a named-prefix hit block:\n{ir}")); + let miss = ir + .find("\npic.miss") + .unwrap_or_else(|| panic!("expected the ordinary PIC miss block:\n{ir}")); + assert!( + guard < token && token < hit && hit < miss, + "prefix guard must precede the ordinary miss path:\n{ir}" + ); + + let guard_body = &ir[guard..token]; + assert!( + guard_body.contains(&format!("i64 {PIC_NAMED_PREFIX_TOKEN}\n")), + "the cheap first gate must read cache word 2 before touching ObjectMeta:\n{guard_body}" + ); + let token_body = &ir[token..hit]; + assert!( + token_body.contains("getelementptr i64") && token_body.contains("i64 6\n"), + "ObjectMeta word 6 must carry the runtime-paired prefix token:\n{token_body}" + ); +} + /// #7753: the polymorphic ways must be consulted BEFORE the miss call, and the /// monomorphic path must not have grown any work. /// diff --git a/crates/perry-codegen/src/expr/unary.rs b/crates/perry-codegen/src/expr/unary.rs index 01f15e13eb..4bd640d99b 100644 --- a/crates/perry-codegen/src/expr/unary.rs +++ b/crates/perry-codegen/src/expr/unary.rs @@ -7,25 +7,34 @@ use anyhow::Result; use perry_hir::{Expr, UnaryOp}; -use crate::lower_conditional::lower_truthy; +use crate::lower_conditional::lower_expr_with_truthy; use crate::type_analysis::{ expr_may_return_boxed_value_from_raw_f64_fallback, is_bigint_expr, is_numeric_expr, + is_provably_not_bigint, }; -use crate::types::{DOUBLE, I64}; +use crate::types::{DOUBLE, I32, I64}; -use super::{lower_expr, FnCtx}; +use super::{is_known_i32_range, lower_expr, FnCtx}; pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { match expr { Expr::Unary { op, operand } => { let numeric = is_numeric_expr(ctx, operand) && !expr_may_return_boxed_value_from_raw_f64_fallback(ctx, operand); + let native_bitnot = + matches!(op, UnaryOp::BitNot) && numeric && is_provably_not_bigint(ctx, operand); + let bitnot_known_i32 = native_bitnot && is_known_i32_range(ctx, operand); // `-` must stay a BigInt (`typeof -1n === "bigint"`). // `fneg` on a NaN-boxed BigInt flips the NaN payload's sign bit // and produces a garbage number, so route negation through the // runtime dynamic helper when the operand is statically bigint. let is_big = matches!(op, UnaryOp::Neg) && is_bigint_expr(ctx, operand); - let v = lower_expr(ctx, operand)?; + let (v, precomputed_truthy) = if matches!(op, UnaryOp::Not) { + let (boxed, truthy) = lower_expr_with_truthy(ctx, operand)?; + (boxed, Some(truthy)) + } else { + (lower_expr(ctx, operand)?, None) + }; let blk = ctx.block(); match op { UnaryOp::Neg => { @@ -49,7 +58,8 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // !x: truthiness inverted, then NaN-box as a JS // boolean (TAG_TRUE / TAG_FALSE) so console.log // prints "true" / "false" instead of 1 / 0. - let bit = lower_truthy(ctx, &v, operand); + let bit = + precomputed_truthy.expect("UnaryOp::Not precomputes operand truthiness"); let blk = ctx.block(); let inv = blk.xor(crate::types::I1, &bit, "true"); let tagged_i64 = blk.select( @@ -62,9 +72,24 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { Ok(blk.bitcast_i64_to_double(&tagged_i64)) } UnaryOp::BitNot => { - // `~x` preserves BigInt when the runtime value is a BigInt - // and otherwise falls back to JS ToInt32 semantics. - Ok(blk.call(DOUBLE, "js_dynamic_bitnot", &[(DOUBLE, &v)])) + // A proven Number result can perform ToInt32 and `~` + // directly. This notably covers coercive arithmetic such + // as `~~(erased / 32)`: the division either throws for a + // mixed BigInt or returns a Number, so both bitwise-NOTs + // are native. An erased direct operand and a potentially + // BigInt-producing chain (`~(a & b)`) retain the dynamic + // helper, which is what preserves BigInt semantics. + if native_bitnot { + let i = if bitnot_known_i32 { + blk.toint32_fast(&v) + } else { + blk.toint32_wrap(&v) + }; + let flipped = blk.xor(I32, &i, "-1"); + Ok(blk.sitofp(I32, &flipped, DOUBLE)) + } else { + Ok(blk.call(DOUBLE, "js_dynamic_bitnot", &[(DOUBLE, &v)])) + } } } } diff --git a/crates/perry-codegen/src/expr/unary_bitnot_tests.rs b/crates/perry-codegen/src/expr/unary_bitnot_tests.rs new file mode 100644 index 0000000000..01c9dcda8e --- /dev/null +++ b/crates/perry-codegen/src/expr/unary_bitnot_tests.rs @@ -0,0 +1,98 @@ +//! Runtime-proof boundary for native unary bitwise-NOT lowering. + +use perry_hir::types::Type; +use perry_hir::{BinaryOp, Expr, Stmt, UnaryOp}; + +use crate::temp_root_coverage::main_ir_for as ir_for; + +const X: u32 = 1; +const Y: u32 = 2; +const RESULT: u32 = 3; + +fn erased(id: u32, name: &str) -> Stmt { + Stmt::Let { + id, + name: name.to_string(), + ty: Type::Any, + mutable: true, + init: Some(Expr::Undefined), + } +} + +fn result(expr: Expr) -> Stmt { + Stmt::Let { + id: RESULT, + name: "result".to_string(), + ty: Type::Any, + mutable: false, + init: Some(expr), + } +} + +fn bitnot(operand: Expr) -> Expr { + Expr::Unary { + op: UnaryOp::BitNot, + operand: Box::new(operand), + } +} + +#[test] +fn double_bitnot_of_coercive_number_result_stays_native() { + let quotient = Expr::Binary { + op: BinaryOp::Div, + left: Box::new(Expr::LocalGet(X)), + right: Box::new(Expr::Integer(32)), + }; + let ir = ir_for( + "native_double_bitnot", + vec![erased(X, "x"), result(bitnot(bitnot(quotient)))], + ); + + assert!( + ir.contains("call double @js_dynamic_div("), + "the erased division must retain ToNumeric and mixed-BigInt behavior:\n{ir}" + ); + assert!( + !ir.contains("call double @js_dynamic_bitnot("), + "a successfully returned division result is necessarily a Number:\n{ir}" + ); + assert!( + ir.matches("xor i32").count() >= 2, + "both bitwise-NOT operators should lower natively:\n{ir}" + ); +} + +#[test] +fn erased_direct_bitnot_retains_bigint_dispatch() { + let ir = ir_for( + "dynamic_erased_bitnot", + vec![erased(X, "x"), result(bitnot(Expr::LocalGet(X)))], + ); + assert!( + ir.contains("call double @js_dynamic_bitnot("), + "an erased operand can be a BigInt and must preserve its tag:\n{ir}" + ); +} + +#[test] +fn potentially_bigint_binary_result_retains_bitnot_dispatch() { + let unknown_value = |id| Expr::PropertyGet { + byte_offset: 0, + object: Box::new(Expr::LocalGet(id)), + property: "value".to_string(), + }; + let dynamic_and = Expr::Binary { + op: BinaryOp::BitAnd, + left: Box::new(unknown_value(X)), + right: Box::new(unknown_value(Y)), + }; + let ir = ir_for( + "dynamic_nested_bigint_bitnot", + vec![erased(X, "x"), erased(Y, "y"), result(bitnot(dynamic_and))], + ); + assert!( + ir.contains("call double @js_dynamic_bitand(") + && ir.contains("call double @js_dynamic_bitnot("), + "a both-erased bitwise chain may produce BigInt and must stay dynamic:\n{ir}" + ); +} diff --git a/crates/perry-codegen/src/expr/write_barrier.rs b/crates/perry-codegen/src/expr/write_barrier.rs index b84c06b582..c740a24356 100644 --- a/crates/perry-codegen/src/expr/write_barrier.rs +++ b/crates/perry-codegen/src/expr/write_barrier.rs @@ -57,10 +57,10 @@ pub(crate) fn emit_write_barrier(ctx: &mut FnCtx<'_>, parent_bits: &str, child_b // touches the incremental-mark latch, the parent decode or the remembered // set — so for every numeric store this call does nothing but cost a call. // - // An array element store pays it unconditionally today: `this.vals[i] = v` - // in `gc-handoff/apps/pipeline.ts`'s `Registry` emits - // `js_typed_feedback_array_set_f64_extend` immediately followed by a bare - // `js_write_barrier`, on a `number[]`. + // Runtime array setters now own a precise destination-slot barrier and do + // not use this opaque compatibility wrapper. Other opaque property-store + // helpers still reach it when their internal destination is unavailable to + // generated code. // // `emit_may_carry_heap_pointer_check` is a deliberate SUPERSET of the // runtime predicate (its doc records why the direction is load-bearing, @@ -347,9 +347,10 @@ pub(crate) fn emit_layout_note_slot_on_block( ); } -/// Scalar-aware layout note: passes the slot's previous value (`old_bits`) so +/// Value-aware layout note: passes the slot's previous value (`old_bits`) so /// the runtime can skip the thread-local layout hashmap when the store does not -/// change the slot's pointer-ness (scalar-over-scalar). See +/// change the slot's pointer-ness (scalar-over-scalar or pointer-over-pointer). +/// Array element-shape bookkeeping still runs for the latter. See /// `js_gc_note_slot_layout_aware`. pub(crate) fn emit_layout_note_slot_aware_on_block( blk: &mut LlBlock, @@ -487,10 +488,12 @@ pub(crate) fn emit_jsvalue_slot_store_with_value_bits_on_block( /// As [`emit_jsvalue_slot_store_on_block`], but for an **in-place element /// overwrite** of a slot that already holds a valid value: routes the layout /// note through `js_gc_note_slot_layout_aware`, which loads the previous slot -/// value and skips the thread-local layout hashmap when neither old nor new is -/// a heap pointer. Use only where the slot is guaranteed initialized (array -/// `arr[i] = …` overwrites), not for fresh-slot appends/literals or object -/// field writes (which are POINTER_FREE-dominated and only pay the extra load). +/// value and skips the thread-local layout hashmap when old and new have the +/// same heap-pointer classification. Array element-shape bookkeeping still +/// runs for pointer-over-pointer. Use only where the slot is guaranteed +/// initialized (array `arr[i] = …` overwrites), not for fresh-slot +/// appends/literals or object field writes (which are POINTER_FREE-dominated +/// and only pay the extra load). /// This is the dominant per-write cost on downgraded `any[]` numeric loops /// (#5094) and gives ~9× on `bench_numeric_array_downgrade` without regressing /// `bench_object_property`. @@ -820,9 +823,10 @@ fn emit_jsvalue_slot_store_on_block_inner( .unwrap_or_else(|| blk.bitcast_double_to_i64(value_double)); if layout_note_needed { match old_bits.as_deref() { - // Scalar-over-scalar stores leave the GC slot layout unchanged — the - // aware note skips the thread-local layout hashmap when neither the - // new nor the old value is a heap pointer (#5094). + // Same-classification overwrites leave the GC slot mask unchanged. + // The aware note skips the general layout pipeline; its runtime + // pointer-over-pointer arm still maintains Array element-shape + // metadata (#5094). Some(old) => emit_layout_note_slot_aware_on_block( blk, layout_parent_bits, diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index ed0c3b99e9..bed89926ff 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -230,6 +230,7 @@ pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { | "js_object_alloc_class_inline_keys" | "js_object_alloc_class_inline_keys_stamped" | "js_array_push_f64" + | "js_array_push_u31_with_length" | "js_array_length" | "js_array_slice_values" // Second audit round (2026-08-01): ctor-return semantics check @@ -846,6 +847,7 @@ mod tests { for name in [ "js_closure_alloc_singleton", "js_array_push_f64", + "js_array_push_u31_with_length", "js_ctor_return_override", "js_array_indexOf_jsvalue", "js_validate_array_comparator", @@ -862,6 +864,7 @@ mod tests { for name in [ "js_value_length_f64", "js_value_length_property_f64", + "js_value_length_property_ic_f64", "js_array_get_f64", ] { assert_eq!( diff --git a/crates/perry-codegen/src/lower_call/method_override.rs b/crates/perry-codegen/src/lower_call/method_override.rs index 533fbeebfb..d2e7eaa31c 100644 --- a/crates/perry-codegen/src/lower_call/method_override.rs +++ b/crates/perry-codegen/src/lower_call/method_override.rs @@ -29,6 +29,177 @@ const GC_OBJECT_METHOD_GUARD_MASK_I32: &str = "142639359"; // 0x0880_80ff const SHAPE_ID_BASE_NEG_I32: &str = "-2147483648"; // subtract 0x8000_0000 const SHAPE_ID_RANGE_LEN: &str = "1073741824"; // 0x4000_0000 +/// A deliberately small constructive Boolean proof for method returns. +/// +/// Source annotations are erased and therefore cannot license a native result. +/// These expression forms, however, produce a Boolean for every JavaScript +/// input. Keeping this proof local to the guarded direct-call site also means a +/// dynamic own/prototype override remains completely unconstrained. +fn expr_constructs_boolean(expr: &perry_hir::Expr) -> bool { + use perry_hir::{Expr, UnaryOp}; + match expr { + Expr::Bool(_) + | Expr::Compare { .. } + | Expr::BooleanCoerce(_) + | Expr::IsFinite(_) + | Expr::IsNaN(_) + | Expr::NumberIsNaN(_) + | Expr::NumberIsFinite(_) + | Expr::NumberIsInteger(_) + | Expr::IsUndefinedOrBareNan(_) + | Expr::SetHas { .. } + | Expr::SetDelete { .. } + | Expr::MapHas { .. } + | Expr::MapDelete { .. } + | Expr::ArrayIncludes { .. } => true, + Expr::Unary { + op: UnaryOp::Not, .. + } => true, + Expr::Logical { left, right, .. } => { + expr_constructs_boolean(left) && expr_constructs_boolean(right) + } + Expr::Conditional { + then_expr, + else_expr, + .. + } => expr_constructs_boolean(then_expr) && expr_constructs_boolean(else_expr), + _ => false, + } +} + +/// `(all encountered returns are Boolean, every normal path exits)` for the +/// conservative straight-line/if subset used by hot predicate methods. +/// Unsupported control flow rejects the proof instead of trying to infer it. +fn block_constructively_returns_boolean(stmts: &[perry_hir::Stmt]) -> (bool, bool) { + use perry_hir::Stmt; + for stmt in stmts { + match stmt { + Stmt::Return(Some(expr)) => return (expr_constructs_boolean(expr), true), + Stmt::Return(None) => return (false, true), + Stmt::Throw(_) => return (true, true), + Stmt::If { + then_branch, + else_branch, + .. + } => { + let (then_ok, then_exits) = block_constructively_returns_boolean(then_branch); + let (else_ok, else_exits) = else_branch + .as_deref() + .map(block_constructively_returns_boolean) + .unwrap_or((true, false)); + if !then_ok || !else_ok { + return (false, false); + } + if then_exits && else_exits { + return (true, true); + } + } + // Neither form can hide a statement-level return. + Stmt::Let { .. } | Stmt::Expr(_) => {} + // Loops, try/finally, switches and labels need a real CFG proof. + // Refuse them here; the runtime truthiness path remains exact. + _ => return (false, false), + } + } + (true, false) +} + +pub(super) fn direct_method_constructively_returns_boolean( + ctx: &FnCtx<'_>, + direct_fn: &str, +) -> bool { + ctx.classes.iter().any(|(class_name, class)| { + class.methods.iter().any(|method| { + !method.is_async + && !method.is_generator + && ctx + .methods + .get(&(class_name.to_string(), method.name.clone())) + .is_some_and(|name| name == direct_fn) + && block_constructively_returns_boolean(&method.body) == (true, true) + }) + }) +} + +pub(super) fn canonical_boolean_truthy(ctx: &mut FnCtx<'_>, value: &str) -> String { + let bits = ctx.block().bitcast_double_to_i64(value); + ctx.block().icmp_eq(I64, &bits, crate::nanbox::TAG_TRUE_I64) +} + +#[derive(Clone, Copy)] +enum ConstructiveMethodTruthiness { + CanonicalBoolean, + RawNumber, +} + +/// The representation-independent truthiness contract of a statically +/// resolved method body. +/// +/// The Number case is intentionally narrower than the Boolean proof. It is +/// licensed only when the complete source body is the canonical ECS bitset +/// return. That expression either returns a Number or throws for every input; +/// selecting its native non-negative-index clone is a separate lowering +/// decision. Erased return annotations never participate in either proof. +fn constructive_method_truthiness( + ctx: &FnCtx<'_>, + direct_fn: &str, +) -> Option { + if direct_method_constructively_returns_boolean(ctx, direct_fn) { + return Some(ConstructiveMethodTruthiness::CanonicalBoolean); + } + ctx.classes.iter().find_map(|(class_name, class)| { + class.methods.iter().find_map(|method| { + let is_target = !method.is_async + && !method.is_generator + && ctx + .methods + .get(&(class_name.to_string(), method.name.clone())) + .is_some_and(|name| name == direct_fn); + let [perry_hir::Stmt::Return(Some(expr))] = method.body.as_slice() else { + return None; + }; + (is_target && crate::expr::is_u32_bitset_test(expr)) + .then_some(ConstructiveMethodTruthiness::RawNumber) + }) + }) +} + +fn constructive_truthy( + ctx: &mut FnCtx<'_>, + kind: ConstructiveMethodTruthiness, + value: &str, +) -> String { + match kind { + ConstructiveMethodTruthiness::CanonicalBoolean => canonical_boolean_truthy(ctx, value), + // `fcmp one` exactly matches Number truthiness: both signed zeroes and + // NaN are false; every other finite or infinite Number is true. + ConstructiveMethodTruthiness::RawNumber => ctx.block().fcmp("one", value, "0.0"), + } +} + +/// Publish a native truthiness result for a call site that has already proved +/// exact method identity (for example Phase 3b's containment route). Unlike +/// the guarded diamond, no arbitrary fallback arm exists here. +pub(super) fn publish_constructive_method_truthy( + ctx: &mut FnCtx<'_>, + direct_fn: &str, + boxed: &str, +) { + if let Some(kind) = ctx + .truthy_call_result_requested + .then(|| constructive_method_truthiness(ctx, direct_fn)) + .flatten() + { + let truthy = constructive_truthy(ctx, kind, boxed); + ctx.pending_truthy_call_result = Some((boxed.to_string(), truthy)); + } +} + +fn total_value_truthy(ctx: &mut FnCtx<'_>, value: &str) -> String { + let raw = ctx.block().call(I32, "js_is_truthy", &[(DOUBLE, value)]); + ctx.block().icmp_ne(I32, &raw, "0") +} + /// Emit the single-arm equivalent of `js_method_direct_shape_guard` directly /// into the generated module. The guard remains dynamic at every call site: /// arbitrary callback code may replace a prototype method or mutate the @@ -594,6 +765,10 @@ pub(super) fn emit_guarded_direct_method_call( shape_only_guard: bool, subclass_arms: &[SubclassDispatchArm], ) -> Option { + let truthy_result_kind = ctx + .truthy_call_result_requested + .then(|| constructive_method_truthiness(ctx, direct_fn)) + .flatten(); let expected_class_id = *ctx.class_ids.get(receiver_class_name)?; let keys_global_name = ctx.class_keys_globals.get(receiver_class_name)?.clone(); // Only the shape-only guard is widened. The typed-feedback guard records an @@ -625,6 +800,11 @@ pub(super) fn emit_guarded_direct_method_call( .pshape_methods .contains_key(&(receiver_class_name.to_string(), property.to_string()))) .then(|| crate::collectors::pshape_method_name(direct_fn)); + let pshape_index_fn = nonnegative_index_direct_fn.and_then(|index_fn| { + let pshape = pshape_fn.as_ref()?; + let index_suffix = index_fn.strip_prefix(direct_fn)?; + Some(format!("{pshape}{index_suffix}")) + }); // The body a failed typed guard falls back to. Arm-invariant (both inputs // are), so it is resolved once here rather than five times below. @@ -1315,7 +1495,9 @@ pub(super) fn emit_guarded_direct_method_call( // `perry_static_` exclusion and the declaring-class argument are // written out) is the same clone the typed arms above now route // their generic fallbacks to. - let target = nonnegative_index_direct_fn + let target = pshape_index_fn + .as_deref() + .or(nonnegative_index_direct_fn) .or(pshape_fn.as_deref()) .unwrap_or(direct_fn); let result = ctx.block().call(DOUBLE, target, direct_arg_slices); @@ -1354,6 +1536,8 @@ pub(super) fn emit_guarded_direct_method_call( } } }; + let fast_truthy = + truthy_result_kind.map(|kind| constructive_truthy(ctx, kind, fast_value.as_str())); let after_fast = ctx.block().label.clone(); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); @@ -1364,13 +1548,24 @@ pub(super) fn emit_guarded_direct_method_call( // proof the declared-class arm rests on, so the statically resolved body // is the one the dispatch tower would have found. let mut sub_values: Vec<(String, String)> = Vec::with_capacity(subclass_arms.len()); + let mut sub_truthy_values: Vec<(String, String)> = Vec::with_capacity(subclass_arms.len()); for (i, arm) in subclass_arms.iter().enumerate() { ctx.current_block = sub_case_idxs[i]; let value = ctx.block().call(DOUBLE, &arm.target_fn, direct_arg_slices); + let truthy = truthy_result_kind.map(|_| { + if let Some(kind) = constructive_method_truthiness(ctx, &arm.target_fn) { + constructive_truthy(ctx, kind, &value) + } else { + total_value_truthy(ctx, &value) + } + }); let after = ctx.block().label.clone(); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); } + if let Some(truthy) = truthy { + sub_truthy_values.push((truthy, after.clone())); + } sub_values.push((value, after)); } @@ -1411,6 +1606,7 @@ pub(super) fn emit_guarded_direct_method_call( (I64, &args_len), ], ); + let fallback_truthy = truthy_result_kind.map(|_| total_value_truthy(ctx, &fallback_value)); let after_fallback = ctx.block().label.clone(); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); @@ -1423,5 +1619,26 @@ pub(super) fn emit_guarded_direct_method_call( phi_inputs.push((value.as_str(), label.as_str())); } phi_inputs.push((fallback_value.as_str(), after_fallback.as_str())); - Some(ctx.block().phi(DOUBLE, &phi_inputs)) + let boxed = ctx.block().phi(DOUBLE, &phi_inputs); + if truthy_result_kind.is_some() { + let mut truthy_inputs: Vec<(&str, &str)> = Vec::with_capacity(sub_truthy_values.len() + 2); + truthy_inputs.push(( + fast_truthy + .as_deref() + .expect("truthy mode constructs a fast truthiness value"), + after_fast.as_str(), + )); + for (value, label) in &sub_truthy_values { + truthy_inputs.push((value.as_str(), label.as_str())); + } + truthy_inputs.push(( + fallback_truthy + .as_deref() + .expect("truthy mode constructs a fallback truthiness value"), + after_fallback.as_str(), + )); + let truthy = ctx.block().phi(I1, &truthy_inputs); + ctx.pending_truthy_call_result = Some((boxed.clone(), truthy)); + } + Some(boxed) } diff --git a/crates/perry-codegen/src/lower_call/native/mod.rs b/crates/perry-codegen/src/lower_call/native/mod.rs index e07ecbb1e8..7b384c1e47 100644 --- a/crates/perry-codegen/src/lower_call/native/mod.rs +++ b/crates/perry-codegen/src/lower_call/native/mod.rs @@ -27,7 +27,8 @@ use perry_hir::types::Type as HirType; use perry_hir::Expr; use crate::expr::{ - emit_root_nanbox_store_on_block, lower_expr, nanbox_pointer_inline, unbox_to_i64, FnCtx, + emit_root_nanbox_store_on_block, lower_expr, lower_expr_as_i32, nanbox_pointer_inline, + unbox_to_i64, FnCtx, }; use crate::nanbox::{double_literal, POINTER_MASK_I64}; use crate::types::{DOUBLE, I32, I64, PTR}; diff --git a/crates/perry-codegen/src/lower_call/native/native_instance_branch.rs b/crates/perry-codegen/src/lower_call/native/native_instance_branch.rs index c6b44be456..c20f9296fc 100644 --- a/crates/perry-codegen/src/lower_call/native/native_instance_branch.rs +++ b/crates/perry-codegen/src/lower_call/native/native_instance_branch.rs @@ -304,21 +304,58 @@ // call (~50-100 cycles) per push. With amortized doubling, real // reallocs are O(log N) of the total pushes — guarding the // writeback elides the overhead on the 99.9% no-realloc path. + // A combined receiver-shape + nonnegative-index clone gives the ECS + // `this.packed.push(x)` kernel two constructive facts the ordinary + // native call cannot use: `x` already has a raw i32 slot, and the + // receiver field read is guarded by the clone's exact `this` shape. + // Route only that narrow form to the fused runtime entry. All other + // calls retain the boxed-value push loop below. + let u31_param = match (args, recv) { + ( + [Expr::LocalGet(id)], + Expr::PropertyGet { + object: obj_expr, .. + }, + ) if ctx.proven_this.is_some() + && matches!(obj_expr.as_ref(), Expr::This) + && ctx.spec_i32_params.contains(id) + && ctx.i32_counter_slots.contains_key(id) => Some(*id), + _ => None, + }; + let u31_value = if let Some(id) = u31_param { + Some(lower_expr_as_i32(ctx, &Expr::LocalGet(id))?) + } else { + None + }; let mut lowered: Vec = Vec::with_capacity(args.len()); - for a in args { - lowered.push(lower_expr(ctx, a)?); + if u31_value.is_none() { + for a in args { + lowered.push(lower_expr(ctx, a)?); + } } let arr_box = lower_expr(ctx, recv)?; let blk = ctx.block(); let mut arr_handle = unbox_to_i64(blk, &arr_box); let orig_handle = arr_handle.clone(); - // Spec §23.1.3.21: Set(O,"length",…,true) fires unconditionally — guard - // even when args is empty so frozen / non-writable-length throw correctly. - blk.call_void("js_array_push_guard", &[(I64, &arr_handle)]); - for v in &lowered { - let blk = ctx.block(); - arr_handle = blk.call(I64, "js_array_push_f64", &[(I64, &arr_handle), (DOUBLE, v)]); - } + let fused_length_slot = if let Some(value) = u31_value { + let length_slot = blk.alloca(I32); + arr_handle = blk.call( + I64, + "js_array_push_u31_with_length", + &[(I64, &arr_handle), (I32, &value), (PTR, &length_slot)], + ); + Some(length_slot) + } else { + // Spec §23.1.3.21: Set(O,"length",…,true) fires unconditionally — guard + // even when args is empty so frozen / non-writable-length throw correctly. + blk.call_void("js_array_push_guard", &[(I64, &arr_handle)]); + for v in &lowered { + let blk = ctx.block(); + arr_handle = + blk.call(I64, "js_array_push_f64", &[(I64, &arr_handle), (DOUBLE, v)]); + } + None + }; let blk = ctx.block(); let new_handle = arr_handle; let new_box = nanbox_pointer_inline(blk, &new_handle); @@ -375,7 +412,11 @@ ctx.current_block = merge_idx; } let blk = ctx.block(); - let len_i32 = blk.call(I32, "js_array_length", &[(I64, &new_handle)]); + let len_i32 = if let Some(length_slot) = fused_length_slot { + blk.load(I32, &length_slot) + } else { + blk.call(I32, "js_array_length", &[(I64, &new_handle)]) + }; return Ok(blk.sitofp(I32, &len_i32, DOUBLE)); } diff --git a/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs b/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs index 688bdba2a1..e0f7bf1617 100644 --- a/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs +++ b/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs @@ -87,6 +87,10 @@ struct TowerPshapeRoute { /// The `{public}$pshape` clone symbol (same `(double this, args…)` ABI as /// the public one — only the body's `this.field` lowering differs). clone_fn: String, + /// Receiver-safe target for the shape-miss arm. When every indexed + /// parameter is already proved nonnegative i32 at this call site, this is + /// the unshaped `$idx_u31` clone; otherwise it is the public method. + generic_fn: String, /// `@perry_class_keys___`, holding the class's canonical /// keys-array pointer. A receiver still carrying it has the declared packed /// layout; `delete` swaps in a freshly cloned array, which is exactly what @@ -110,6 +114,7 @@ fn tower_pshape_route( owner: Option<&str>, property: &str, fname: &str, + args: &[Expr], ) -> Option { let owner = owner?; // Carried forward from both existing routing sites (the #1787 @@ -124,8 +129,40 @@ fn tower_pshape_route( return None; } let keys_global = ctx.class_keys_globals.get(owner)?.clone(); + let index_params = ctx.nonnegative_index_methods.get(&key); + let index_proven = index_params.is_some_and(|params| { + let Some(method) = ctx + .classes + .get(owner) + .and_then(|class| class.methods.iter().find(|method| method.name == property)) + else { + return false; + }; + args.len() == method.params.len() + && params.iter().all(|id| { + method + .params + .iter() + .position(|param| param.id == *id) + .and_then(|position| args.get(position)) + .is_some_and(|arg| { + crate::expr::numeric_index_has_integer_array_index_proof(ctx, arg) + }) + }) + }); + let pshape_fn = crate::collectors::pshape_method_name(fname); + let (clone_fn, generic_fn) = if index_proven { + let params = index_params.expect("proved indexed tower method remains registered"); + ( + crate::codegen::nonnegative_index_method_name(&pshape_fn, params), + crate::codegen::nonnegative_index_method_name(fname, params), + ) + } else { + (pshape_fn, fname.to_string()) + }; Some(TowerPshapeRoute { - clone_fn: crate::collectors::pshape_method_name(fname), + clone_fn, + generic_fn, keys_global, }) } @@ -160,7 +197,6 @@ fn emit_tower_pshape_call( case_no: usize, route: &TowerPshapeRoute, recv_handle: &str, - fname: &str, case_arg_slices: &[(crate::types::LlvmType, &str)], ) -> String { // The global is read ONCE per function (entry-hoisted); the case block only @@ -191,7 +227,7 @@ fn emit_tower_pshape_call( ctx.block().br(&join_label); ctx.current_block = generic_idx; - let v_generic = ctx.block().call(DOUBLE, fname, case_arg_slices); + let v_generic = ctx.block().call(DOUBLE, &route.generic_fn, case_arg_slices); let generic_end = ctx.block().label.clone(); ctx.block().br(&join_label); @@ -783,13 +819,12 @@ pub(crate) fn try_lower_instance_method_call( ); let case_arg_slices: Vec<(crate::types::LlvmType, &str)> = case_args.iter().map(|s| (DOUBLE, s.as_str())).collect(); - match tower_pshape_route(ctx, owner.as_deref(), property, fname) { + match tower_pshape_route(ctx, owner.as_deref(), property, fname, args) { Some(route) => emit_tower_pshape_call( ctx, case_no, &route, &recv_handle, - fname, &case_arg_slices, ), None => ctx.block().call(DOUBLE, fname, &case_arg_slices), @@ -1466,8 +1501,17 @@ pub(crate) fn try_lower_instance_method_call( (!crate::collectors::ptr_array_cache_fields(class, method).is_empty()) .then(|| crate::collectors::ptr_array_cache_method_name(&fallback_fn)) }); - let generic_target = nonnegative_index_direct_name + let pshape_index_target = + nonnegative_index_direct_name.as_ref().and_then(|_| { + let pshape = pshape_target.as_ref()?; + let params = ctx.nonnegative_index_methods.get(&typed_method_key)?; + Some(crate::codegen::nonnegative_index_method_name( + pshape, params, + )) + }); + let generic_target = pshape_index_target .as_deref() + .or(nonnegative_index_direct_name.as_deref()) .or(ptr_array_cache_target.as_deref()) .or(pshape_target.as_deref()) .unwrap_or(fallback_fn.as_str()); @@ -1484,6 +1528,11 @@ pub(crate) fn try_lower_instance_method_call( &arg_slices, args, ) { + super::super::method_override::publish_constructive_method_truthy( + ctx, + &fallback_fn, + &argument_specialized, + ); return Ok(Some(argument_specialized)); } // Prefer the typed-receiver clone (bare gep+load field @@ -1547,6 +1596,11 @@ pub(crate) fn try_lower_instance_method_call( (v_generic.as_str(), &generic_end), ], ); + super::super::method_override::publish_constructive_method_truthy( + ctx, + &fallback_fn, + &merged, + ); return Ok(Some(merged)); } // Representation-selection Phase 5a: route to the @@ -1556,6 +1610,11 @@ pub(crate) fn try_lower_instance_method_call( // all. Same ABI, so the call is unchanged apart from the // callee name. let direct = ctx.block().call(DOUBLE, generic_target, &arg_slices); + super::super::method_override::publish_constructive_method_truthy( + ctx, + &fallback_fn, + &direct, + ); return Ok(Some(direct)); } if let Some(guarded) = emit_guarded_direct_method_call( diff --git a/crates/perry-codegen/src/lower_conditional.rs b/crates/perry-codegen/src/lower_conditional.rs index 72824d0472..bceb56648d 100644 --- a/crates/perry-codegen/src/lower_conditional.rs +++ b/crates/perry-codegen/src/lower_conditional.rs @@ -6,7 +6,8 @@ use anyhow::Result; use perry_hir::{Expr, LogicalOp}; -use crate::expr::{lower_expr, FnCtx}; +use crate::expr::{lower_expr, lower_expr_value, FnCtx}; +use crate::native_value::{materialize_js_value, MaterializationReason, NativeRep}; use crate::type_analysis::{ expr_may_return_boxed_value_from_raw_f64_fallback, is_bool_expr, is_numeric_expr, }; @@ -71,6 +72,44 @@ pub(crate) fn lower_truthy(ctx: &mut FnCtx<'_>, cond_val: &str, cond_expr: &Expr ctx.block().icmp_ne(I32, &i32_truthy, "0") } +/// Lower one expression once and return both its ordinary boxed value and its +/// JavaScript truthiness as native `i1`. +/// +/// Guarded direct user-method calls may publish a use-sensitive truthiness +/// result: the proven method arm tests a constructively-Boolean return inline, +/// while the dynamic override arm still uses the total runtime predicate. The +/// boxed SSA-name equality below makes publication compositional — a call in +/// the receiver or an argument cannot impersonate the outer expression. +pub(crate) fn lower_expr_with_truthy(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<(String, String)> { + if let Some(lowered) = lower_expr_value(ctx, expr)? { + if matches!(lowered.rep, NativeRep::I1) { + let truthy = lowered.value.clone(); + let boxed = materialize_js_value(ctx, lowered, MaterializationReason::RuntimeApi); + return Ok((boxed, truthy)); + } + let boxed = materialize_js_value(ctx, lowered, MaterializationReason::RuntimeApi); + let truthy = lower_truthy(ctx, &boxed, expr); + return Ok((boxed, truthy)); + } + + let saved_request = ctx.truthy_call_result_requested; + let saved_pending = ctx.pending_truthy_call_result.take(); + ctx.truthy_call_result_requested = true; + let lowered = lower_expr(ctx, expr); + let published = ctx.pending_truthy_call_result.take(); + ctx.truthy_call_result_requested = saved_request; + ctx.pending_truthy_call_result = saved_pending; + let boxed = lowered?; + + if let Some((published_boxed, truthy)) = published { + if published_boxed == boxed { + return Ok((boxed, truthy)); + } + } + let truthy = lower_truthy(ctx, &boxed, expr); + Ok((boxed, truthy)) +} + /// Lower `cond ? then_expr : else_expr` to a 4-block CFG with a phi at /// the merge: condition → conditional cond_br → then → merge ← else. /// Both then and else are always lowered (no short-circuit), but only one @@ -85,8 +124,7 @@ pub(crate) fn lower_conditional( let saved_guarded_proof = branch_proofs .as_ref() .and_then(|(id, _, _)| ctx.snapshot_guarded_proof(id)); - let cond = lower_expr(ctx, condition)?; - let cond_bool = lower_truthy(ctx, &cond, condition); + let (_cond, cond_bool) = lower_expr_with_truthy(ctx, condition)?; let then_idx = ctx.new_block("ternary.then"); let else_idx = ctx.new_block("ternary.else"); @@ -206,13 +244,10 @@ pub(crate) fn lower_logical( } // Lower left in the current block. - let l = lower_expr(ctx, left)?; + let (l, l_bool) = lower_expr_with_truthy(ctx, left)?; // Capture the post-left block — left's lowering may have created new // blocks via nested control flow. let l_block_label = ctx.block().label.clone(); - // Truthiness test: fast fcmp for numeric, js_is_truthy for NaN-boxed. - let l_bool = lower_truthy(ctx, &l, left); - let then_idx = ctx.new_block("logical.then"); let merge_idx = ctx.new_block("logical.merge"); let then_label = ctx.block_label(then_idx); @@ -230,9 +265,17 @@ pub(crate) fn lower_logical( LogicalOp::Coalesce => unreachable!("guarded above"), } - // The "then" block evaluates the right side. + // The "then" block evaluates the right side. When an enclosing condition + // requested native truthiness, preserve the right operand's truthiness as + // well as its actual JavaScript value; `&&` / `||` return an operand, so + // replacing the value itself with a Boolean would be observably wrong. ctx.current_block = then_idx; - let r = lower_expr(ctx, right)?; + let (r, r_bool) = if ctx.truthy_call_result_requested { + let (value, truthy) = lower_expr_with_truthy(ctx, right)?; + (value, Some(truthy)) + } else { + (lower_expr(ctx, right)?, None) + }; let r_block_label = ctx.block().label.clone(); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); @@ -240,7 +283,23 @@ pub(crate) fn lower_logical( // Merge block: phi between l (short-circuit path) and r (normal path). ctx.current_block = merge_idx; - Ok(ctx + let result = ctx .block() - .phi(DOUBLE, &[(&l, &l_block_label), (&r, &r_block_label)])) + .phi(DOUBLE, &[(&l, &l_block_label), (&r, &r_block_label)]); + if let Some(r_bool) = r_bool { + let short_circuit_truthy = match op { + LogicalOp::And => "false", + LogicalOp::Or => "true", + LogicalOp::Coalesce => unreachable!("guarded above"), + }; + let truthy = ctx.block().phi( + crate::types::I1, + &[ + (short_circuit_truthy, &l_block_label), + (&r_bool, &r_block_label), + ], + ); + ctx.pending_truthy_call_result = Some((result.clone(), truthy)); + } + Ok(result) } diff --git a/crates/perry-codegen/src/module.rs b/crates/perry-codegen/src/module.rs index dc81409128..00d461c359 100644 --- a/crates/perry-codegen/src/module.rs +++ b/crates/perry-codegen/src/module.rs @@ -579,6 +579,19 @@ impl LlModule { self.functions.get_mut(idx) } + /// Render-free body-size estimate for an already-lowered function. + /// + /// Guarded entry wrappers are emitted after their private specialization + /// bodies. They use this lookup to decide whether flattening that body + /// before statepoint rewriting stays inside the explicit native-roots + /// code-size budget. + pub(crate) fn function_estimated_ir_bytes(&self, name: &str) -> Option { + self.functions + .iter() + .find(|function| function.name == name) + .map(LlFunction::estimated_ir_bytes) + } + /// Every defined function, mutably — for the whole-module passes that run /// after lowering and before any rendering path. See /// [`crate::root_reload`], and note that "before ANY rendering path" is the diff --git a/crates/perry-codegen/src/runtime_decls/arrays.rs b/crates/perry-codegen/src/runtime_decls/arrays.rs index 5449b0d3c0..84abf596c0 100644 --- a/crates/perry-codegen/src/runtime_decls/arrays.rs +++ b/crates/perry-codegen/src/runtime_decls/arrays.rs @@ -42,6 +42,7 @@ pub fn declare_phase_b_arrays(module: &mut LlModule) { // blob_len). Returns the nanboxed JS value (a fresh, mutable array). module.declare_function("js_value_from_const_descriptor", DOUBLE, &[PTR, I32]); module.declare_function("js_array_push_f64", I64, &[I64, DOUBLE]); + module.declare_function("js_array_push_u31_with_length", I64, &[I64, I32, PTR]); module.declare_function("js_array_push_guard", VOID, &[I64]); module.declare_function("js_array_push_hole", I64, &[I64]); module.declare_function("js_array_numeric_push_f64_unboxed", I64, &[I64, DOUBLE]); @@ -106,6 +107,7 @@ pub fn declare_phase_b_arrays(module: &mut LlModule) { // it preserves `undefined` and non-numeric property values and throws for // nullish receivers. module.declare_function("js_value_length_property_f64", DOUBLE, &[DOUBLE]); + module.declare_function("js_value_length_property_ic_f64", DOUBLE, &[DOUBLE, PTR]); // Shadow stack for precise root tracking (gen-GC Phase A per // docs/generational-gc-plan.md). Declared now so codegen can diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index d8d21bf9ff..3e498ed243 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -162,6 +162,9 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { module.declare_function("js_text_encoder_encode_into_llvm", I64, &[DOUBLE, DOUBLE]); // typeof: returns a string handle ("number"/"string"/"boolean"/"undefined"/"object"/"function") module.declare_function("js_value_typeof", I64, &[DOUBLE]); + // Integer classifier used by `typeof value === "literal"`, with the same + // exceptional-representation semantics and no cached-string round trip. + module.declare_function("js_value_typeof_tag", I32, &[DOUBLE]); module.declare_function("js_string_starts_with", I32, &[I64, I64]); module.declare_function("js_string_ends_with", I32, &[I64, I64]); module.declare_function("js_string_search_value_to_string", I64, &[DOUBLE, I32]); diff --git a/crates/perry-codegen/src/runtime_decls/strings_part2.rs b/crates/perry-codegen/src/runtime_decls/strings_part2.rs index 406d442753..552c2cd4bb 100644 --- a/crates/perry-codegen/src/runtime_decls/strings_part2.rs +++ b/crates/perry-codegen/src/runtime_decls/strings_part2.rs @@ -309,6 +309,17 @@ pub(crate) fn declare_phase_b_strings_part2(module: &mut LlModule) { &[DOUBLE, DOUBLE], ); module.declare_function("js_object_get_symbol_property", DOUBLE, &[DOUBLE, DOUBLE]); + module.declare_function( + "js_object_get_symbol_property_ic_miss", + DOUBLE, + &[DOUBLE, DOUBLE, PTR], + ); + module.declare_function( + "js_object_get_symbol_then_field_ic_miss", + DOUBLE, + &[DOUBLE, DOUBLE, I64, I64, PTR, PTR], + ); + module.add_external_global("PERRY_SYMBOL_PROPERTY_IC_EPOCH", I64); module.declare_function("js_object_create", DOUBLE, &[DOUBLE]); // #2816: Object.create(proto[, propertiesObject]) — validates the // prototype and applies the optional descriptor bag. diff --git a/crates/perry-codegen/src/stmt/cached_field_index_return.rs b/crates/perry-codegen/src/stmt/cached_field_index_return.rs new file mode 100644 index 0000000000..4cbd480574 --- /dev/null +++ b/crates/perry-codegen/src/stmt/cached_field_index_return.rs @@ -0,0 +1,321 @@ +//! Guarded fast return for `if (!owner.table[i]) { ... } return owner.table[i]`. +//! +//! The source performs the indexed read twice on the already-populated path. +//! Reusing the first result is not generally valid: either property access can +//! invoke a getter/Proxy and the second observation is required. This lowering +//! therefore adds only a speculative *proof* path. It returns early solely +//! after proving an own shape-cached data field, an ordinary dense Array, an +//! in-bounds data slot, and a truthy result. Every failure enters the original +//! statements unchanged. + +use anyhow::Result; +use perry_hir::{Expr, Stmt, UnaryOp}; + +use crate::expr::{lower_expr, FnCtx, PropertyGetIcOverride}; +use crate::lower_conditional::lower_truthy; +use crate::types::{DOUBLE, I1, I16, I32, I64, I8}; + +struct Candidate<'a> { + base_local_id: u32, + property: &'a str, + index_local_id: u32, + access: &'a Expr, +} + +fn field_index_access(expr: &Expr) -> Option<(u32, &str, u32)> { + let Expr::IndexGet { object, index } = expr else { + return None; + }; + let Expr::PropertyGet { + object: base, + property, + .. + } = object.as_ref() + else { + return None; + }; + let Expr::LocalGet(base_local_id) = base.as_ref() else { + return None; + }; + let Expr::LocalGet(index_local_id) = index.as_ref() else { + return None; + }; + Some((*base_local_id, property.as_str(), *index_local_id)) +} + +fn match_candidate(stmts: &[Stmt]) -> Option> { + let ( + Stmt::If { + condition, + else_branch: None, + .. + }, + Stmt::Return(Some(returned)), + ) = (stmts.first()?, stmts.get(1)?) + else { + return None; + }; + let Expr::Unary { + op: UnaryOp::Not, + operand, + } = condition + else { + return None; + }; + let (base_local_id, property, index_local_id) = field_index_access(operand)?; + let (return_base, return_property, return_index) = field_index_access(returned)?; + if (base_local_id, property, index_local_id) != (return_base, return_property, return_index) { + return None; + } + Some(Candidate { + base_local_id, + property, + index_local_id, + access: operand, + }) +} + +fn allocate_shared_cache(ctx: &mut FnCtx<'_>, candidate: &Candidate<'_>) -> String { + let site_id = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let cache_name = crate::expr::inline_cache_global_name(ctx, site_id); + ctx.pending_declares + .push((format!("__ic_decl_{site_id}"), DOUBLE, vec![])); + ctx.ic_globals.push(cache_name.clone()); + ctx.property_get_ic_override = Some(PropertyGetIcOverride { + base_local_id: candidate.base_local_id, + property: candidate.property.to_string(), + cache_name: cache_name.clone(), + }); + cache_name +} + +/// Emit the speculative early-return path and leave `ctx.current_block` on the +/// normal fallback block. The caller then lowers the original statements. +pub(super) fn try_emit_cached_field_index_return( + ctx: &mut FnCtx<'_>, + stmts: &[Stmt], +) -> Result { + let Some(candidate) = match_candidate(stmts) else { + return Ok(false); + }; + let Some(index_slot) = ctx + .i32_counter_slots + .get(&candidate.index_local_id) + .cloned() + else { + return Ok(false); + }; + if !ctx + .nonnegative_integer_locals + .contains(&candidate.index_local_id) + || ctx.property_get_ic_override.is_some() + || ctx.is_async_fn + || ctx.try_depth != 0 + || !ctx.inline_ctor_return.is_empty() + || ctx.shared_super_scope_active + { + return Ok(false); + } + + let cache_name = allocate_shared_cache(ctx, &candidate); + let cache_ref = format!("@{cache_name}"); + let base_box = lower_expr(ctx, &Expr::LocalGet(candidate.base_local_id))?; + let index_i32 = ctx.block().load(I32, &index_slot); + + let object_header_idx = ctx.new_block("cached_field_index.object_header"); + let exact_or_prefix_idx = ctx.new_block("cached_field_index.exact_or_prefix"); + let exact_token_idx = ctx.new_block("cached_field_index.exact_token"); + let prefix_meta_idx = ctx.new_block("cached_field_index.prefix_meta"); + let prefix_token_idx = ctx.new_block("cached_field_index.prefix_token"); + let field_load_idx = ctx.new_block("cached_field_index.field_load"); + let array_header_idx = ctx.new_block("cached_field_index.array_header"); + let array_load_idx = ctx.new_block("cached_field_index.array_load"); + let truthy_idx = ctx.new_block("cached_field_index.truthy"); + let return_idx = ctx.new_block("cached_field_index.return"); + let normal_idx = ctx.new_block("cached_field_index.normal"); + let object_header_label = ctx.block_label(object_header_idx); + let exact_or_prefix_label = ctx.block_label(exact_or_prefix_idx); + let exact_token_label = ctx.block_label(exact_token_idx); + let prefix_meta_label = ctx.block_label(prefix_meta_idx); + let prefix_token_label = ctx.block_label(prefix_token_idx); + let field_load_label = ctx.block_label(field_load_idx); + let array_header_label = ctx.block_label(array_header_idx); + let array_load_label = ctx.block_label(array_load_idx); + let truthy_label = ctx.block_label(truthy_idx); + let return_label = ctx.block_label(return_idx); + let normal_label = ctx.block_label(normal_idx); + + let tag_mask = crate::nanbox::i64_literal(crate::nanbox::TAG_MASK); + let object_raw = { + let blk = ctx.block(); + let bits = blk.bitcast_double_to_i64(&base_box); + let raw = blk.and(I64, &bits, crate::nanbox::POINTER_MASK_I64); + let tag = blk.and(I64, &bits, &tag_mask); + let is_pointer = blk.icmp_eq(I64, &tag, crate::nanbox::POINTER_TAG_I64); + let above_handles = blk.icmp_ugt(I64, &raw, "1048575"); + let eligible = blk.and(I1, &is_pointer, &above_handles); + blk.cond_br(&eligible, &object_header_label, &normal_label); + raw + }; + + ctx.current_block = object_header_idx; + let gc_type_addr = ctx.block().sub(I64, &object_raw, "8"); + let gc_type_ptr = ctx.block().inttoptr(I64, &gc_type_addr); + let gc_type = ctx.block().load(I8, &gc_type_ptr); + let is_object = ctx.block().icmp_eq(I8, &gc_type, "2"); + let gc_flags_addr = ctx.block().sub(I64, &object_raw, "7"); + let gc_flags_ptr = ctx.block().inttoptr(I64, &gc_flags_addr); + let gc_flags = ctx.block().load(I8, &gc_flags_ptr); + let forwarded_bits = ctx.block().and(I8, &gc_flags, "128"); + let not_forwarded = ctx.block().icmp_eq(I8, &forwarded_bits, "0"); + let object_ok = ctx.block().and(I1, &is_object, ¬_forwarded); + ctx.block() + .cond_br(&object_ok, &exact_or_prefix_label, &normal_label); + + ctx.current_block = exact_or_prefix_idx; + let reserved_addr = ctx.block().sub(I64, &object_raw, "6"); + let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); + let reserved = ctx.block().load(I16, &reserved_ptr); + let descriptor_bits = ctx.block().and(I16, &reserved, "2048"); + let no_descriptors = ctx.block().icmp_eq(I16, &descriptor_bits, "0"); + ctx.block() + .cond_br(&no_descriptors, &exact_token_label, &prefix_meta_label); + + // Descriptor-bearing instances (including Array subclasses with an own + // `length`) cannot use an exact ShapeId property slot. Send them straight + // to the data-only named-prefix proof instead of loading a dead shape. + ctx.current_block = exact_token_idx; + let shape_addr = ctx.block().add(I64, &object_raw, "4"); + let shape_ptr = ctx.block().inttoptr(I64, &shape_addr); + let shape_id = ctx.block().load(I32, &shape_ptr); + let shape_nonzero = ctx.block().icmp_ne(I32, &shape_id, "0"); + let shape_i64 = ctx.block().zext(I32, &shape_id, I64); + let live_token = ctx.block().or(I64, &shape_i64, "4611686018427387904"); + let cached_token_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]); + let cached_token = ctx.block().load(I64, &cached_token_ptr); + let token_matches = ctx.block().icmp_eq(I64, &live_token, &cached_token); + let exact = ctx.block().and(I1, &shape_nonzero, &token_matches); + ctx.block() + .cond_br(&exact, &field_load_label, &prefix_meta_label); + + ctx.current_block = prefix_meta_idx; + let cached_prefix_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "2")]); + let cached_prefix = ctx.block().load(I64, &cached_prefix_ptr); + let prefix_armed = ctx.block().icmp_ne(I64, &cached_prefix, "0"); + let pointer_bytes = if crate::target_layout::target_is_ilp32(ctx.target_triple) { + 4 + } else { + 8 + }; + let meta_offset = (crate::target_layout::object_header_size_bytes(ctx.target_triple) + - pointer_bytes) + .to_string(); + let meta_addr = ctx.block().add(I64, &object_raw, &meta_offset); + let meta_slot = ctx.block().inttoptr(I64, &meta_addr); + let meta_ty = if pointer_bytes == 4 { I32 } else { I64 }; + let meta_raw = ctx.block().load(meta_ty, &meta_slot); + let meta = if pointer_bytes == 4 { + ctx.block().zext(I32, &meta_raw, I64) + } else { + meta_raw + }; + let meta_nonzero = ctx.block().icmp_ne(I64, &meta, "0"); + let can_check_prefix = ctx.block().and(I1, &prefix_armed, &meta_nonzero); + ctx.block() + .cond_br(&can_check_prefix, &prefix_token_label, &normal_label); + + ctx.current_block = prefix_token_idx; + let meta_ptr = ctx.block().inttoptr(I64, &meta); + let object_prefix_ptr = ctx.block().gep(I64, &meta_ptr, &[(I64, "6")]); + let object_prefix = ctx.block().load(I64, &object_prefix_ptr); + let prefix_matches = ctx.block().icmp_eq(I64, &object_prefix, &cached_prefix); + ctx.block() + .cond_br(&prefix_matches, &field_load_label, &normal_label); + + ctx.current_block = field_load_idx; + let cached_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]); + let cached_slot = ctx.block().load(I64, &cached_slot_ptr); + let field_offset = ctx.block().shl(I64, &cached_slot, "3"); + let header_size = crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string(); + let fields_base = ctx.block().add(I64, &object_raw, &header_size); + let field_addr = ctx.block().add(I64, &fields_base, &field_offset); + let field_ptr = ctx.block().inttoptr(I64, &field_addr); + let array_box = ctx.block().load(DOUBLE, &field_ptr); + let array_bits = ctx.block().bitcast_double_to_i64(&array_box); + let array_raw = ctx + .block() + .and(I64, &array_bits, crate::nanbox::POINTER_MASK_I64); + let array_tag = ctx.block().and(I64, &array_bits, &tag_mask); + let array_is_pointer = ctx + .block() + .icmp_eq(I64, &array_tag, crate::nanbox::POINTER_TAG_I64); + let array_above_handles = ctx.block().icmp_ugt(I64, &array_raw, "1048575"); + let array_address_ok = ctx.block().and(I1, &array_is_pointer, &array_above_handles); + ctx.block() + .cond_br(&array_address_ok, &array_header_label, &normal_label); + + ctx.current_block = array_header_idx; + let array_gc_type_addr = ctx.block().sub(I64, &array_raw, "8"); + let array_gc_type_ptr = ctx.block().inttoptr(I64, &array_gc_type_addr); + let array_gc_type = ctx.block().load(I8, &array_gc_type_ptr); + let is_array = ctx.block().icmp_eq(I8, &array_gc_type, "1"); + let array_gc_flags_addr = ctx.block().sub(I64, &array_raw, "7"); + let array_gc_flags_ptr = ctx.block().inttoptr(I64, &array_gc_flags_addr); + let array_gc_flags = ctx.block().load(I8, &array_gc_flags_ptr); + let array_forwarded_bits = ctx.block().and(I8, &array_gc_flags, "128"); + let array_not_forwarded = ctx.block().icmp_eq(I8, &array_forwarded_bits, "0"); + let array_reserved_addr = ctx.block().sub(I64, &array_raw, "6"); + let array_reserved_ptr = ctx.block().inttoptr(I64, &array_reserved_addr); + let array_reserved = ctx.block().load(I16, &array_reserved_ptr); + let array_descriptor_bits = ctx.block().and(I16, &array_reserved, "1024"); + let array_no_descriptors = ctx.block().icmp_eq(I16, &array_descriptor_bits, "0"); + let invalidated = ctx + .block() + .load_volatile(I8, "@PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED"); + let default_prototypes = ctx.block().icmp_eq(I8, &invalidated, "0"); + let array_ptr = ctx.block().inttoptr(I64, &array_raw); + let length = ctx.block().load(I32, &array_ptr); + let capacity_addr = ctx.block().add(I64, &array_raw, "4"); + let capacity_ptr = ctx.block().inttoptr(I64, &capacity_addr); + let capacity = ctx.block().load(I32, &capacity_ptr); + let index_in_bounds = ctx.block().icmp_ult(I32, &index_i32, &length); + let sane_capacity = ctx.block().icmp_ule(I32, &length, &capacity); + let array_ok = ctx.block().and(I1, &is_array, &array_not_forwarded); + let array_ok = ctx.block().and(I1, &array_ok, &array_no_descriptors); + let array_ok = ctx.block().and(I1, &array_ok, &default_prototypes); + let array_ok = ctx.block().and(I1, &array_ok, &index_in_bounds); + let array_ok = ctx.block().and(I1, &array_ok, &sane_capacity); + ctx.block() + .cond_br(&array_ok, &array_load_label, &normal_label); + + ctx.current_block = array_load_idx; + let index_i64 = ctx.block().zext(I32, &index_i32, I64); + let element_word = ctx.block().add(I64, &index_i64, "1"); + let element_ptr = ctx + .block() + .gep_inbounds(I64, &array_ptr, &[(I64, &element_word)]); + let raw_value = ctx.block().load(DOUBLE, &element_ptr); + let raw_bits = ctx.block().bitcast_double_to_i64(&raw_value); + let is_hole = ctx + .block() + .icmp_eq(I64, &raw_bits, crate::nanbox::TAG_HOLE_I64); + let undefined = ctx + .block() + .bitcast_i64_to_double(crate::nanbox::TAG_UNDEFINED_I64); + let value = ctx + .block() + .select(I1, &is_hole, DOUBLE, &undefined, &raw_value); + ctx.block().br(&truthy_label); + + ctx.current_block = truthy_idx; + let is_truthy = lower_truthy(ctx, &value, candidate.access); + ctx.block() + .cond_br(&is_truthy, &return_label, &normal_label); + + ctx.current_block = return_idx; + ctx.block().ret(DOUBLE, &value); + + ctx.current_block = normal_idx; + Ok(true) +} diff --git a/crates/perry-codegen/src/stmt/if_stmt.rs b/crates/perry-codegen/src/stmt/if_stmt.rs index 0e47954615..dd56e00554 100644 --- a/crates/perry-codegen/src/stmt/if_stmt.rs +++ b/crates/perry-codegen/src/stmt/if_stmt.rs @@ -4,8 +4,7 @@ use std::collections::{HashMap, HashSet}; use super::*; -use crate::lower_conditional::lower_truthy; -use crate::native_value::NativeRep; +use crate::lower_conditional::lower_expr_with_truthy; #[derive(Clone)] struct NativeArenaOwnerAliasSnapshot { @@ -270,14 +269,6 @@ pub(crate) fn lower_if( } fn lower_if_condition_i1(ctx: &mut FnCtx<'_>, condition: &perry_hir::Expr) -> Result { - if let Some(lowered) = lower_expr_value(ctx, condition)? { - if matches!(lowered.rep, NativeRep::I1) { - return Ok(lowered.value); - } - let boxed = materialize_js_value(ctx, lowered, MaterializationReason::RuntimeApi); - return Ok(lower_truthy(ctx, &boxed, condition)); - } - - let cond_val = lower_expr(ctx, condition)?; - Ok(lower_truthy(ctx, &cond_val, condition)) + let (_boxed, truthy) = lower_expr_with_truthy(ctx, condition)?; + Ok(truthy) } diff --git a/crates/perry-codegen/src/stmt/mod.rs b/crates/perry-codegen/src/stmt/mod.rs index 7b52c4a0d8..6a2f571648 100644 --- a/crates/perry-codegen/src/stmt/mod.rs +++ b/crates/perry-codegen/src/stmt/mod.rs @@ -13,6 +13,7 @@ use crate::types::DOUBLE; #[cfg(test)] mod boxed_slot_no_root_tests; +mod cached_field_index_return; #[cfg(test)] mod class_field_loop_tests; mod counter_range; @@ -140,6 +141,14 @@ fn lower_async_rejecting_stmts_inner( fn lower_stmts_inner(ctx: &mut FnCtx<'_>, stmts: &[Stmt], emit_shadow_clears: bool) -> Result<()> { let mut i = 0; while i < stmts.len() { + // A common memo-table method shape is + // `if (!owner.table[i]) { ...fill... } return owner.table[i]`. + // Before lowering the untouched statements, add a guarded direct + // data-field/Array probe that can return the first truthy value. Every + // miss falls into the ordinary lowering below, including accessors, + // proxies, sparse/OOB arrays and the falsy fill branch. + cached_field_index_return::try_emit_cached_field_index_return(ctx, &stmts[i..])?; + // Channel-reduction fusion: detect a length-3-or-4 sequence of // `acc[c] += arr[idx + c] * k` accumulator updates and emit a // single `<4 x i32>` SIMD multiply-add. The canonical hot shape diff --git a/crates/perry-runtime/src/array/element_shape.rs b/crates/perry-runtime/src/array/element_shape.rs index 35bfdc16ce..ba75238d13 100644 --- a/crates/perry-runtime/src/array/element_shape.rs +++ b/crates/perry-runtime/src/array/element_shape.rs @@ -111,6 +111,13 @@ const MAX_VERIFIED_LEN: usize = 16_000_000; struct ElementShapeRecord { /// `ObjectHeader::class_id` shared by every element in `[0, length)`. class_id: u32, + /// One exact ordinary-object ShapeId already validated by the complete + /// classifier. Most homogeneous arrays also have a homogeneous exact + /// shape, so matching stores can validate two scalar object-header words + /// without probing the global shape table. A same-class value with a + /// different shape still takes the complete classifier below; this word + /// narrows no existing class-level proof. + ordinary_shape_id: u32, /// The `length` this record was verified against. A query requires the /// array's current `length` to still equal it, so every length-changing /// mutation invalidates the proof without needing its own call site. @@ -124,7 +131,11 @@ struct ElementShapeRecord { epoch: u64, /// `CLASS_SHAPE_GENERATION` at install time. A prototype write bumps the /// global and retires every record at once. - generation: u64, + /// Low-width snapshot keeps this hot side-table record at its original + /// 24-byte size after adding `ordinary_shape_id`. Once the global counter + /// exceeds `u32`, proofs simply stop establishing/fail closed; observable + /// array behavior still uses the generic path. + generation: u32, } /// What a query hands back. Deliberately not the raw record: `generation` is @@ -147,6 +158,18 @@ crate::perry_thread_local! { RefCell::new(crate::fast_hash::new_ptr_hash_map()); } +#[cfg(test)] +std::thread_local! { + static ARRAY_SUBCLASS_PREFIX_STORE_HITS: std::cell::Cell = const { + std::cell::Cell::new(0) + }; +} + +#[cfg(test)] +pub(crate) fn test_array_subclass_prefix_store_hits() -> u64 { + ARRAY_SUBCLASS_PREFIX_STORE_HITS.with(std::cell::Cell::get) +} + // #7946: all three counters are `per_test_global!`, so a test build gives each // libtest thread its own instance and a product build gets the plain `static` // back, byte for byte. @@ -237,8 +260,11 @@ pub(crate) fn invalidate_all_element_shapes() { /// garbage that could compare equal across unrelated arrays. Requiring the /// authoritative object kind/marker and a nonzero class id keeps every /// accepted value a genuine shaped instance. +/// Complete shaped-object classifier used to establish an exact fast +/// identity. The header is validated once; after that, the descriptor probe +/// supplies the authoritative ordinary-vs-class-object distinction. #[inline] -pub(crate) fn element_class_of_bits(value_bits: u64) -> Option { +fn element_identity_of_bits(value_bits: u64) -> Option<(u32, u32)> { if value_bits & crate::value::TAG_MASK != crate::value::POINTER_TAG { return None; } @@ -251,21 +277,89 @@ pub(crate) fn element_class_of_bits(value_bits: u64) -> Option { let Some(header) = crate::value::addr_class::try_read_gc_header(addr) else { return None; }; - if header.obj_type != crate::gc::GC_TYPE_OBJECT { + if header.obj_type != crate::gc::GC_TYPE_OBJECT + || header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + { return None; } let obj = addr as *const crate::object::ObjectHeader; - if !crate::object::object_is_regular(obj) { + let shape_id = (*obj).parent_class_id; + if !crate::object::shapes::shape_descriptor_by_id(shape_id).is_some_and(|shape| { + shape.object_kind == crate::object::shapes::ShapeObjectKind::Ordinary + }) { return None; } let class_id = (*obj).class_id; if class_id == 0 { return None; } - Some(class_id) + Some((class_id, shape_id)) } } +/// Validate a value against an established class proof. Exact-shape matches +/// need no descriptor-table lookup because `ordinary_shape_id` was admitted +/// only by [`element_identity_of_bits`] and ShapeIds are never reused. A +/// different shape keeps the historical same-class behavior by running the +/// complete classifier. +#[inline] +fn element_matches_record(value_bits: u64, record: ElementShapeRecord) -> bool { + if value_bits & crate::value::TAG_MASK != crate::value::POINTER_TAG { + return false; + } + let addr = (value_bits & crate::value::POINTER_MASK) as usize; + unsafe { + let Some(header) = crate::value::addr_class::try_read_gc_header(addr) else { + return false; + }; + if header.obj_type != crate::gc::GC_TYPE_OBJECT + || header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + { + return false; + } + let obj = addr as *const crate::object::ObjectHeader; + if (*obj).class_id != record.class_id { + return false; + } + if (*obj).parent_class_id == record.ordinary_shape_id { + #[cfg(test)] + EXACT_SHAPE_STORE_HITS.with(|hits| hits.set(hits.get().wrapping_add(1))); + return true; + } + // Object-backed Array subclasses publish a move-stable, class-wide + // ordinary-prefix proof precisely because their numeric tail mints a + // different ShapeId on every push/pop. Once present, that token is a + // stronger ordinary-instance discriminator than another global + // ShapeId-kind lookup. It is cleared before every generic semantic or + // structural transition; exact numeric-tail transitions are its only + // preserving publisher. + if crate::array::subclass::array_subclass_named_prefix_token_matches_class( + obj, + record.class_id, + ) { + #[cfg(test)] + ARRAY_SUBCLASS_PREFIX_STORE_HITS.with(|hits| hits.set(hits.get().wrapping_add(1))); + return true; + } + element_identity_of_validated_object(obj) + .is_some_and(|identity| identity.0 == record.class_id) + } +} + +#[inline] +unsafe fn element_identity_of_validated_object( + obj: *const crate::object::ObjectHeader, +) -> Option<(u32, u32)> { + let shape_id = (*obj).parent_class_id; + if crate::object::shapes::shape_object_kind_by_id(shape_id) + != Some(crate::object::shapes::ShapeObjectKind::Ordinary) + { + return None; + } + let class_id = (*obj).class_id; + (class_id != 0).then_some((class_id, shape_id)) +} + #[inline] unsafe fn header_has_bit(header: *const crate::gc::GcHeader) -> bool { (*header)._reserved & GC_ARRAY_ELEMENT_SHAPE != 0 @@ -350,15 +444,24 @@ pub(crate) fn forget_element_shape(user_ptr: usize) { /// defence against address recycling: a survivor record — left by a /// fail-closed transfer, a dead array whose prune has not run yet — can never /// donate its identity to the array established here next. -unsafe fn establish(arr: *mut ArrayHeader, class_id: u32, verified_len: u32) { +unsafe fn establish( + arr: *mut ArrayHeader, + class_id: u32, + ordinary_shape_id: u32, + verified_len: u32, +) { let Some(header) = array_gc_header(arr) else { return; }; + let Ok(generation) = u32::try_from(class_shape_generation()) else { + return; + }; let record = ElementShapeRecord { class_id, + ordinary_shape_id, verified_len, epoch: ELEMENT_SHAPE_PROOF_SEQ.fetch_add(1, Ordering::Relaxed), - generation: class_shape_generation(), + generation, }; ELEMENT_SHAPES.with(|m| { m.borrow_mut().insert(arr as usize, record); @@ -366,24 +469,6 @@ unsafe fn establish(arr: *mut ArrayHeader, class_id: u32, verified_len: u32) { set_bit(header); } -/// **Keep** an existing proof while extending its verified prefix. The -/// identity is carried unchanged — a consumer that pinned it stays valid, -/// which is the point: appending a matching element does not retire anything. -unsafe fn extend_verified_len(arr: *mut ArrayHeader, record: ElementShapeRecord, new_len: u32) { - if array_gc_header(arr).is_none() { - return; - } - ELEMENT_SHAPES.with(|m| { - m.borrow_mut().insert( - arr as usize, - ElementShapeRecord { - verified_len: new_len, - ..record - }, - ); - }); -} - /// The O(1) query: does `arr` still carry a homogeneous element-shape proof? /// /// Self-healing in the invalidating direction — a record that lost its @@ -404,7 +489,7 @@ pub(crate) unsafe fn element_shape_proof(arr: *const ArrayHeader) -> Option Option Option = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn test_exact_shape_store_hits() -> u64 { + EXACT_SHAPE_STORE_HITS.with(std::cell::Cell::get) +} + #[cfg(test)] pub(crate) unsafe fn test_element_shape_bit_set(arr: *const ArrayHeader) -> bool { array_gc_header(arr).is_some_and(|header| header_has_bit(header)) diff --git a/crates/perry-runtime/src/array/element_shape_tests.rs b/crates/perry-runtime/src/array/element_shape_tests.rs index e8675109d5..2bdbdbe799 100644 --- a/crates/perry-runtime/src/array/element_shape_tests.rs +++ b/crates/perry-runtime/src/array/element_shape_tests.rs @@ -25,6 +25,7 @@ use crate::array::{ /// registers for itself. const CLASS_A: u32 = 0x0007_4801; const CLASS_B: u32 = 0x0007_4802; +const CLASS_SAME_CLASS_VARIANT: u32 = 0x0007_4803; fn instance(class_id: u32) -> f64 { let obj = crate::object::js_object_alloc(class_id, 2); @@ -54,6 +55,11 @@ fn proof(arr: *mut ArrayHeader) -> Option { // SET // --------------------------------------------------------------------------- +#[test] +fn element_shape_record_keeps_the_hot_table_footprint() { + assert_eq!(std::mem::size_of::(), 24); +} + #[test] fn first_push_of_a_shaped_object_into_an_empty_array_sets_the_invariant() { let _serialized = test_serialize(); @@ -159,11 +165,81 @@ fn an_in_bounds_overwrite_with_a_matching_shape_keeps_the_invariant() { let _serialized = test_serialize(); let arr = built_from_pushes(CLASS_A, 4); let before = proof(arr).expect("proven"); + let exact_hits_before = test_exact_shape_store_hits(); js_array_set_f64(arr, 2, instance(CLASS_A)); let after = proof(arr).expect("a same-class overwrite must keep the proof"); assert_eq!(after.class_id, CLASS_A); assert_eq!(after.verified_len, 4); assert_eq!(after.epoch, before.epoch, "the proof itself is unchanged"); + assert!( + test_exact_shape_store_hits() > exact_hits_before, + "the already-validated exact shape should avoid another descriptor-table probe" + ); +} + +#[test] +fn resolved_dense_pointer_overwrite_keeps_or_retires_element_shape_exactly() { + let _serialized = test_serialize(); + let arr = built_from_pushes(CLASS_A, 3); + let before = proof(arr).expect("proven"); + let fast_hits_before = crate::array::indexing::test_strict_dense_pointer_overwrite_hits(); + + assert_eq!( + crate::array::indexing::try_strict_dense_index_set(arr, 1, instance(CLASS_A)), + Some(arr) + ); + assert!( + crate::array::indexing::test_strict_dense_pointer_overwrite_hits() > fast_hits_before, + "an existing object-over-object slot must take the resolved pointer path" + ); + let after = proof(arr).expect("a same-class resolved overwrite must keep the proof"); + assert_eq!(after, before); + + assert_eq!( + crate::array::indexing::try_strict_dense_index_set(arr, 1, instance(CLASS_B)), + Some(arr) + ); + assert!( + proof(arr).is_none(), + "the pointer-over-pointer layout shortcut must still retire a mismatched element shape" + ); +} + +#[test] +fn a_same_class_different_exact_shape_keeps_the_class_level_invariant() { + let _serialized = test_serialize(); + let arr = built_from_pushes(CLASS_SAME_CLASS_VARIANT, 4); + let before = proof(arr).expect("proven"); + let variant = instance(CLASS_SAME_CLASS_VARIANT); + let obj = (variant.to_bits() & crate::value::POINTER_MASK) as *mut crate::object::ObjectHeader; + let original_shape = unsafe { (*obj).parent_class_id }; + unsafe { crate::object::shapes::transition_object_shape_semantics(obj) }; + assert_ne!(unsafe { (*obj).parent_class_id }, original_shape); + assert_eq!( + element_identity_of_bits(variant.to_bits()).map(|identity| identity.0), + Some(CLASS_SAME_CLASS_VARIANT), + "the complete fallback classifier must retain same-class ordinary objects" + ); + let record = record_for(arr as usize).expect("live class proof record"); + assert!( + element_matches_record(variant.to_bits(), record), + "a different exact shape must fall back to the class-level classifier" + ); + + js_array_set_f64(arr, 2, variant); + unsafe { + assert!( + test_element_shape_bit_set(arr), + "store must not clear the authority bit" + ) + }; + let retained = record_for(arr as usize).expect("store must retain the class proof record"); + assert_eq!(retained.verified_len, unsafe { (*arr).length }); + assert_eq!(u64::from(retained.generation), class_shape_generation()); + let after = proof(arr).expect("same class with a different shape remains class-homogeneous"); + assert_eq!(after.class_id, CLASS_SAME_CLASS_VARIANT); + assert_eq!(after.verified_len, 4); + assert_eq!(after.epoch, before.epoch); } #[test] diff --git a/crates/perry-runtime/src/array/header.rs b/crates/perry-runtime/src/array/header.rs index 6c21dfeef3..d83a7aa059 100644 --- a/crates/perry-runtime/src/array/header.rs +++ b/crates/perry-runtime/src/array/header.rs @@ -1841,6 +1841,35 @@ pub(crate) unsafe fn note_array_slot(arr: *mut ArrayHeader, index: usize, value_ crate::gc::runtime_write_barrier_slot(arr as usize, slot, value_bits); } +/// [`note_array_slot`] for a live plain Array whose flag word was already +/// read by the caller's receiver guard. This preserves the exact store, +/// numeric-layout, element-shape, per-slot-layout, and barrier sequence +/// without redispatching through `array_numeric_layout` merely to recover the +/// same raw-f64 bits. +/// +/// # Safety +/// +/// `arr` must be a live, forwarding-resolved `GC_TYPE_ARRAY`; `index` must be +/// inside its allocation, and `flags` must be the current preceding +/// `GcHeader::_reserved` word with no intervening safepoint. +#[inline] +pub(crate) unsafe fn note_array_slot_resolved_flags( + arr: *mut ArrayHeader, + index: usize, + value: f64, + flags: u16, +) { + let value = canonicalize_array_numeric_store_value_from_flags(flags, value); + let mut value_bits = value.to_bits(); + let slot_ptr = array_elements_ptr(arr).add(index); + let old_bits = std::ptr::read(slot_ptr); + std::ptr::write(slot_ptr, value_bits); + value_bits = note_array_numeric_index_write(arr, index, value_bits); + crate::gc::layout_note_slot_aware(arr as usize, index, value_bits, old_bits); + let slot = slot_ptr as usize; + crate::gc::runtime_write_barrier_slot(arr as usize, slot, value_bits); +} + #[inline] pub(crate) unsafe fn note_array_slot_layout_only( arr: *mut ArrayHeader, diff --git a/crates/perry-runtime/src/array/indexing.rs b/crates/perry-runtime/src/array/indexing.rs index efde64e781..f42f18a3a3 100644 --- a/crates/perry-runtime/src/array/indexing.rs +++ b/crates/perry-runtime/src/array/indexing.rs @@ -79,6 +79,18 @@ pub(crate) fn test_swap_array_index_fast_path_invalidated(value: u8) -> u8 { PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED.swap(value, Ordering::Relaxed) } +#[cfg(test)] +thread_local! { + static STRICT_DENSE_POINTER_OVERWRITE_HITS: std::cell::Cell = const { + std::cell::Cell::new(0) + }; +} + +#[cfg(test)] +pub(crate) fn test_strict_dense_pointer_overwrite_hits() -> u64 { + STRICT_DENSE_POINTER_OVERWRITE_HITS.with(std::cell::Cell::get) +} + /// Record (if `obj` is the canonical `Object.prototype`) that it now carries /// an indexed property. Called from the object index-write / numeric /// defineProperty paths; cheap (relaxed loads + compare). @@ -618,6 +630,14 @@ pub extern "C" fn js_array_length(arr: *const ArrayHeader) -> u32 { && ((*gc_header).obj_type == crate::gc::GC_TYPE_OBJECT || (*gc_header).obj_type == crate::gc::GC_TYPE_CLOSURE) { + if let Some(v) = crate::array::subclass::array_subclass_fast_length_raw(raw_ptr) { + let n = crate::builtins::js_number_coerce(v); + return if n.is_nan() || n <= 0.0 { + 0 + } else { + n.min(u32::MAX as f64) as u32 + }; + } let key = crate::string::js_string_from_bytes(b"length".as_ptr(), 6); let v = crate::object::js_object_get_field_by_name_f64( raw_ptr as *const crate::object::ObjectHeader, @@ -677,6 +697,9 @@ pub extern "C" fn js_array_get_f64_unchecked(arr: *const ArrayHeader, index: u32 let cleaned = clean_arr_ptr(arr); if cleaned.is_null() { // #7574: array-like OBJECT receiver — see `js_array_get_f64`. + if let Some(value) = crate::array::subclass::array_subclass_fast_index_get_raw(arr, index) { + return value; + } if crate::array::subclass::array_object_receiver(arr).is_some() { return js_array_get_f64(arr, index); } @@ -840,6 +863,9 @@ pub extern "C" fn js_array_get_f64(arr: *const ArrayHeader, index: u32) -> f64 { // #7574: `a[i]` on a `class X extends Array` instance held in a // `T[]`-annotated binding. Read the object's indexed property through // the spec-generic `Get`, not the `ObjectHeader` words. + if let Some(value) = crate::array::subclass::array_subclass_fast_index_get_raw(arr, index) { + return value; + } if let Some(recv) = crate::array::subclass::array_object_receiver(arr) { return crate::array::subclass::array_object_index_get(recv, index); } @@ -1158,10 +1184,166 @@ pub extern "C" fn js_array_set_f64_extend_strict( index: u32, value: f64, ) -> *mut ArrayHeader { + if let Some(resolved) = try_strict_dense_index_set(arr, index, value) { + return resolved; + } array_strict_index_write_guard(arr, index); js_array_set_f64_extend(arr, index, value) } +/// Complete a strict existing-slot assignment without redispatching +/// the receiver through the guard, extending setter, layout classifier, and +/// write barrier independently. +/// +/// This is deliberately narrower than the ordinary dense-array setter: +/// +/// - a plain Array must have an existing own dense slot (not a hole); a dense +/// raw-f64 Number-to-Number overwrite takes the metadata-free sub-path; +/// - an object-backed Array subclass must prove the exact dense shape and a +/// writable existing numeric slot through its own guarded fast path; and +/// - frozen arrays, descriptors, growth, holes, and forwarding failures +/// decline to the unchanged strict implementation. +/// +/// General values retain the ordinary numeric-layout note, element-shape note, +/// slot-layout update, and write barrier, but reuse the receiver flags already +/// read here instead of reclassifying the Array in each layer. +#[inline] +pub(crate) fn try_strict_dense_index_set( + arr: *mut ArrayHeader, + index: u32, + value: f64, +) -> Option<*mut ArrayHeader> { + let value_bits = value.to_bits(); + let number = value_bits_to_number(value_bits); + // Complete the overwhelmingly common Number-to-Number ordinary-Array + // overwrites from the live header and slot themselves. The generated + // guarded store already uses this exact magnitude/header discipline; this + // tier is for dynamic-key sites that reach the feedback helper instead + // (notably both sparse-set number moves and ECS archetype pointer moves). + // + // Both values are constructively classified as Numbers, so the store + // cannot add or remove a GC edge, change the per-slot pointer mask, demote + // a unique string, or require a write barrier. Requiring an existing own + // non-hole slot plus the same frozen/descriptor/prototype guards as the + // resolved path preserves every observable assignment case. Forwarding, + // growth, sparse holes, accessors and non-number values retain the complete + // implementation below. + if number.is_some() { + if let Some(header) = unsafe { crate::value::addr_class::try_read_gc_header(arr as usize) } + { + if header.obj_type == crate::gc::GC_TYPE_ARRAY + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + && header._reserved + & (crate::gc::OBJ_FLAG_FROZEN | crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS) + == 0 + && super::PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED.load(Ordering::Relaxed) == 0 + { + unsafe { + let length = (*arr).length; + let capacity = (*arr).capacity; + if index < length && length <= capacity && length <= 100_000_000 { + let elements = (arr as *mut u8) + .add(std::mem::size_of::()) + .cast::(); + let slot = elements.add(index as usize); + let old = ptr::read(slot); + let old_bits = old.to_bits(); + if let Some(new_number) = number.filter(|_| { + old_bits != crate::value::TAG_HOLE + && value_bits_to_number(old_bits).is_some() + }) { + // GC_STORE_AUDIT(POINTER_FREE): old and new were + // constructively decoded as ECMAScript Numbers. + ptr::write(slot, new_number); + return Some(arr); + } + } + } + } + } + } + + // Object-backed Array subclasses are rejected by `clean_arr_ptr_mut`. + // Ask their exact shape/descriptor proof first so a hit performs only its + // one validated-object resolution rather than two failed Array cleans. + if let Some(number) = number { + if crate::array::subclass::array_subclass_fast_index_set_raw(arr, index, number) { + return Some(arr); + } + } + + let resolved = clean_arr_ptr_mut(arr); + if resolved.is_null() { + return None; + } + let flags = unsafe { array_object_flags_resolved(resolved) }; + if flags & (crate::gc::OBJ_FLAG_FROZEN | crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS) != 0 { + return None; + } + if super::PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED.load(Ordering::Relaxed) != 0 { + return None; + } + unsafe { + if index >= (*resolved).length || index >= (*resolved).capacity { + return None; + } + let elements = (resolved as *mut u8).add(std::mem::size_of::()) as *mut f64; + if flags & crate::gc::GC_ARRAY_RAW_F64_LAYOUT != 0 { + if let Some(number) = number { + // GC_STORE_AUDIT(POINTER_FREE): `GC_ARRAY_RAW_F64_LAYOUT` + // proves the retired value is a Number, and + // `value_bits_to_number` constructively produced its + // replacement above. + ptr::write(elements.add(index as usize), number); + return Some(resolved); + } + } + + // An in-range hole is not an existing own property: a prototype + // accessor may intercept it and sealed/non-extensible Arrays may + // reject creating it. The unchanged strict fallback owns that case. + let slot = elements.add(index as usize); + let old_bits = ptr::read(slot).to_bits(); + if old_bits == crate::value::TAG_HOLE { + return None; + } + + let pointer_tag = crate::value::POINTER_TAG; + let pointer_mask = crate::value::POINTER_MASK; + let raw_numeric_flags = + crate::gc::GC_ARRAY_RAW_F64_LAYOUT | crate::gc::GC_ARRAY_RAW_F64_HOLES; + if flags & raw_numeric_flags == 0 + && value_bits & crate::value::TAG_MASK == pointer_tag + && value_bits & pointer_mask != 0 + && old_bits & crate::value::TAG_MASK == pointer_tag + && old_bits & pointer_mask != 0 + { + #[cfg(test)] + STRICT_DENSE_POINTER_OVERWRITE_HITS.with(|hits| hits.set(hits.get().wrapping_add(1))); + // GC_STORE_AUDIT(BARRIERED): old and new are constructively + // pointer-bearing, so the slot mask is unchanged. Maintain the + // independent element proof and the mandatory generational/SATB + // edge. + ptr::write(slot, value); + crate::array::element_shape::note_element_store_resolved_flags( + resolved, + index as usize, + value_bits, + flags, + ); + crate::gc::runtime_write_barrier_slot(resolved as usize, slot as usize, value_bits); + return Some(resolved); + } + + // A heap string assigned into an existing slot becomes shared before + // the store, exactly as in `js_array_set_f64_extend`. This call does + // not allocate or safepoint, so the resolved receiver remains live. + crate::string::js_string_addref_if_heap_string(value); + crate::array::note_array_slot_resolved_flags(resolved, index as usize, value, flags); + } + Some(resolved) +} + /// Set an element in an array by index, extending the array if needed /// Returns the (possibly reallocated) array pointer /// This mimics JavaScript's arr[i] = value behavior @@ -1180,6 +1362,9 @@ pub extern "C" fn js_array_set_f64_extend( // `ObjectHeader.keys_array` / `.meta`. Run the object `[[Set]]` plus // the Array-exotic `length` maintenance, and return the ORIGINAL // receiver so the caller's realloc write-back keeps the binding. + if crate::array::subclass::array_subclass_fast_index_set_raw(arr, index, value) { + return arr; + } if let Some(recv) = crate::array::subclass::array_object_receiver(arr) { crate::array::subclass::array_object_index_set(recv, index, value); return arr; diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index 488bf0c444..c4eaf45e9c 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -174,6 +174,8 @@ pub(crate) use indexing::test_swap_array_index_fast_path_invalidated; // store needs for a `class X extends Array` receiver. pub(crate) use self::subclass::{ array_object_set_length, array_subclass_fast_index_get, array_subclass_fast_length, + array_subclass_fast_length_with_ic, array_subclass_named_prefix_token_for_slot, + array_subclass_tail_descriptors_are_plain, clear_array_subclass_named_prefix_token, clear_packed_subclass_numeric_proof, is_array_subclass_class_id, is_array_subclass_value, note_array_subclass_index_write, note_packed_subclass_spill_store, }; @@ -192,9 +194,9 @@ pub use self::jsvalue_api::{ pub(crate) use self::push_pop::guard_writable_length; pub use self::push_pop::{ js_array_delete, js_array_grow, js_array_numeric_push_f64_unboxed, js_array_pop_f64, - js_array_push_f64, js_array_push_hole, js_array_push_spread_f64, js_array_set_length, - js_array_set_length_strict, js_array_shift_f64, js_array_unshift_f64, js_array_unshift_jsvalue, - js_array_unshift_variadic, + js_array_push_f64, js_array_push_hole, js_array_push_spread_f64, js_array_push_u31_with_length, + js_array_set_length, js_array_set_length_strict, js_array_shift_f64, js_array_unshift_f64, + js_array_unshift_jsvalue, js_array_unshift_variadic, }; pub use self::reduce_right::js_array_reduce_right; pub use self::search::{ @@ -222,10 +224,11 @@ pub(crate) use self::header::{ canonicalize_array_numeric_store_value_from_flags, clean_arr_ptr, clean_arr_ptr_mut, clear_array_numeric_layout, clear_array_numeric_layout_ptr, gc_element_slot_range, mark_array_layout_unknown, mark_array_raw_f64_holes_fresh, normalize_array_receiver, - note_array_slot, note_array_slot_layout_only, rebuild_array_layout, rebuild_array_layout_exact, - refresh_array_numeric_layout, replay_array_growth_write_barriers, set_array_numeric_layout, - store_array_slot, transfer_array_numeric_layout, typed_array_receiver, value_bits_to_number, - NumericArrayLayout, MIN_ARRAY_CAPACITY, + note_array_slot, note_array_slot_layout_only, note_array_slot_resolved_flags, + rebuild_array_layout, rebuild_array_layout_exact, refresh_array_numeric_layout, + replay_array_growth_write_barriers, set_array_numeric_layout, store_array_slot, + transfer_array_numeric_layout, typed_array_receiver, value_bits_to_number, NumericArrayLayout, + MIN_ARRAY_CAPACITY, }; // Sole caller is the regex-engine-gated `regex::exec_array`, so the helper and diff --git a/crates/perry-runtime/src/array/push_pop.rs b/crates/perry-runtime/src/array/push_pop.rs index 78b76b889a..87260753b5 100644 --- a/crates/perry-runtime/src/array/push_pop.rs +++ b/crates/perry-runtime/src/array/push_pop.rs @@ -1,6 +1,7 @@ //! push / pop / shift / unshift / set_length / delete + grow primitive. use super::*; use std::ptr; +use std::sync::atomic::Ordering; /// `pop`/`shift`/`push`/`unshift` on a frozen array perform a `Set`/`Delete` /// with `Throw = true` internally (ECMA-262 §23.1.3.*), so a non-writable @@ -662,13 +663,24 @@ pub extern "C" fn js_array_push_f64(arr: *mut ArrayHeader, value: f64) -> *mut A // receiver so codegen's realloc write-back leaves the binding pointing // at the instance (returning a fresh empty array here is what made the // push look silently dropped). + if crate::array::subclass::array_subclass_fast_push_one_raw(arr, value).is_some() { + return arr; + } if let Some(recv) = crate::array::subclass::array_object_receiver(arr) { crate::array::subclass::array_object_method(recv, "push", &[value]); return arr; } return js_array_alloc(0); } - let arr = cleaned; + unsafe { js_array_push_f64_resolved(cleaned, value) } +} + +/// Push into a live, forwarding-resolved plain Array. The caller owns all +/// receiver-brand and Proxy handling; keeping this core separate lets the +/// guarded u31 entry reuse the resolved header instead of classifying it a +/// second time through `js_array_push_f64`. +#[inline] +unsafe fn js_array_push_f64_resolved(arr: *mut ArrayHeader, value: f64) -> *mut ArrayHeader { // One resolved header word answers every policy/layout question below. // Re-entering the public helpers here used to run `clean_arr_ptr` (and its // allocator-ownership proof) once for each individual bit test. @@ -682,25 +694,103 @@ pub extern "C" fn js_array_push_f64(arr: *mut ArrayHeader, value: f64) -> *mut A if flags & (crate::gc::OBJ_FLAG_SEALED | crate::gc::OBJ_FLAG_NO_EXTEND) != 0 { return arr; } - unsafe { - let length = (*arr).length; - let capacity = (*arr).capacity; + let length = (*arr).length; + let capacity = (*arr).capacity; + + if length >= capacity { + return js_array_push_f64_grow(arr, length, value); + } + + let value = canonicalize_array_numeric_store_value_from_flags(flags, value); + let value_bits = value.to_bits(); + let elements_ptr = (arr as *mut u8).add(std::mem::size_of::()) as *mut f64; + // GC_STORE_AUDIT(BARRIERED): push slot is immediately recorded via note_array_slot. + ptr::write(elements_ptr.add(length as usize), value); + note_array_slot(arr, length as usize, value_bits); + (*arr).length = length + 1; + arr +} - if length >= capacity { - return js_array_push_f64_grow(arr, length, value); +/// Single-element push for a value constructively proved by generated code to +/// be a nonnegative signed-i32 Number. Besides avoiding value classification, +/// this entry returns the semantic push result through `new_length`, so the +/// caller does not immediately redispatch `js_array_length` on the receiver. +/// +/// Every unproved receiver state retains the complete public fallback. In +/// particular Proxy traps, descriptor mutations, Array-subclass integrity +/// flags, and first-seen tail transitions all run the same generic algorithms +/// as `js_array_push_f64`. +#[no_mangle] +pub extern "C" fn js_array_push_u31_with_length( + arr: *mut ArrayHeader, + value: u32, + new_length: *mut u32, +) -> *mut ArrayHeader { + let number = f64::from(value); + + // Generated callers hand this entry a freshly decoded JS receiver. The + // ordinary-array and object-backed Array-subclass headers are therefore + // safe to classify with the same magnitude-checked live-header probe used + // by the generated Array element tiers. Doing that before the complete + // forwarding/allocator-ownership resolver matters for the ECS kernels: + // every plain `SparseSet.packed.push(id)` used to pay a tracked-allocation + // lookup, and every Array-subclass push paid that lookup only to learn that + // it was not an `ArrayHeader` before repeating the header read in the + // subclass path. + // + // Only a non-forwarded, sane ordinary Array is consumed here. Forwarding + // stubs, lazy/external receivers and every other brand retain + // `clean_arr_ptr_mut` below; the resolved helper retains the complete + // frozen/sealed/descriptor/grow and GC-bookkeeping behavior. + let direct_plain = unsafe { crate::value::addr_class::try_read_gc_header(arr as usize) } + .filter(|header| { + header.obj_type == crate::gc::GC_TYPE_ARRAY + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + }) + .and_then(|_| unsafe { + let length = (*arr).length; + let capacity = (*arr).capacity; + (length <= capacity && length <= 100_000_000).then_some(arr) + }); + if let Some(cleaned) = direct_plain { + let pushed = unsafe { js_array_push_f64_resolved(cleaned, number) }; + if !new_length.is_null() { + unsafe { *new_length = (*pushed).length }; } + return pushed; + } - let value = canonicalize_array_numeric_store_value_from_flags(flags, value); - let value_bits = value.to_bits(); - let elements_ptr = (arr as *mut u8).add(std::mem::size_of::()) as *mut f64; - // GC_STORE_AUDIT(BARRIERED): push slot is immediately recorded via note_array_slot. - ptr::write(elements_ptr.add(length as usize), value); - note_array_slot(arr, length as usize, value_bits); - (*arr).length = length + 1; - arr + if let Some(length) = crate::array::subclass::array_subclass_fast_push_u31_raw(arr, value) { + if !new_length.is_null() { + unsafe { *new_length = length as u32 }; + } + return arr; + } + + let cleaned = clean_arr_ptr_mut(arr); + if !cleaned.is_null() { + let pushed = unsafe { js_array_push_f64_resolved(cleaned, number) }; + if !new_length.is_null() { + unsafe { *new_length = (*pushed).length }; + } + return pushed; + } + + let pushed = js_array_push_f64(arr, number); + if !new_length.is_null() { + unsafe { *new_length = crate::array::js_array_length(pushed) }; } + pushed } +#[cfg(feature = "keepalive-anchors")] +#[used] +static KEEP_JS_ARRAY_PUSH_U31_WITH_LENGTH: extern "C" fn( + *mut ArrayHeader, + u32, + *mut u32, +) -> *mut ArrayHeader = js_array_push_u31_with_length; + #[no_mangle] pub extern "C" fn js_array_push_hole(arr: *mut ArrayHeader) -> *mut ArrayHeader { js_array_push_f64(arr, f64::from_bits(crate::value::TAG_HOLE)) @@ -831,9 +921,53 @@ pub extern "C" fn js_array_push_spread_f64( #[no_mangle] pub extern "C" fn js_array_pop_f64(arr: *mut ArrayHeader) -> f64 { const TAG_UNDEFINED_F64: f64 = f64::from_bits(0x7FFC_0000_0000_0001u64); + // The common plain-Array case can be completed from one live header read. + // `clean_arr_ptr_mut` is intentionally much stronger: it proves allocator + // ownership, follows forwarding chains, recognizes lazy/external storage, + // and validates several foreign receiver families. That proof is needed + // by the generic public entry but redundant after the guards below have + // established the exact non-forwarded Array layout. + // + // A dense own final slot makes Get/Delete/Set(length) unobservable. Any + // integrity/descriptor flag, indexed-prototype invalidation, hole, + // forwarding stub, empty receiver, or malformed bound declines to the + // unchanged algorithms below. Leaving the retired physical word intact + // matches the existing dense branch later in this function; the logical + // length is the GC trace bound and a later push overwrites the word before + // publishing the larger length. + if let Some(header) = unsafe { crate::value::addr_class::try_read_gc_header(arr as usize) } { + let guarded_flags = crate::gc::OBJ_FLAG_FROZEN + | crate::gc::OBJ_FLAG_SEALED + | crate::gc::OBJ_FLAG_NO_EXTEND + | crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS; + if header.obj_type == crate::gc::GC_TYPE_ARRAY + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + && header._reserved & guarded_flags == 0 + && super::PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED.load(Ordering::Relaxed) == 0 + { + unsafe { + let length = (*arr).length; + let capacity = (*arr).capacity; + if length != 0 && length <= capacity && length <= 100_000_000 { + let new_length = length - 1; + let elements = (arr as *mut u8) + .add(std::mem::size_of::()) + .cast::(); + let value = ptr::read(elements.add(new_length as usize)); + if value.to_bits() != crate::value::TAG_HOLE { + (*arr).length = new_length; + return value; + } + } + } + } + } // Borrowed array-like receiver (`obj.pop = Array.prototype.pop; obj.pop()`): // the thunk hands this dense helper the plain object pointer. Run the // spec-generic engine instead of reading the object as an `ArrayHeader`. + if let Some(value) = crate::array::subclass::array_subclass_fast_pop_raw(arr) { + return value; + } if let Some(recv) = crate::array::plain_object_value(arr) { return crate::array::generic_object_pop(recv); } diff --git a/crates/perry-runtime/src/array/subclass.rs b/crates/perry-runtime/src/array/subclass.rs index 9c5d60197a..8118e4e4d0 100644 --- a/crates/perry-runtime/src/array/subclass.rs +++ b/crates/perry-runtime/src/array/subclass.rs @@ -45,7 +45,11 @@ const PACKED_NUMERIC_META_MASK: u64 = PACKED_NUMERIC_META_VALID // The cache stores no heap pointer, so it is not a GC root. ShapeIds are never // reused, and the class id prevents an unrelated class with the same ordered // keys from borrowing the Array-subclass proof. -const DENSE_SUBCLASS_CACHE_SLOTS: usize = 256; +// Lifecycle-heavy Array subclasses revisit one shape per historical length. +// Keep the common 1k-entity lattice resident so an allocation-free tail +// transition does not fall back to an O(length) ordered-key rescan next cycle. +const DENSE_SUBCLASS_CACHE_SLOTS: usize = 16384; +const ARRAY_SUBCLASS_NAMED_PREFIX_TOKEN_BIT: u64 = 1 << 63; struct DenseSubclassCacheEntry { /// Even while stable, odd while a colliding writer publishes a payload. @@ -80,6 +84,29 @@ struct DenseSubclassLayout { live_inline_slots: u32, } +/// A live, non-forwarded ordinary object whose `GcHeader` has already been +/// validated. Keeping the integrity flags beside the pointer lets a hot +/// Array-subclass mutation reuse that single header read for brand, layout, +/// and frozen/sealed/no-extend checks. +#[derive(Clone, Copy)] +struct ValidatedObjectReceiver { + object: *const ObjectHeader, + object_flags: u16, +} + +/// Read the per-instance prototype-divergence bit after the caller has already +/// proved a live, non-forwarded `GC_TYPE_OBJECT` receiver. +/// +/// The public prototype-chain predicate accepts arbitrary addresses and must +/// re-run buffer/heap/header classification before touching `ObjectHeader`. +/// Dense Array-subclass paths have just completed that proof, so repeating it +/// ahead of every receiver-local layout-cache hit is both redundant and hot. +#[inline(always)] +unsafe fn validated_object_has_prototype_override(obj: *const ObjectHeader) -> bool { + let meta = (*obj).meta; + !meta.is_null() && (*meta).flags & crate::object::OBJECT_META_FLAG_PROTO_OVERRIDE != 0 +} + #[inline(always)] fn dense_cache_key(class_id: u32, shape_id: u32) -> u64 { ((class_id as u64) << 32) | shape_id as u64 @@ -151,6 +178,46 @@ fn publish_dense_layout(key: u64, layout: DenseSubclassLayout) { .store(sequence.wrapping_add(2), Ordering::Release); } +/// Receiver-local front cache for the current Array-subclass layout. Unlike +/// the process-wide collision cache above, these scalar words move with the +/// owner and need no atomics: Perry heap objects are agent-local, and workers +/// deep-copy rather than concurrently share ObjectHeaders. +#[inline(always)] +unsafe fn owner_cached_dense_layout(obj: *const ObjectHeader) -> Option { + let meta = (*obj).meta; + if meta.is_null() { + return None; + } + let key = dense_cache_key((*obj).class_id, (*obj).parent_class_id); + if key == 0 || (*meta).array_subclass_dense_key != key { + return None; + } + let slots = (*meta).array_subclass_dense_slots; + let bounds = (*meta).array_subclass_dense_bounds; + Some(DenseSubclassLayout { + length_slot: (slots >> 32) as u32, + element_base: slots as u32, + dense_prefix_len: bounds as u32, + live_inline_slots: (bounds >> 32) as u32, + }) +} + +#[inline(always)] +unsafe fn publish_owner_dense_layout(obj: *const ObjectHeader, layout: DenseSubclassLayout) { + let meta = (*obj).meta; + if meta.is_null() { + return; + } + // Publish the key last. This is single-agent state, but retaining the + // payload-before-authority ordering also makes an accidental diagnostic + // read fail closed rather than combine a new key with old bounds. + (*meta).array_subclass_dense_slots = + ((layout.length_slot as u64) << 32) | layout.element_base as u64; + (*meta).array_subclass_dense_bounds = + ((layout.live_inline_slots as u64) << 32) | layout.dense_prefix_len as u64; + (*meta).array_subclass_dense_key = dense_cache_key((*obj).class_id, (*obj).parent_class_id); +} + fn decimal_u32<'a>(mut value: u32, buf: &'a mut [u8; 10]) -> &'a [u8] { let mut start = buf.len(); loop { @@ -170,7 +237,7 @@ unsafe fn build_dense_layout(obj: *const ObjectHeader) -> Option Option Option Option Option<(*const ObjectHeader, DenseSubclassLayout)> { - let js = JSValue::from_bits(value.to_bits()); - if !js.is_pointer() { - return None; - } - let obj = js.as_pointer::(); - let header = unsafe { crate::value::addr_class::try_read_gc_header(obj as usize)? }; +fn validated_object_receiver(raw: usize) -> Option { + let header = unsafe { crate::value::addr_class::try_read_gc_header(raw)? }; if header.obj_type != crate::gc::GC_TYPE_OBJECT || header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 { return None; } + Some(ValidatedObjectReceiver { + object: raw as *const ObjectHeader, + object_flags: header._reserved, + }) +} + +#[inline] +fn validated_object_receiver_for_value(value: f64) -> Option { + let js = JSValue::from_bits(value.to_bits()); + js.is_pointer() + .then(|| validated_object_receiver(js.as_pointer::() as usize)) + .flatten() +} + +/// Resolve the cached dense layout after the caller has proved that `obj` is +/// a live, non-forwarded ordinary object. Every rejected Array-subclass brand, +/// descriptor, hole, or prototype case returns `None`. +#[inline] +fn dense_layout_for_validated_object(obj: *const ObjectHeader) -> Option { // This is per receiver, not per ShapeId. A cached layout built before // Object.setPrototypeOf must not let this object borrow the old proof. - if crate::object::prototype_chain::object_has_prototype_override(obj as usize) { + if unsafe { validated_object_has_prototype_override(obj) } { return None; } + if let Some(layout) = unsafe { owner_cached_dense_layout(obj) } { + return Some(layout); + } let (class_id, shape_id) = unsafe { ((*obj).class_id, (*obj).parent_class_id) }; let key = dense_cache_key(class_id, shape_id); let layout = cached_dense_layout(key).or_else(|| { @@ -276,7 +365,244 @@ fn dense_layout_for_value(value: f64) -> Option<(*const ObjectHeader, DenseSubcl publish_dense_layout(key, layout); Some(layout) })?; - Some((obj, layout)) + unsafe { publish_owner_dense_layout(obj, layout) }; + Some(layout) +} + +/// Resolve a live Array-subclass object and its cached dense layout. Every +/// rejected brand, forwarding, descriptor, hole, or prototype case returns +/// `None`; callers retain their existing fully generic fallback. +#[inline] +fn dense_layout_for_value(value: f64) -> Option<(*const ObjectHeader, DenseSubclassLayout)> { + let receiver = validated_object_receiver_for_value(value)?; + let layout = dense_layout_for_validated_object(receiver.object)?; + Some((receiver.object, layout)) +} + +/// Return the class-wide identity of a proved Array-subclass named prefix. +/// +/// Perry's object-backed Array subclasses append numeric keys to the same +/// ordered keys array that holds their declared fields. Consequently every +/// numeric tail length has a different ShapeId, even though the slots before +/// `"0"` are byte-for-byte the class allocation shape. A property-read PIC +/// can safely keep using one declared-field slot across those tail shapes only +/// after this function proves all of the following: +/// +/// - the receiver is a live ordinary Array-subclass instance; +/// - its prefix matches the class's registered allocation keys exactly; +/// - the only additional named keys are the canonical Array-subclass +/// `length` and `fill` slots; +/// - every remaining key is the complete dense numeric suffix already proved +/// by `DenseSubclassLayout`; and +/// - `requested_slot` lies before that numeric suffix. +/// +/// The token is stored on the object, not in a pointer-keyed side table, so it +/// moves with the receiver. Generic structural/semantic shape publication +/// clears it via `clear_array_subclass_named_prefix_token`; exact learned +/// numeric-tail transitions deliberately do not. +pub(crate) unsafe fn array_subclass_named_prefix_token_for_slot( + obj: *const ObjectHeader, + requested_slot: usize, +) -> u64 { + if obj.is_null() { + return 0; + } + let header = match crate::value::addr_class::try_read_gc_header(obj as usize) { + Some(header) => header, + None => return 0, + }; + if header.obj_type != crate::gc::GC_TYPE_OBJECT + || header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + { + return 0; + } + let meta = (*obj).meta; + if meta.is_null() { + return 0; + } + let class_id = (*obj).class_id; + if class_id == 0 || !is_array_subclass_class_id(class_id) { + return 0; + } + let Some((declared_keys, declared_count)) = + crate::object::registered_class_keys_array(class_id) + else { + return 0; + }; + if declared_keys.is_null() { + return 0; + } + let shape_id = (*obj).parent_class_id; + let cache_key = dense_cache_key(class_id, shape_id); + let layout = cached_dense_layout(cache_key).or_else(|| { + let layout = build_dense_layout(obj)?; + publish_dense_layout(cache_key, layout); + Some(layout) + }); + let Some(layout) = layout else { + return 0; + }; + // Descriptor-bearing Array subclasses cannot use the ordinary exact-shape + // raw-load PIC even while empty: their unrelated `length` descriptor sends + // them through the descriptor arm. Admit the fully validated named prefix + // before the first numeric key exists as well. `element_base` is the first + // prospective numeric slot and `dense_prefix_len == 0` proves there is no + // tail yet; the complete-prefix equality below remains the authority. + if requested_slot >= layout.element_base as usize { + return 0; + } + let cached = (*meta).array_subclass_named_prefix_token; + if cached != 0 { + return cached; + } + + let Some(shape) = crate::object::shapes::object_shape_descriptor(obj) else { + return 0; + }; + if shape.object_kind != crate::object::shapes::ShapeObjectKind::Ordinary { + return 0; + } + let current_keys = shape.keys as usize as *const ArrayHeader; + let (current_slots, current_physical_len) = crate::object::keys_array_dense_slots(current_keys); + let (declared_slots, declared_physical_len) = + crate::object::keys_array_dense_slots(declared_keys as *const ArrayHeader); + let current_count = (shape.logical_key_count as usize).min(current_physical_len); + let declared_count = (declared_count as usize).min(declared_physical_len); + if current_slots.is_null() + || declared_slots.is_null() + || declared_count > current_count + || layout.element_base as usize + layout.dense_prefix_len as usize != current_count + { + return 0; + } + + // Declared slots must occupy the identical prefix positions. Stored object + // keys are heap strings, so string equality is exact even after one keys + // array was cloned and the two pointer words differ after a moving GC. + let mut declared_length_slot = None; + let mut declared_fill = false; + for slot in 0..declared_count { + let current_bits = (*current_slots.add(slot)).to_bits(); + let declared_bits = (*declared_slots.add(slot)).to_bits(); + let current_key = (current_bits & 0x0000_FFFF_FFFF_FFFF) as *const crate::StringHeader; + let declared_key = (declared_bits & 0x0000_FFFF_FFFF_FFFF) as *const crate::StringHeader; + if current_key.is_null() + || declared_key.is_null() + || crate::string::js_string_equals(current_key, declared_key) == 0 + { + return 0; + } + // An unrelated descriptor (notably Array-subclass `length`) must not + // disable direct reads of class-declared data fields. Prove every key + // covered by this class-wide token is not an accessor on THIS object; + // any later descriptor mutation mints a semantic ShapeId and clears + // the token before it becomes observable. + let Some(name) = crate::object::has_own_helpers::str_from_string_header(current_key) else { + return 0; + }; + if name == "length" { + declared_length_slot = Some(slot as u32); + } else if name == "fill" { + declared_fill = true; + } + if crate::object::get_accessor_descriptor(obj as usize, name).is_some() { + return 0; + } + } + + // `js_array_subclass_init` installs two canonical own properties that are + // absent from most class allocation shapes: `length` and the generic + // `fill` method. If a class declared either name, init overwrites its + // existing slot; otherwise the exact missing names must follow the + // declared prefix in that order. Anything else is instance-specific. + let declared_count = declared_count as u32; + let expected_length_slot = if let Some(slot) = declared_length_slot { + slot + } else { + declared_count + }; + if layout.length_slot != expected_length_slot { + return 0; + } + let mut expected_runtime_names: [&[u8]; 2] = [&[]; 2]; + let mut expected_runtime_count = 0usize; + if declared_length_slot.is_none() { + expected_runtime_names[expected_runtime_count] = b"length"; + expected_runtime_count += 1; + } + if !declared_fill { + expected_runtime_names[expected_runtime_count] = b"fill"; + expected_runtime_count += 1; + } + if layout.element_base != declared_count.saturating_add(expected_runtime_count as u32) { + return 0; + } + for (offset, expected) in expected_runtime_names[..expected_runtime_count] + .iter() + .enumerate() + { + let slot = declared_count as usize + offset; + let bits = (*current_slots.add(slot)).to_bits(); + let key = (bits & 0x0000_FFFF_FFFF_FFFF) as *const crate::StringHeader; + if key.is_null() + || !crate::string::js_string_key_matches_bytes(JSValue::from_bits(bits), expected) + { + return 0; + } + let Some(name) = crate::object::has_own_helpers::str_from_string_header(key) else { + return 0; + }; + if crate::object::get_accessor_descriptor(obj as usize, name).is_some() { + return 0; + } + } + + let token = ARRAY_SUBCLASS_NAMED_PREFIX_TOKEN_BIT | u64::from(class_id); + (*meta).array_subclass_named_prefix_token = token; + token +} + +/// Retire the named-prefix proof before any generic shape or semantic +/// publication. Value-only field stores leave slot identity unchanged and do +/// not call this; exact numeric-tail shape installs bypass it deliberately. +#[inline] +pub(crate) unsafe fn clear_array_subclass_named_prefix_token(obj: *mut ObjectHeader) { + if obj.is_null() { + return; + } + let meta = (*obj).meta; + if !meta.is_null() { + (*meta).array_subclass_named_prefix_token = 0; + } +} + +/// Test an already-published Array-subclass named-prefix proof against the +/// class expected by a consumer. +/// +/// The token is stronger than an ordinary-object ShapeId-kind query for this +/// purpose: its publisher admitted only a live ordinary instance of this +/// exact Array-subclass class, validated the complete declared prefix and the +/// dense numeric suffix, and stored the class id in the token itself. Generic +/// structural/semantic transitions clear it before publishing a new ShapeId; +/// only the exact learned numeric-tail transitions preserve it. +/// +/// # Safety +/// +/// `obj` must already have been validated as a live, non-forwarded +/// `GC_TYPE_OBJECT`. The helper reads only its inline `meta` edge and scalar +/// token payload. +#[inline(always)] +pub(crate) unsafe fn array_subclass_named_prefix_token_matches_class( + obj: *const ObjectHeader, + class_id: u32, +) -> bool { + if obj.is_null() || class_id == 0 { + return false; + } + let meta = (*obj).meta; + !meta.is_null() + && (*meta).array_subclass_named_prefix_token + == (ARRAY_SUBCLASS_NAMED_PREFIX_TOKEN_BIT | u64::from(class_id)) } /// Clear an established Array-subclass numeric-prefix proof before an owner @@ -526,6 +852,46 @@ pub(crate) fn array_subclass_fast_length(value: f64) -> Option { Some(f64::from_bits(layout_length_value(obj, layout).bits())) } +/// Fast own `length` read that also primes a pointer-free generated-code IC. +/// +/// The three published words are `(identity, length slot, inline bound)`. +/// `identity` is either the exact `(class_id, ShapeId)` pair or the stable +/// Array-subclass named-prefix token used by the dense indexed-read IC. The +/// payload is published before the identity, and no managed pointer escapes +/// into the cache, so moving GC needs neither a root nor a rewrite hook. +#[inline] +pub(crate) fn array_subclass_fast_length_with_ic(value: f64, cache: *mut u64) -> Option { + let (obj, layout) = dense_layout_for_value(value)?; + let result = f64::from_bits(layout_length_value(obj, layout).bits()); + if !cache.is_null() { + let family_token = if crate::object::object_spill_enabled() { + unsafe { array_subclass_named_prefix_token_for_slot(obj, layout.length_slot as usize) } + } else { + 0 + }; + unsafe { + cache.add(1).write(layout.length_slot as u64); + cache.add(2).write(layout.live_inline_slots as u64); + cache.write(if family_token != 0 { + family_token + } else { + dense_cache_key((*obj).class_id, (*obj).parent_class_id) + }); + } + } + Some(result) +} + +#[inline] +pub(crate) fn array_subclass_fast_length_raw(arr: *const ArrayHeader) -> Option { + let raw = (arr as u64 & crate::value::POINTER_MASK) as usize; + let receiver = validated_object_receiver(raw)?; + let layout = dense_layout_for_validated_object(receiver.object)?; + Some(f64::from_bits( + layout_length_value(receiver.object, layout).bits(), + )) +} + /// Guarded dense numeric read for an object-backed Array subclass. The live /// `length` value is checked on every hit, while `dense_prefix_len` caps the /// proof when a length-only grow created holes without changing the shape. @@ -535,6 +901,526 @@ pub(crate) fn array_subclass_fast_index_get(value: f64, index: u32) -> Option Option { + let raw = (arr as u64 & crate::value::POINTER_MASK) as usize; + let receiver = validated_object_receiver(raw)?; + let layout = dense_layout_for_validated_object(receiver.object)?; + dense_index_get_with_layout(receiver.object, layout, index) +} + +#[inline] +unsafe fn dense_slot_exists(obj: *const ObjectHeader, slot: u32, live_inline_slots: u32) -> bool { + if slot < live_inline_slots { + return true; + } + if !crate::object::object_spill_enabled() { + return false; + } + let meta = (*obj).meta; + if meta.is_null() { + return false; + } + let spill = (*meta).spill as *const ArrayHeader; + !spill.is_null() && slot < (*spill).length && slot < (*spill).capacity +} + +#[inline] +unsafe fn store_dense_slot( + obj: *mut ObjectHeader, + slot: u32, + live_inline_slots: u32, + value_bits: u64, +) -> bool { + if slot < live_inline_slots { + crate::object::store_object_field_slot(obj, slot as usize, value_bits); + return true; + } + if !crate::object::object_spill_enabled() { + return false; + } + let meta = (*obj).meta; + if meta.is_null() { + return false; + } + let spill = (*meta).spill as *mut ArrayHeader; + if spill.is_null() || slot >= (*spill).length || slot >= (*spill).capacity { + return false; + } + note_packed_subclass_spill_store(obj, meta); + let elements = (spill as *mut u8) + .add(std::mem::size_of::()) + .cast::(); + ptr::write(elements.add(slot as usize), value_bits); + note_array_slot(spill, slot as usize, value_bits); + true +} + +/// Store a raw Number into a dense Array-subclass slot without changing its +/// pointer-layout metadata. +/// +/// This is deliberately narrower than [`store_dense_slot`]. The caller has +/// already proved either that the slot was outside the predecessor shape or +/// that its old value was also pointer-free, that its physical storage exists, +/// and that the new value is a nonnegative `i32` encoded as raw f64 bits. +/// Consequently the store cannot publish or remove a heap edge, cannot demote +/// a unique string, and cannot change a pointer-layout bit. Skipping the +/// general layout note and write barrier here removes two full metadata +/// pipelines from numeric tail mutation and the hot ECS swap-with-last write. +#[inline] +unsafe fn store_dense_nonpointer_number_slot( + obj: *mut ObjectHeader, + slot: u32, + live_inline_slots: u32, + number: f64, +) -> bool { + let value_bits = number.to_bits(); + if slot < live_inline_slots { + let fields = (obj as *mut u8) + .add(std::mem::size_of::()) + .cast::(); + ptr::write(fields.add(slot as usize), value_bits); + return true; + } + if !crate::object::object_spill_enabled() { + return false; + } + let meta = (*obj).meta; + if meta.is_null() { + return false; + } + let spill = (*meta).spill as *mut ArrayHeader; + if spill.is_null() || slot >= (*spill).length || slot >= (*spill).capacity { + return false; + } + let elements = (spill as *mut u8) + .add(std::mem::size_of::()) + .cast::(); + ptr::write(elements.add(slot as usize), value_bits); + true +} + +#[inline] +unsafe fn clear_retired_dense_slot( + obj: *mut ObjectHeader, + slot: u32, + former_live_inline_slots: u32, +) { + if slot < former_live_inline_slots { + let fields = (obj as *mut u8) + .add(std::mem::size_of::()) + .cast::(); + // The predecessor ShapeId is already installed, so this physical tail + // is outside the object's traced slot range. Clearing it is storage + // hygiene, not publication of a new edge. + ptr::write(fields.add(slot as usize), crate::value::TAG_UNDEFINED); + return; + } + let meta = (*obj).meta; + if meta.is_null() { + return; + } + let spill = (*meta).spill as *mut ArrayHeader; + if spill.is_null() || slot >= (*spill).length { + return; + } + let elements = (spill as *mut u8) + .add(std::mem::size_of::()) + .cast::(); + ptr::write(elements.add(slot as usize), crate::value::TAG_UNDEFINED); + note_array_slot(spill, slot as usize, crate::value::TAG_UNDEFINED); +} + +/// Clear a numeric tail slot after its exact predecessor shape has been +/// installed. The removed value was constructively classified as a +/// nonnegative i32 Number, so replacing it with `undefined` cannot remove or +/// add a heap edge. Unlike the generic helper above this may therefore leave +/// both the object's and its spill buffer's pointer-layout metadata untouched. +#[inline] +unsafe fn clear_retired_dense_numeric_tail_slot( + obj: *mut ObjectHeader, + slot: u32, + former_live_inline_slots: u32, +) { + if slot < former_live_inline_slots { + let fields = (obj as *mut u8) + .add(std::mem::size_of::()) + .cast::(); + ptr::write(fields.add(slot as usize), crate::value::TAG_UNDEFINED); + return; + } + let meta = (*obj).meta; + if meta.is_null() { + return; + } + let spill = (*meta).spill as *mut ArrayHeader; + if spill.is_null() || slot >= (*spill).length { + return; + } + let elements = (spill as *mut u8) + .add(std::mem::size_of::()) + .cast::(); + ptr::write(elements.add(slot as usize), crate::value::TAG_UNDEFINED); +} + +/// Prove once, while learning an exact semantic shape transition, that neither +/// `length` nor the appended numeric property has custom mutation semantics. +/// Descriptor installation/removal mints a new ShapeId, so a later exact cache +/// hit can consume this proof without rebuilding decimal keys and probing two +/// descriptor maps on every push/pop. +pub(crate) fn array_subclass_tail_descriptors_are_plain( + obj: *const ObjectHeader, + index: u32, +) -> bool { + if !crate::object::object_has_descriptors(obj as usize) { + return true; + } + if crate::object::get_accessor_descriptor(obj as usize, "length").is_some() + || crate::object::get_property_attrs(obj as usize, "length") + .is_some_and(|attrs| !attrs.writable()) + { + return false; + } + let mut decimal = [0u8; 10]; + let bytes = decimal_u32(index, &mut decimal); + let key = unsafe { std::str::from_utf8_unchecked(bytes) }; + if crate::object::get_accessor_descriptor(obj as usize, key).is_some() { + return false; + } + crate::object::get_property_attrs(obj as usize, key).is_none() +} + +#[inline(always)] +fn mutation_receiver_allows_plain_tail(object_flags: u16) -> bool { + object_flags + & (crate::gc::OBJ_FLAG_FROZEN | crate::gc::OBJ_FLAG_SEALED | crate::gc::OBJ_FLAG_NO_EXTEND) + == 0 + && super::PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED.load(Ordering::Relaxed) == 0 +} + +/// In-bounds indexed write for an exact dense Array-subclass shape. The dense +/// layout builder proves every numeric prefix property is a writable data +/// property; descriptor or prototype mutations publish a different ShapeId +/// and therefore miss that cached proof. +#[inline] +pub(crate) fn array_subclass_fast_index_set(receiver: f64, index: u32, value: f64) -> bool { + let Some(receiver) = validated_object_receiver_for_value(receiver) else { + return false; + }; + array_subclass_fast_index_set_validated(receiver, index, value) +} + +#[inline] +pub(crate) fn array_subclass_fast_index_set_raw( + arr: *const ArrayHeader, + index: u32, + value: f64, +) -> bool { + let raw = (arr as u64 & crate::value::POINTER_MASK) as usize; + let Some(receiver) = validated_object_receiver(raw) else { + return false; + }; + array_subclass_fast_index_set_validated(receiver, index, value) +} + +#[inline] +fn array_subclass_fast_index_set_validated( + receiver: ValidatedObjectReceiver, + index: u32, + value: f64, +) -> bool { + let obj = receiver.object; + let Some(layout) = dense_layout_for_validated_object(obj) else { + return false; + }; + let Some(length) = nonnegative_u32_length(layout_length_value(obj, layout)) else { + return false; + }; + let Some(slot) = layout.element_base.checked_add(index) else { + return false; + }; + if index >= length || index >= layout.dense_prefix_len { + return false; + } + if receiver.object_flags & crate::gc::OBJ_FLAG_FROZEN != 0 + || !unsafe { dense_slot_exists(obj, slot, layout.live_inline_slots) } + { + return false; + } + let obj = obj as *mut ObjectHeader; + unsafe { + let old_value = layout_field_value(obj, slot, layout.live_inline_slots); + let numeric_u31 = |bits| { + crate::array::value_bits_to_number(bits).filter(|number| { + number.is_finite() + && *number >= 0.0 + && *number <= i32::MAX as f64 + && number.fract() == 0.0 + }) + }; + if let (Some(_old), Some(new)) = + (numeric_u31(old_value.bits()), numeric_u31(value.to_bits())) + { + // Both sides are pointer-free Numbers, so neither the pointer mask + // nor an established packed-u32 prefix changes. Keep that proof + // live and overwrite the slot without the general metadata path. + store_dense_nonpointer_number_slot(obj, slot, layout.live_inline_slots, new) + } else { + clear_packed_subclass_numeric_proof(obj); + store_dense_slot(obj, slot, layout.live_inline_slots, value.to_bits()) + } + } +} + +/// Allocation-free `Array.prototype.push` for a previously observed dense +/// Array-subclass tail transition. The first append at each length learns the +/// ordinary object transition; later cycles reuse it under exact shape and +/// descriptor guards. +#[inline] +pub(crate) fn array_subclass_fast_push_one(receiver: f64, value: f64) -> Option { + let receiver = validated_object_receiver_for_value(receiver)?; + array_subclass_fast_push_one_validated(receiver, value, None) +} + +/// Raw-entry counterpart to [`array_subclass_fast_push_one`]. The pointer is +/// magnitude- and header-validated exactly once before the dense mutation. +#[inline] +pub(crate) fn array_subclass_fast_push_one_raw(arr: *const ArrayHeader, value: f64) -> Option { + let raw = (arr as u64 & crate::value::POINTER_MASK) as usize; + let receiver = validated_object_receiver(raw)?; + array_subclass_fast_push_one_validated(receiver, value, None) +} + +/// Raw-entry counterpart for a value constructively proved by generated code +/// to be a nonnegative signed-i32 Number. Besides keeping tagged integers and +/// ClassRefs out of this path, that proof lets the hot Array-subclass append +/// skip `value_bits_to_number`, finiteness, range, and fractional checks. +#[inline] +pub(crate) fn array_subclass_fast_push_u31_raw(arr: *const ArrayHeader, value: u32) -> Option { + let raw = (arr as u64 & crate::value::POINTER_MASK) as usize; + let receiver = validated_object_receiver(raw)?; + debug_assert!(value <= i32::MAX as u32); + array_subclass_fast_push_one_validated(receiver, f64::from(value), Some(value)) +} + +#[inline] +fn array_subclass_fast_push_one_validated( + receiver: ValidatedObjectReceiver, + value: f64, + proven_u31: Option, +) -> Option { + let obj = receiver.object; + let layout = dense_layout_for_validated_object(obj)?; + let length = nonnegative_u32_length(layout_length_value(obj, layout))?; + if length > layout.dense_prefix_len + || !mutation_receiver_allows_plain_tail(receiver.object_flags) + { + return None; + } + let predecessor_shape_id = unsafe { (*obj).parent_class_id }; + let transition = crate::object::array_tail_transition::lookup_forward_for_owner( + obj, + predecessor_shape_id, + length, + )?; + if length != 0 && transition.slot != layout.element_base.checked_add(length)? { + return None; + } + if layout.live_inline_slots != transition.predecessor_live_inline_slots + || !unsafe { + dense_slot_exists(obj, transition.slot, transition.successor_live_inline_slots) + && dense_slot_exists( + obj, + layout.length_slot, + transition.successor_live_inline_slots, + ) + } + { + return None; + } + + let obj = obj as *mut ObjectHeader; + let installed = unsafe { + crate::object::shapes::install_cache_carried_object_shape_version( + obj, + predecessor_shape_id, + transition.successor_shape_id, + transition.successor_keys as *mut ArrayHeader, + transition.slot.saturating_add(1), + ) + }; + if !installed { + return None; + } + let new_length = length.checked_add(1)?; + unsafe { + // ECS entity ids arrive here as exact nonnegative i32 Numbers. Keep + // this proof constructive and local: tagged class references share the + // INT32 tag, and arbitrary doubles can look pointer-bearing to the + // conservative layout classifier, so neither is admitted. The generic + // barriered store below remains the complete fallback for them. + let numeric_entity = proven_u31.map(f64::from).or_else(|| { + crate::array::value_bits_to_number(value.to_bits()).filter(|number| { + number.is_finite() + && *number >= 0.0 + && *number <= i32::MAX as f64 + && number.fract() == 0.0 + }) + }); + let (value_stored, length_stored) = if let Some(number) = numeric_entity { + // `layout_note_slot` used to retire this proof as a side effect. + // Retire it explicitly before bypassing that general hook. + clear_packed_subclass_numeric_proof(obj); + ( + store_dense_nonpointer_number_slot( + obj, + transition.slot, + transition.successor_live_inline_slots, + number, + ), + store_dense_nonpointer_number_slot( + obj, + layout.length_slot, + transition.successor_live_inline_slots, + f64::from(new_length), + ), + ) + } else { + ( + store_dense_slot( + obj, + transition.slot, + transition.successor_live_inline_slots, + value.to_bits(), + ), + store_dense_slot( + obj, + layout.length_slot, + transition.successor_live_inline_slots, + f64::from(new_length).to_bits(), + ), + ) + }; + debug_assert!(value_stored && length_stored); + publish_owner_dense_layout( + obj, + DenseSubclassLayout { + length_slot: layout.length_slot, + element_base: layout.element_base, + dense_prefix_len: layout.dense_prefix_len.max(new_length), + live_inline_slots: transition.successor_live_inline_slots, + }, + ); + } + Some(f64::from(new_length)) +} + +/// Allocation-free `Array.prototype.pop` for an exact learned dense-tail +/// transition. All rejected cases retain the generic observable algorithm. +#[inline] +pub(crate) fn array_subclass_fast_pop(receiver: f64) -> Option { + let receiver = validated_object_receiver_for_value(receiver)?; + array_subclass_fast_pop_validated(receiver) +} + +/// Raw-entry counterpart to [`array_subclass_fast_pop`], sharing one validated +/// object-header read across the entire dense-tail mutation. +#[inline] +pub(crate) fn array_subclass_fast_pop_raw(arr: *const ArrayHeader) -> Option { + let raw = (arr as u64 & crate::value::POINTER_MASK) as usize; + let receiver = validated_object_receiver(raw)?; + array_subclass_fast_pop_validated(receiver) +} + +#[inline] +fn array_subclass_fast_pop_validated(receiver: ValidatedObjectReceiver) -> Option { + let obj = receiver.object; + let layout = dense_layout_for_validated_object(obj)?; + let length = nonnegative_u32_length(layout_length_value(obj, layout))?; + let index = length.checked_sub(1)?; + if length > layout.dense_prefix_len + || !mutation_receiver_allows_plain_tail(receiver.object_flags) + { + return None; + } + let successor_shape_id = unsafe { (*obj).parent_class_id }; + let transition = + crate::object::array_tail_transition::lookup_reverse_for_owner(obj, successor_shape_id)?; + if transition.array_index != index + || transition.slot != layout.element_base.checked_add(index)? + { + return None; + } + if layout.live_inline_slots != transition.successor_live_inline_slots + || !unsafe { + dense_slot_exists(obj, transition.slot, transition.successor_live_inline_slots) + && dense_slot_exists( + obj, + layout.length_slot, + transition.predecessor_live_inline_slots, + ) + } + { + return None; + } + let value = layout_field_value(obj, transition.slot, transition.successor_live_inline_slots); + let numeric_entity = crate::array::value_bits_to_number(value.bits()).filter(|number| { + number.is_finite() && *number >= 0.0 && *number <= i32::MAX as f64 && number.fract() == 0.0 + }); + let obj = obj as *mut ObjectHeader; + unsafe { clear_packed_subclass_numeric_proof(obj) }; + crate::object::prop_plan::prop_plan_epoch_bump(); + let installed = unsafe { + crate::object::shapes::install_cache_carried_object_shape_version( + obj, + successor_shape_id, + transition.predecessor_shape_id, + transition.predecessor_keys as *mut ArrayHeader, + transition.slot, + ) + }; + if !installed { + return None; + } + unsafe { + let length_stored = if numeric_entity.is_some() { + clear_retired_dense_numeric_tail_slot( + obj, + transition.slot, + transition.successor_live_inline_slots, + ); + store_dense_nonpointer_number_slot( + obj, + layout.length_slot, + transition.predecessor_live_inline_slots, + f64::from(index), + ) + } else { + clear_retired_dense_slot(obj, transition.slot, transition.successor_live_inline_slots); + store_dense_slot( + obj, + layout.length_slot, + transition.predecessor_live_inline_slots, + f64::from(index).to_bits(), + ) + }; + debug_assert!(length_stored); + publish_owner_dense_layout( + obj, + DenseSubclassLayout { + length_slot: layout.length_slot, + element_base: layout.element_base, + dense_prefix_len: index, + live_inline_slots: transition.predecessor_live_inline_slots, + }, + ); + } + Some(f64::from_bits(value.bits())) +} + #[inline(always)] fn dense_index_get_with_layout( obj: *const ObjectHeader, @@ -591,25 +1477,37 @@ pub extern "C" fn js_packed_arraylike_index_get(receiver: f64, index: f64, cache index_u32, ); } - if header.obj_type == crate::gc::GC_TYPE_OBJECT { - if let Some((obj, layout)) = dense_layout_for_value(receiver) { - // The codegen hit path reads length inline and wide - // slots through ObjectMeta::spill. Decline to prime in - // the legacy side-table mode or for a pathological - // layout whose length itself spilled. - if !cache.is_null() - && crate::object::object_spill_enabled() - && layout.length_slot < layout.live_inline_slots - { + if header.obj_type == crate::gc::GC_TYPE_OBJECT + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + { + let obj = raw.cast::(); + if let Some(layout) = dense_layout_for_validated_object(obj) { + // The codegen hit path handles both inline and + // object-owned spill slots. In spill mode, publish a + // class-wide dense-tail identity when the owner has + // proved one. Exact push/pop transitions preserve + // that move-stable token, so a lifecycle loop does not + // miss once for every historical tail ShapeId. The + // cached dense-prefix word remains the admitted high + // water mark: a generic `length` grow beyond it still + // side-exits and re-establishes the complete proof. + if !cache.is_null() && crate::object::object_spill_enabled() { + let family_token = unsafe { + array_subclass_named_prefix_token_for_slot( + obj, + layout.length_slot as usize, + ) + }; unsafe { cache.add(1).write(layout.length_slot as u64); cache.add(2).write(layout.element_base as u64); cache.add(3).write(layout.dense_prefix_len as u64); cache.add(4).write(layout.live_inline_slots as u64); - cache.write(dense_cache_key( - (*obj).class_id, - (*obj).parent_class_id, - )); + cache.write(if family_token != 0 { + family_token + } else { + dense_cache_key((*obj).class_id, (*obj).parent_class_id) + }); } } if let Some(value) = dense_index_get_with_layout(obj, layout, index_u32) { @@ -727,8 +1625,8 @@ fn packed_arraylike_loop_guard( if header.obj_type != crate::gc::GC_TYPE_OBJECT { return None; } - let live_receiver = f64::from_bits(crate::value::js_nanbox_pointer(raw as i64).to_bits()); - let Some((object, layout)) = dense_layout_for_value(live_receiver) else { + let object = raw.cast::(); + let Some(layout) = dense_layout_for_validated_object(object) else { return None; }; // Stable-loop codegen already handles both inline and object-owned spill @@ -1274,13 +2172,7 @@ pub fn array_subclass_has_iterator_override(value: f64) -> bool { #[inline] pub(crate) fn raw_receiver_is_heap_object(arr: *const ArrayHeader) -> bool { let raw = ((arr as u64) & 0x0000_FFFF_FFFF_FFFF) as usize; - if raw == 0 { - return false; - } - match unsafe { crate::value::addr_class::try_read_gc_header(raw) } { - Some(header) => header.obj_type == crate::gc::GC_TYPE_OBJECT, - None => false, - } + validated_object_receiver(raw).is_some() } #[cold] @@ -1320,6 +2212,15 @@ pub(crate) fn is_array_subclass_value(value: f64) -> bool { #[cold] #[inline(never)] pub(crate) fn array_object_method(recv: f64, method: &str, args: &[f64]) -> Option { + if method == "push" && args.len() == 1 { + if let Some(length) = array_subclass_fast_push_one(recv, args[0]) { + return Some(length); + } + } else if method == "pop" && args.is_empty() { + if let Some(value) = array_subclass_fast_pop(recv) { + return Some(value); + } + } let (ptr, len) = (args.as_ptr(), args.len()); if let Some(result) = super::generic::run_object_mutator(recv, method, ptr, len) { return Some(result); @@ -1331,6 +2232,9 @@ pub(crate) fn array_object_method(recv: f64, method: &str, args: &[f64]) -> Opti #[cold] #[inline(never)] pub(crate) fn array_object_index_get(recv: f64, index: u32) -> f64 { + if let Some(value) = array_subclass_fast_index_get(recv, index) { + return value; + } al_get(recv, index as i64) } @@ -1346,6 +2250,9 @@ pub(crate) fn array_object_index_get(recv: f64, index: u32) -> f64 { #[cold] #[inline(never)] pub(crate) fn array_object_index_set(recv: f64, index: u32, value: f64) { + if array_subclass_fast_index_set(recv, index, value) { + return; + } // The store interns a key string and can allocate, so root the receiver // across it — it is a movable `ObjectHeader` and is read again below. let scope = crate::gc::RuntimeHandleScope::new(); diff --git a/crates/perry-runtime/src/array/subclass_tests.rs b/crates/perry-runtime/src/array/subclass_tests.rs index 2b67a7ad5e..ba066af87f 100644 --- a/crates/perry-runtime/src/array/subclass_tests.rs +++ b/crates/perry-runtime/src/array/subclass_tests.rs @@ -18,11 +18,17 @@ //! vacuous — which is exactly the failure mode the module is written to avoid. use super::subclass::{ - array_object_receiver, array_subclass_fast_index_get, array_subclass_fast_length, - is_array_subclass_class_id, js_packed_arraylike_index_get, js_packed_arraylike_loop_guard, - js_packed_ecs_u32_loop_guard, raw_receiver_is_heap_object, + array_object_receiver, array_subclass_fast_index_get, array_subclass_fast_index_set, + array_subclass_fast_length, array_subclass_fast_length_with_ic, array_subclass_fast_pop, + array_subclass_fast_push_one, array_subclass_named_prefix_token_for_slot, + array_subclass_named_prefix_token_matches_class, is_array_subclass_class_id, + js_packed_arraylike_index_get, js_packed_arraylike_loop_guard, js_packed_ecs_u32_loop_guard, + raw_receiver_is_heap_object, +}; +use crate::array::{ + clean_arr_ptr, js_array_alloc, js_array_pop_f64, js_array_push_f64, + js_array_push_u31_with_length, ArrayHeader, }; -use crate::array::{clean_arr_ptr, js_array_alloc, ArrayHeader}; use crate::object::{js_object_alloc, ObjectHeader}; /// The reserved parent class id `class X extends Array` records. @@ -185,12 +191,49 @@ fn dense_array_subclass_reads_slots_until_its_shape_changes() { assert_eq!(array_subclass_fast_length(receiver), Some(3.0)); assert_eq!(array_subclass_fast_index_get(receiver, 1), Some(22.0)); + let stable_shape = unsafe { (*obj).parent_class_id }; + let stable_dense_key = unsafe { (*(*obj).meta).array_subclass_dense_key }; + assert_eq!( + stable_dense_key, + (u64::from(class_id) << 32) | u64::from(stable_shape), + "the first proved dense read must publish the receiver-local layout" + ); + assert!(array_subclass_fast_index_set(receiver, 1, 44.0)); + assert_eq!(array_subclass_fast_index_get(receiver, 1), Some(44.0)); + assert_eq!(crate::array::js_array_length(obj as *const ArrayHeader), 3); + assert_eq!( + crate::array::js_array_get_f64(obj as *const ArrayHeader, 1), + 44.0 + ); + assert_eq!( + crate::array::js_array_set_f64_extend(obj as *mut ArrayHeader, 1, 55.0), + obj as *mut ArrayHeader + ); + assert_eq!( + crate::array::js_array_get_f64(obj as *const ArrayHeader, 1), + 55.0 + ); + assert_eq!( + super::indexing::try_strict_dense_index_set(obj as *mut ArrayHeader, 1, 66.0,), + Some(obj as *mut ArrayHeader) + ); + assert_eq!( + crate::array::js_array_get_f64(obj as *const ArrayHeader, 1), + 66.0 + ); + assert_eq!(unsafe { (*obj).parent_class_id }, stable_shape); + assert!(!array_subclass_fast_index_set(receiver, 3, 55.0)); assert_eq!( js_packed_arraylike_index_get(receiver, 2.0, std::ptr::null_mut()), 33.0 ); crate::object::js_object_delete_dynamic(obj, 1.0); + assert_ne!( + stable_dense_key, + (u64::from(class_id) << 32) | unsafe { u64::from((*obj).parent_class_id) }, + "a structural mutation must make the receiver-local proof miss by ShapeId" + ); assert_eq!( array_subclass_fast_index_get(receiver, 1), None, @@ -203,6 +246,621 @@ fn dense_array_subclass_reads_slots_until_its_shape_changes() { ); } +#[test] +fn dense_array_subclass_cache_declines_a_per_instance_prototype_override() { + let class_id = 0x0074_865A; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let obj = js_object_alloc(class_id, 2); + assert!(!obj.is_null()); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + crate::object::js_object_set_index_polymorphic(obj as i64, 0.0, 11.0); + + assert_eq!(array_subclass_fast_index_get(receiver, 0), Some(11.0)); + crate::object::prototype_chain::object_set_static_prototype( + obj as usize, + crate::value::TAG_NULL, + ); + assert_eq!( + array_subclass_fast_index_get(receiver, 0), + None, + "a receiver-local dense-layout record must not survive prototype divergence" + ); +} + +/// A learned sequential numeric append is an exact reversible shape edge. +/// Pin both the direct helpers and the public native push/pop integration: a +/// generic delete would clone the keys array and mint a different predecessor +/// ShapeId, so exact identity makes this test non-vacuous. +#[test] +fn dense_array_subclass_tail_transitions_reuse_exact_shapes_and_slots() { + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + let class_id = 0x0074_8657; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let obj = js_object_alloc(class_id, 2); + assert!(!obj.is_null()); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + + let mut shapes = vec![unsafe { (*obj).parent_class_id }]; + for value in [11.0, 22.0, 33.0] { + assert_eq!( + js_array_push_f64(obj as *mut ArrayHeader, value), + obj as *mut ArrayHeader + ); + shapes.push(unsafe { (*obj).parent_class_id }); + } + assert_eq!(array_subclass_fast_length(receiver), Some(3.0)); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_dense_key }, + (u64::from(class_id) << 32) | u64::from(shapes[3]), + "the warm dense lookup must bind its scalar layout to the current shape" + ); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_dense_bounds as u32 }, + 3 + ); + assert!(crate::object::array_tail_transition::lookup_reverse(shapes[3]).is_some()); + assert_ne!( + unsafe { (*(*obj).meta).array_tail_object_hot }, + 0, + "learning a spill-backed tail must bind this receiver to its agent-local transition tables" + ); + + assert_eq!(array_subclass_fast_pop(receiver), Some(33.0)); + assert_eq!(unsafe { (*obj).parent_class_id }, shapes[2]); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_dense_key }, + (u64::from(class_id) << 32) | u64::from(shapes[2]), + "pop must publish the exact predecessor layout without a cache rebuild" + ); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_dense_bounds as u32 }, + 2 + ); + assert_eq!(array_subclass_fast_length(receiver), Some(2.0)); + assert_eq!(array_subclass_fast_index_get(receiver, 2), None); + + assert_eq!(array_subclass_fast_push_one(receiver, 44.0), Some(3.0)); + assert_eq!(unsafe { (*obj).parent_class_id }, shapes[3]); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_dense_key }, + (u64::from(class_id) << 32) | u64::from(shapes[3]), + "push must publish the exact successor layout without a cache rebuild" + ); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_dense_bounds as u32 }, + 3 + ); + assert_eq!(array_subclass_fast_index_get(receiver, 2), Some(44.0)); + + assert_eq!(js_array_pop_f64(obj as *mut ArrayHeader), 44.0); + assert_eq!(unsafe { (*obj).parent_class_id }, shapes[2]); + assert_eq!(array_subclass_fast_length(receiver), Some(2.0)); +} + +#[test] +fn array_subclass_length_ic_publishes_only_scalar_exact_or_family_facts() { + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + let class_id = 0x0074_867b; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let packed = b"sset\0mask\0"; + let keys = + crate::object::js_build_class_keys_array(class_id, 2, packed.as_ptr(), packed.len() as u32); + let obj = crate::object::js_object_alloc_class_inline_keys(class_id, CLASS_ID_ARRAY, 2, keys); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + + let mut cache = [0_u64; 3]; + assert_eq!( + array_subclass_fast_length_with_ic(receiver, cache.as_mut_ptr()), + Some(0.0) + ); + if crate::object::object_spill_enabled() { + assert_ne!(cache[0] & (1_u64 << 63), 0); + assert_eq!(cache[0] & u64::from(u32::MAX), u64::from(class_id)); + assert_eq!( + cache[0], + unsafe { (*(*obj).meta).array_subclass_named_prefix_token }, + "an empty subclass can publish the same pointer-free family proof" + ); + } else { + assert_eq!( + cache[0], + (u64::from(class_id) << 32) | unsafe { u64::from((*obj).parent_class_id) } + ); + } + assert!( + cache[1] >= cache[2], + "this wolf-shaped receiver deliberately stores length in ObjectMeta::spill" + ); + + js_array_push_f64(obj as *mut ArrayHeader, 11.0); + assert_eq!( + array_subclass_fast_length_with_ic(receiver, cache.as_mut_ptr()), + Some(1.0) + ); + if crate::object::object_spill_enabled() { + assert_ne!(cache[0] & (1_u64 << 63), 0); + assert_eq!(cache[0] & u64::from(u32::MAX), u64::from(class_id)); + assert_eq!( + cache[0], + unsafe { (*(*obj).meta).array_subclass_named_prefix_token }, + "the cache stores the receiver-owned scalar proof, not a heap address" + ); + } +} + +/// The metadata-free tail store is admitted only for constructive numeric +/// entity IDs. Tagged values must retain their exact bits and continue through +/// the ordinary barriered slot-store path. +#[test] +fn dense_array_subclass_numeric_tail_store_preserves_tagged_fallbacks() { + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + let class_id = 0x0074_865c; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let obj = js_object_alloc(class_id, 2); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + + // Learn a forward/reverse edge, then revisit it with each value kind. + js_array_push_f64(obj as *mut ArrayHeader, 11.0); + assert_eq!(array_subclass_fast_pop(receiver), Some(11.0)); + + let tagged_i32 = f64::from_bits(crate::value::JSValue::int32(123).bits()); + assert_eq!( + array_subclass_fast_push_one(receiver, tagged_i32), + Some(1.0) + ); + assert_eq!( + array_subclass_fast_index_get(receiver, 0), + Some(123.0), + "a genuine INT32 Number may be canonicalized to its raw-f64 form" + ); + assert_eq!(array_subclass_fast_pop(receiver), Some(123.0)); + + let sso = f64::from_bits( + crate::value::JSValue::try_short_string(b"ecs") + .expect("three bytes fit the inline-string representation") + .bits(), + ); + assert_eq!(array_subclass_fast_push_one(receiver, sso), Some(1.0)); + assert_eq!( + array_subclass_fast_index_get(receiver, 0).map(f64::to_bits), + Some(sso.to_bits()), + "an inline string must not be reinterpreted as a Number" + ); + assert_eq!( + array_subclass_fast_pop(receiver).map(f64::to_bits), + Some(sso.to_bits()) + ); + + // Class references deliberately share INT32_TAG with small integers. The + // class registry is the disambiguating guard used by value_bits_to_number. + unsafe { crate::object::js_register_class_id(class_id) }; + let class_ref = f64::from_bits(crate::value::INT32_TAG | u64::from(class_id)); + assert_eq!(array_subclass_fast_push_one(receiver, class_ref), Some(1.0)); + assert_eq!( + array_subclass_fast_index_get(receiver, 0).map(f64::to_bits), + Some(class_ref.to_bits()), + "a ClassRef must keep its tag for downstream property dispatch" + ); + assert_eq!( + array_subclass_fast_pop(receiver).map(f64::to_bits), + Some(class_ref.to_bits()), + "the numeric pop specialization must keep ClassRefs on its barriered fallback" + ); +} + +#[test] +fn fused_u31_push_reports_length_for_plain_and_subclass_arrays() { + let mut length = u32::MAX; + let plain = js_array_alloc(1); + let plain = js_array_push_u31_with_length(plain, 7, &mut length); + assert_eq!(length, 1); + assert_eq!(crate::array::js_array_get_f64(plain, 0), 7.0); + let plain = js_array_push_u31_with_length(plain, 9, &mut length); + assert_eq!( + length, 2, + "the fused result must follow a reallocating grow" + ); + assert_eq!(crate::array::js_array_get_f64(plain, 1), 9.0); + + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + let class_id = 0x0074_865d; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let obj = js_object_alloc(class_id, 2); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + js_array_push_f64(obj as *mut ArrayHeader, 11.0); + assert_eq!(array_subclass_fast_pop(receiver), Some(11.0)); + + let returned = js_array_push_u31_with_length(obj as *mut ArrayHeader, 42, &mut length); + assert_eq!(returned, obj as *mut ArrayHeader); + assert_eq!(length, 1); + assert_eq!(array_subclass_fast_index_get(receiver, 0), Some(42.0)); +} + +#[test] +fn array_subclass_named_prefix_token_survives_only_exact_numeric_tail_transitions() { + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + let class_id = 0x0074_865b; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let packed = b"sset\0mask\0"; + let keys = + crate::object::js_build_class_keys_array(class_id, 2, packed.as_ptr(), packed.len() as u32); + let obj = crate::object::js_object_alloc_class_inline_keys(class_id, CLASS_ID_ARRAY, 2, keys); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + crate::object::descriptor_state::set_property_attrs( + obj as usize, + "fill".to_string(), + crate::object::descriptor_state::PropertyAttrs::new(true, false, true), + ); + + // Learn both edges and force numeric storage beyond the two declared + // inline slots, which gives this object the ObjectMeta that carries the + // move-stable family proof. + js_array_push_f64(obj as *mut ArrayHeader, 11.0); + js_array_push_f64(obj as *mut ArrayHeader, 22.0); + let mask_key = crate::string::js_string_from_bytes(b"mask".as_ptr(), 4); + let mut cache = [0i64; crate::object::PIC_CACHE_WORDS]; + crate::object::js_object_get_field_ic_miss(obj, mask_key, &mut cache); + let token = cache[2] as u64; + assert_ne!(token, 0); + assert!(unsafe { array_subclass_named_prefix_token_matches_class(obj, class_id) }); + assert!( + !unsafe { array_subclass_named_prefix_token_matches_class(obj, class_id.wrapping_add(1)) }, + "the token must pin the exact class rather than merely the Array-subclass family" + ); + assert_eq!( + unsafe { array_subclass_named_prefix_token_for_slot(obj, 1) }, + token, + "the IC miss must publish the same owner-side token it caches" + ); + + let mut index_ic = [0u64; 5]; + assert_eq!( + js_packed_arraylike_index_get(receiver, 0.0, index_ic.as_mut_ptr()), + 11.0 + ); + assert_eq!( + index_ic[0], token, + "a dense numeric read must publish the tail-family token instead of an exact ShapeId" + ); + assert!( + index_ic[1] >= index_ic[4], + "this fixture keeps Array-subclass length in ObjectMeta::spill" + ); + assert_eq!( + index_ic[3], 2, + "the cached prefix is a safe high-water mark" + ); + + let zero_key = crate::string::js_string_from_bytes(b"0".as_ptr(), 1); + let mut element_cache = [0i64; crate::object::PIC_CACHE_WORDS]; + assert_eq!( + crate::object::js_object_get_field_ic_miss(obj, zero_key, &mut element_cache), + 11.0 + ); + assert_eq!( + element_cache[2], 0, + "a numeric-element site must never borrow the stable named-prefix identity" + ); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_named_prefix_token }, + token, + "declining a numeric site must not retire the independently valid named-prefix proof" + ); + + assert_eq!(array_subclass_fast_pop(receiver), Some(22.0)); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_named_prefix_token }, + token, + "the exact reverse numeric-tail edge must preserve the named-prefix proof" + ); + assert_eq!(array_subclass_fast_push_one(receiver, 33.0), Some(2.0)); + assert_eq!( + unsafe { array_subclass_named_prefix_token_for_slot(obj, 1) }, + token, + "the exact forward edge must retain the same class-wide token" + ); + + let extra = crate::string::js_string_from_bytes(b"extra".as_ptr(), 5); + crate::object::js_object_set_field_by_name(obj, extra, 7.0); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_named_prefix_token }, + 0, + "generic named structural mutation must retire the family proof before publication" + ); + assert!( + !unsafe { array_subclass_named_prefix_token_matches_class(obj, class_id) }, + "a retired token must not remain consumable as an ordinary-object proof" + ); + assert_eq!( + unsafe { array_subclass_named_prefix_token_for_slot(obj, 1) }, + 0, + "an instance-specific named suffix must not borrow the class token again" + ); +} + +/// Wolf's `_ent` / `_updateTo` arrays store `Archetype` instances while each +/// Archetype's dense numeric tail changes on every entity migration. The +/// enclosing plain array's element-class proof must be able to consume the +/// subclass's stable named-prefix token; consulting the new ShapeId on every +/// overwrite defeats the transition cache that made the tail mutation cheap. +#[test] +fn plain_array_element_shape_consumes_array_subclass_prefix_proof() { + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + let class_id = 0x0074_8667; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let packed = b"sset\0mask\0change\0"; + let keys = + crate::object::js_build_class_keys_array(class_id, 3, packed.as_ptr(), packed.len() as u32); + let obj = crate::object::js_object_alloc_class_inline_keys(class_id, CLASS_ID_ARRAY, 3, keys); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + crate::object::descriptor_state::set_property_attrs( + obj as usize, + "fill".to_string(), + crate::object::descriptor_state::PropertyAttrs::new(true, false, true), + ); + + // Learn the 1 -> 2 edge before publishing the token. The first generic + // edge is allowed to rebuild shape metadata; the warm exact edge below is + // the one whose preservation contract the ECS path relies on. + js_array_push_f64(obj as *mut ArrayHeader, 11.0); + js_array_push_f64(obj as *mut ArrayHeader, 22.0); + assert_eq!(array_subclass_fast_pop(receiver), Some(22.0)); + assert_ne!( + unsafe { array_subclass_named_prefix_token_for_slot(obj, 1) }, + 0, + "precondition: the subclass must carry the ordinary-prefix proof" + ); + + let mut owners = js_array_alloc(1); + owners = js_array_push_f64(owners, receiver); + assert_eq!( + crate::array::js_array_element_shape_class(owners), + class_id as i32, + "the enclosing plain array should establish a class proof" + ); + + let original_shape = unsafe { (*obj).parent_class_id }; + assert_eq!(array_subclass_fast_push_one(receiver, 33.0), Some(2.0)); + assert_ne!( + unsafe { (*obj).parent_class_id }, + original_shape, + "the numeric tail transition must actually change the exact ShapeId" + ); + let hits_before = crate::array::element_shape::test_array_subclass_prefix_store_hits(); + assert_eq!( + crate::array::indexing::try_strict_dense_index_set(owners, 0, receiver), + Some(owners) + ); + assert!( + crate::array::element_shape::test_array_subclass_prefix_store_hits() > hits_before, + "the overwrite should use the stable subclass proof rather than classify the changing ShapeId" + ); + assert_eq!( + crate::array::js_array_element_shape_class(owners), + class_id as i32, + "consuming the subclass proof must retain the enclosing class invariant" + ); +} + +/// Entity migration removes the last id from its source Archetype before +/// `_archChange` reads `arch.change` and `arch.mask`. That receiver is an empty, +/// descriptor-bearing Array subclass: the ordinary exact-shape PIC is closed, +/// so the class-prefix token must be available before a `"0"` key exists. +#[test] +fn empty_array_subclass_named_prefix_token_survives_warm_tail_cycle() { + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + let class_id = 0x0074_8659; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let packed = b"sset\0mask\0change\0"; + let keys = + crate::object::js_build_class_keys_array(class_id, 3, packed.as_ptr(), packed.len() as u32); + let obj = crate::object::js_object_alloc_class_inline_keys(class_id, CLASS_ID_ARRAY, 3, keys); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + crate::object::descriptor_state::set_property_attrs( + obj as usize, + "fill".to_string(), + crate::object::descriptor_state::PropertyAttrs::new(true, false, true), + ); + + // Learn the 0 -> 1 edge once, then return to the exact empty predecessor. + // Subsequent cycles use the allocation-free transition cache. + js_array_push_f64(obj as *mut ArrayHeader, 11.0); + assert_eq!(array_subclass_fast_pop(receiver), Some(11.0)); + + let change_key = crate::string::js_string_from_bytes(b"change".as_ptr(), 6); + let mut cache = [0i64; crate::object::PIC_CACHE_WORDS]; + let via_ic = crate::object::js_object_get_field_ic_miss(obj, change_key, &mut cache); + let via_ladder = crate::object::js_object_get_field_by_name_f64(obj, change_key); + assert_eq!(via_ic.to_bits(), via_ladder.to_bits()); + let token = cache[2] as u64; + assert_ne!( + token, 0, + "an empty subclass must arm the declared-prefix PIC" + ); + assert_eq!( + unsafe { array_subclass_named_prefix_token_for_slot(obj, 2) }, + token + ); + + assert_eq!(array_subclass_fast_push_one(receiver, 22.0), Some(1.0)); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_named_prefix_token }, + token, + "the warm forward edge must retain the empty-prefix proof" + ); + assert_eq!(array_subclass_fast_pop(receiver), Some(22.0)); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_named_prefix_token }, + token, + "the warm reverse edge must retain the empty-prefix proof" + ); + + crate::object::set_accessor_descriptor( + obj as usize, + "change".to_string(), + crate::object::AccessorDescriptor { get: 1, set: 0 }, + ); + assert_eq!( + unsafe { (*(*obj).meta).array_subclass_named_prefix_token }, + 0, + "an accessor mutation must retire the direct-load proof" + ); + assert_eq!( + unsafe { array_subclass_named_prefix_token_for_slot(obj, 2) }, + 0, + "an accessor-backed declared key must not re-arm the family token" + ); +} + +/// A direct-mapped transition cache can appear correct on short arrays yet +/// lose one historical edge to a hash collision. One miss changes the shape +/// lineage and makes every older edge unusable, which is catastrophic for ECS +/// archetypes that drain and refill a thousand entities per tick. +#[test] +fn dense_array_subclass_tail_cache_preserves_a_1024_shape_lattice() { + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + let class_id = 0x0074_865a; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let obj = js_object_alloc(class_id, 2); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + + let mut shapes = Vec::with_capacity(1025); + shapes.push(unsafe { (*obj).parent_class_id }); + for value in 0u32..1024 { + js_array_push_f64(obj as *mut ArrayHeader, f64::from(value)); + shapes.push(unsafe { (*obj).parent_class_id }); + } + + for length in (1u32..=1024).rev() { + assert_eq!( + array_subclass_fast_pop(receiver), + Some(f64::from(length - 1)), + "reverse edge at length {length} must survive unrelated hash collisions" + ); + assert_eq!( + unsafe { (*obj).parent_class_id }, + shapes[length as usize - 1] + ); + } + for value in 0u32..1024 { + assert_eq!( + array_subclass_fast_push_one(receiver, f64::from(value)), + Some(f64::from(value + 1)), + "forward edge at length {value} must remain reusable" + ); + assert_eq!( + unsafe { (*obj).parent_class_id }, + shapes[value as usize + 1] + ); + } +} + +#[test] +fn dense_array_subclass_tail_fast_path_declines_restricted_receivers() { + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + let class_id = 0x0074_8658; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let obj = js_object_alloc(class_id, 2); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + js_array_push_f64(obj as *mut ArrayHeader, 11.0); + + let header = unsafe { + (obj as *mut u8) + .sub(crate::gc::GC_HEADER_SIZE) + .cast::() + }; + unsafe { (*header)._reserved |= crate::gc::OBJ_FLAG_SEALED }; + assert_eq!(array_subclass_fast_pop(receiver), None); + assert_eq!(array_subclass_fast_push_one(receiver, 22.0), None); + assert_eq!(array_subclass_fast_length(receiver), Some(1.0)); + assert_eq!(array_subclass_fast_index_get(receiver, 0), Some(11.0)); + unsafe { (*header)._reserved &= !crate::gc::OBJ_FLAG_SEALED }; + + let shape_before_descriptor = unsafe { (*obj).parent_class_id }; + crate::object::descriptor_state::set_property_attrs( + obj as usize, + "0".to_string(), + crate::object::descriptor_state::PropertyAttrs::new(true, true, false), + ); + assert_ne!(unsafe { (*obj).parent_class_id }, shape_before_descriptor); + assert_eq!( + array_subclass_fast_pop(receiver), + None, + "descriptor mutation must mint a ShapeId that cannot reuse the learned plain edge" + ); +} + +#[test] +fn dense_array_subclass_tail_transition_edges_survive_moving_gc() { + let _copying_nursery = crate::gc::CopyingNurseryTestGuard::new(0); + let _triggers = crate::gc::GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _force_evacuation = crate::gc::knob_overrides::ForcedEvacuationTestGuard::on(); + crate::gc::register_runtime_handle_root_scanner_for_tests(); + crate::gc::gc_register_mutable_root_scanner(crate::object::shapes::scan_shape_table_rekey_mut); + crate::gc::gc_register_mutable_root_scanner(crate::object::scan_transition_cache_roots_mut); + crate::object::array_tail_transition::test_clear(); + + let class_id = 0x0074_8659; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let obj = js_object_alloc(class_id, 2); + let scope = crate::gc::RuntimeHandleScope::new(); + let receiver_h = scope.root_nanbox_f64(crate::value::js_nanbox_pointer(obj as i64)); + crate::node_stream::js_array_subclass_init(receiver_h.get_nanbox_f64(), 0.0); + for value in [11.0, 22.0, 33.0] { + let live = + (receiver_h.get_nanbox_f64().to_bits() & 0x0000_FFFF_FFFF_FFFF) as *mut ArrayHeader; + js_array_push_f64(live, value); + } + let pre_gc_shape = unsafe { + let live = + (receiver_h.get_nanbox_f64().to_bits() & 0x0000_FFFF_FFFF_FFFF) as *mut ObjectHeader; + (*live).parent_class_id + }; + assert!(crate::object::array_tail_transition::lookup_reverse(pre_gc_shape).is_some()); + let before = crate::gc::copying_minor_cycles(); + let _ = crate::gc::gc_collect_minor(); + assert!(crate::gc::copying_minor_cycles() > before); + + let live_receiver = receiver_h.get_nanbox_f64(); + let live_obj = (live_receiver.to_bits() & 0x0000_FFFF_FFFF_FFFF) as *mut ObjectHeader; + assert_eq!(array_subclass_fast_length(live_receiver), Some(3.0)); + assert_eq!(unsafe { (*live_obj).parent_class_id }, pre_gc_shape); + assert_eq!( + unsafe { (*(*live_obj).meta).array_subclass_dense_key }, + (u64::from(class_id) << 32) | u64::from(pre_gc_shape), + "the receiver-local scalar layout must survive owner/meta evacuation" + ); + assert!( + crate::object::array_tail_transition::lookup_reverse(pre_gc_shape).is_some(), + "moving GC must repair both rooted key-array edges in the reverse cache" + ); + assert_eq!(array_subclass_fast_pop(live_receiver), Some(33.0)); + assert_eq!(array_subclass_fast_push_one(live_receiver, 44.0), Some(3.0)); + assert_eq!(array_subclass_fast_index_get(live_receiver, 2), Some(44.0)); + assert!(crate::object::shapes::is_shape_id(unsafe { + (*live_obj).parent_class_id + })); +} + /// #8690: pointer-free tagged values skip the GC write barrier. The generic /// successful-index hook must still retire a numeric-prefix proof, otherwise a /// later loop clone would reinterpret the SSO bits as an f64 Number. @@ -265,6 +923,45 @@ fn packed_numeric_proof_is_retired_by_sso_index_overwrite() { ); } +#[test] +fn packed_numeric_proof_survives_pointer_free_index_swap() { + let class_id = 0x0074_8692; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let obj = js_object_alloc(class_id, 2); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + for (index, value) in [11.0, 22.0, 33.0].into_iter().enumerate() { + crate::object::js_object_set_index_polymorphic(obj as i64, index as f64, value); + } + + let mut facts = [0u64; 7]; + assert_eq!( + js_packed_arraylike_loop_guard(receiver, 3.0, 1, facts.as_mut_ptr()), + 2 + ); + let header = unsafe { crate::value::addr_class::try_read_gc_header(obj as usize) } + .expect("the subclass receiver is live"); + assert_ne!( + header._reserved & crate::gc::OBJ_FLAG_PACKED_NUMERIC_PROOF, + 0 + ); + + assert!(array_subclass_fast_index_set(receiver, 1, 33.0)); + assert_eq!(array_subclass_fast_index_get(receiver, 1), Some(33.0)); + let header = unsafe { crate::value::addr_class::try_read_gc_header(obj as usize) } + .expect("the subclass receiver remains live"); + assert_ne!( + header._reserved & crate::gc::OBJ_FLAG_PACKED_NUMERIC_PROOF, + 0, + "a numeric-for-numeric overwrite must preserve the exact packed-u32 proof" + ); + assert_eq!( + js_packed_arraylike_loop_guard(receiver, 3.0, 1, facts.as_mut_ptr()), + 2, + "the next packed loop must consume the still-valid proof" + ); +} + #[test] fn fused_ecs_guard_requires_distinct_owning_u32_columns_and_exact_entity_ids() { let class_id = 0x0074_8691; diff --git a/crates/perry-runtime/src/array/tests.rs b/crates/perry-runtime/src/array/tests.rs index 2a595eb833..42819aae7b 100644 --- a/crates/perry-runtime/src/array/tests.rs +++ b/crates/perry-runtime/src/array/tests.rs @@ -499,6 +499,68 @@ fn test_array_grow_capacity() { ); } +#[test] +fn strict_dense_overwrite_preserves_numeric_and_pointer_layouts() { + let arr = js_array_alloc(4); + js_array_push_f64(arr, 10.0); + js_array_push_f64(arr, 20.0); + assert_eq!(js_array_is_numeric_f64_layout(arr), 1); + + let tagged_i32 = f64::from_bits(crate::value::INT32_TAG | 33); + assert_eq!( + indexing::try_strict_dense_index_set(arr, 1, tagged_i32), + Some(arr) + ); + assert_eq!(js_array_get_f64(arr, 1), 33.0); + assert_eq!(unsafe { raw_slot_bits(arr, 1) }, 33.0f64.to_bits()); + + let class_id = 0x0074_8693; + crate::object::js_register_class_parent(class_id, 0); + unsafe { crate::object::js_register_class_id(class_id) }; + let class_ref = f64::from_bits(crate::value::INT32_TAG | u64::from(class_id)); + assert_eq!( + indexing::try_strict_dense_index_set(arr, 1, class_ref), + Some(arr), + "a ClassRef may use the general barriered path but not the numeric path" + ); + assert_eq!(js_array_get_f64(arr, 1).to_bits(), class_ref.to_bits()); + assert_eq!(js_array_is_numeric_f64_layout(arr), 0); + + let pointer_arr = js_array_alloc(2); + let first = boxed_pointer(crate::object::js_object_alloc(0, 0).cast()); + let second = boxed_pointer(crate::object::js_object_alloc(0, 0).cast()); + js_array_push_f64(pointer_arr, first); + assert_eq!( + indexing::try_strict_dense_index_set(pointer_arr, 0, second), + Some(pointer_arr) + ); + assert_eq!(js_array_get_f64(pointer_arr, 0).to_bits(), second.to_bits()); + assert_eq!( + crate::gc::test_layout_pointer_slot_count(pointer_arr as usize, 1), + Some(1), + "the resolved general path must retain the ordinary GC slot note" + ); + + let holes = js_array_alloc_with_length(2); + assert_eq!( + indexing::try_strict_dense_index_set(holes, 0, 1.0), + None, + "a hole may be intercepted by a prototype setter and is not an existing own slot" + ); + + let header = unsafe { + (arr as *mut u8) + .sub(crate::gc::GC_HEADER_SIZE) + .cast::() + }; + unsafe { (*header)._reserved |= crate::gc::OBJ_FLAG_FROZEN }; + assert_eq!( + indexing::try_strict_dense_index_set(arr, 1, 44.0), + None, + "the fast path must leave strict frozen-array throwing to the fallback" + ); +} + #[test] fn test_array_push_f64_no_grow_fast_path() { let arr = js_array_alloc(4); diff --git a/crates/perry-runtime/src/builtins/arithmetic.rs b/crates/perry-runtime/src/builtins/arithmetic.rs index 810ce20e5b..0fe88b2381 100644 --- a/crates/perry-runtime/src/builtins/arithmetic.rs +++ b/crates/perry-runtime/src/builtins/arithmetic.rs @@ -424,14 +424,16 @@ unsafe fn string_content_for_bigint(value: f64) -> String { String::from_utf8_lossy(bytes).into_owned() } -/// Both operands already numeric (plain IEEE double or int32-tagged)? +/// Can this primitive operand be converted to Number without allocation, +/// user code, or observable coercion ordering? /// /// `abstract_relational` opens a `RuntimeHandleScope`, roots both operands and /// runs `ToPrimitive` on each — necessary only because a *heap* operand can run -/// user `valueOf`/`toString`. For a number `ToPrimitive` is the identity and -/// there is no pointer to root, so the whole apparatus is dead weight. This is -/// the same predicate and the same reasoning `dynamic_arith`'s binary operators -/// already use; the relational operators were simply never given it. +/// user `valueOf`/`toString`. Numbers, undefined, null, and booleans contain no +/// pointer and their ToPrimitive/ToNumber results are fixed by the spec, so the +/// whole apparatus is dead weight. Strings stay on the full path because two +/// strings compare lexicographically; BigInts, Symbols, objects, and internal +/// sentinels stay there for their distinct semantics and errors. /// /// NaN must stay `false` for all four operators, which Rust's `<`/`>`/`<=`/`>=` /// on `f64` already deliver. @@ -445,7 +447,12 @@ fn rel_numeric_operand(v: f64) -> Option { if jv.is_int32() { return Some(jv.as_int32() as f64); } - None + match v.to_bits() { + crate::value::TAG_UNDEFINED => Some(f64::NAN), + crate::value::TAG_NULL | crate::value::TAG_FALSE => Some(0.0), + crate::value::TAG_TRUE => Some(1.0), + _ => None, + } } /// `x < y` — codegen routes here for any relational `<` whose operands are not @@ -666,38 +673,50 @@ pub(crate) fn typeof_string_cache_cells_for_test() -> [*mut StringHeader; 8] { typeof_cache_entries_for_test().map(|(cache, _)| cache.with(|cell| cell.get())) } -/// Return the typeof a value as a string -/// Takes an f64 that uses NaN-boxing to distinguish types. -/// Returns a pointer to a string: "undefined", "boolean", "number", "string", "object", "function" -/// -/// Optimization: typeof only returns 8 possible strings, so we cache them as -/// pre-allocated StringHeader pointers to avoid heap allocation on every call. -/// The cache is a registered GC root — see the `thread_local!` above. -#[no_mangle] -pub extern "C" fn js_value_typeof(value: f64) -> *mut StringHeader { +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +enum ValueTypeofTag { + Undefined = 0, + Object = 1, + Boolean = 2, + Number = 3, + String = 4, + Function = 5, + BigInt = 6, + Symbol = 7, +} + +/// Classify a value once, independently of the string representation exposed +/// by the `typeof` operator. Keeping every exceptional Perry representation in +/// this one classifier makes the literal-comparison entry point below exactly +/// agree with [`js_value_typeof`]: class refs, callable proxies, raw typed-array +/// pointers, stream handles, Symbols, closures, and class-expression objects +/// cannot drift between the two APIs. +#[inline] +fn classify_value_typeof(value: f64) -> ValueTypeofTag { let jsval = JSValue::from_bits(value.to_bits()); if jsval.is_undefined() { - get_cached(&TYPEOF_UNDEFINED, "undefined") + ValueTypeofTag::Undefined } else if jsval.is_null() { // typeof null === "object" in JavaScript - get_cached(&TYPEOF_OBJECT, "object") + ValueTypeofTag::Object } else if jsval.is_bool() { - get_cached(&TYPEOF_BOOLEAN, "boolean") + ValueTypeofTag::Boolean } else if jsval.is_any_string() { // String pointer (STRING_TAG) OR inline SSO (SHORT_STRING_TAG). // `typeof` doesn't distinguish between representations — both // are observed as "string" from user code. - get_cached(&TYPEOF_STRING, "string") + ValueTypeofTag::String } else if crate::value::is_js_handle(value) { // JS handle from V8 runtime — ask V8 whether it's a callable, otherwise default // to "object". Issue #258: pre-fix this always returned "object" even for // V8 functions; the registered callback now flips it to "function" when the // handle wraps a v8::Function. if crate::value::js_handle_is_function(value) { - get_cached(&TYPEOF_FUNCTION, "function") + ValueTypeofTag::Function } else { - get_cached(&TYPEOF_OBJECT, "object") + ValueTypeofTag::Object } } else if jsval.is_pointer() { // Object/array/closure/symbol pointer - check via the side-table first. @@ -713,43 +732,43 @@ pub extern "C" fn js_value_typeof(value: f64) -> *mut StringHeader { // (possibly nested) [[ProxyTarget]] is callable. if crate::proxy::js_proxy_is_proxy(value) == 1 { return if crate::proxy::proxy_wraps_callable(value) { - get_cached(&TYPEOF_FUNCTION, "function") + ValueTypeofTag::Function } else { - get_cached(&TYPEOF_OBJECT, "object") + ValueTypeofTag::Object }; } if crate::value::addr_class::is_above_handle_band(ptr as usize) { // Symbols: registered in SYMBOL_POINTERS (handles both gc_malloc'd // and Box-leaked symbols, which have no GcHeader). if crate::symbol::is_registered_symbol(ptr as usize) { - get_cached(&TYPEOF_SYMBOL, "symbol") + ValueTypeofTag::Symbol } else if crate::date::is_date_cell_addr(ptr as usize) { // Date is a NaN-boxed pointer to an 8-byte `DateCell` (#2089). // `typeof aDate === "object"`. Check this BEFORE reading the // `type_tag` at offset 12 below — the cell is only 8 bytes, so // that read would fall off the end of the allocation. - get_cached(&TYPEOF_OBJECT, "object") + ValueTypeofTag::Object } else { // ClosureHeader has type_tag at offset 12 (after func_ptr:8 + capture_count:4) let type_tag = unsafe { *(ptr.add(crate::closure::CLOSURE_TYPE_TAG_OFFSET) as *const u32) }; if type_tag == crate::closure::CLOSURE_MAGIC { - get_cached(&TYPEOF_FUNCTION, "function") + ValueTypeofTag::Function } else if crate::object::is_class_object_ptr(ptr) { // #1789: a class-expression VALUE is a heap object stamped // with OBJECT_TYPE_CLASS — `typeof aClassObject === // "function"` (classes are callable in JS), matching the // INT32 ClassRef case below. - get_cached(&TYPEOF_FUNCTION, "function") + ValueTypeofTag::Function } else { - get_cached(&TYPEOF_OBJECT, "object") + ValueTypeofTag::Object } } } else { - get_cached(&TYPEOF_OBJECT, "object") + ValueTypeofTag::Object } } else if jsval.is_bigint() { - get_cached(&TYPEOF_BIGINT, "bigint") + ValueTypeofTag::BigInt } else if jsval.is_int32() { // Refs #618 / #420 followup: class refs share INT32_TAG storage // shape (codegen emits `INT32_TAG | class_id` as the value form @@ -759,9 +778,9 @@ pub extern "C" fn js_value_typeof(value: f64) -> *mut StringHeader { let raw = jsval.bits() & 0xFFFF_FFFF; let class_id = raw as u32; if crate::object::is_class_id_registered(class_id) { - get_cached(&TYPEOF_FUNCTION, "function") + ValueTypeofTag::Function } else { - get_cached(&TYPEOF_NUMBER, "number") + ValueTypeofTag::Number } } else { // Issue #654: typed-array pointers arrive as a raw `i64 → f64` @@ -776,7 +795,7 @@ pub extern "C" fn js_value_typeof(value: f64) -> *mut StringHeader { if top16 == 0 && bits >= 0x10000 { let addr = bits as usize; if crate::typedarray::lookup_typed_array_kind(addr).is_some() { - return get_cached(&TYPEOF_OBJECT, "object"); + return ValueTypeofTag::Object; } } // Date is now a NaN-boxed `DateCell` pointer (#2089), handled in the @@ -792,12 +811,42 @@ pub extern "C" fn js_value_typeof(value: f64) -> *mut StringHeader { if value.is_finite() && value > 0.0 && value.fract() == 0.0 { if let Some(probe) = crate::object::stream_handle_kind_probe() { if unsafe { probe(value as usize) } != 0 { - return get_cached(&TYPEOF_OBJECT, "object"); + return ValueTypeofTag::Object; } } } // Regular f64 number - get_cached(&TYPEOF_NUMBER, "number") + ValueTypeofTag::Number + } +} + +/// Integer form of `typeof`, for comparisons against a compile-time literal. +/// Avoids materializing a cached heap string and then comparing its contents. +/// The numeric values are part of the codegen/runtime ABI; keep them in sync +/// with `TYPEOF_LITERAL_TAGS` in `perry-codegen/src/expr/compare.rs`. +#[no_mangle] +pub extern "C" fn js_value_typeof_tag(value: f64) -> u32 { + classify_value_typeof(value) as u32 +} + +/// Return the typeof a value as a string +/// Takes an f64 that uses NaN-boxing to distinguish types. +/// Returns a pointer to a string: "undefined", "boolean", "number", "string", "object", "function" +/// +/// Optimization: typeof only returns 8 possible strings, so each classified +/// result is mapped to a pre-allocated cached StringHeader pointer. The cache +/// is a registered GC root — see the `thread_local!` above. +#[no_mangle] +pub extern "C" fn js_value_typeof(value: f64) -> *mut StringHeader { + match classify_value_typeof(value) { + ValueTypeofTag::Undefined => get_cached(&TYPEOF_UNDEFINED, "undefined"), + ValueTypeofTag::Object => get_cached(&TYPEOF_OBJECT, "object"), + ValueTypeofTag::Boolean => get_cached(&TYPEOF_BOOLEAN, "boolean"), + ValueTypeofTag::Number => get_cached(&TYPEOF_NUMBER, "number"), + ValueTypeofTag::String => get_cached(&TYPEOF_STRING, "string"), + ValueTypeofTag::Function => get_cached(&TYPEOF_FUNCTION, "function"), + ValueTypeofTag::BigInt => get_cached(&TYPEOF_BIGINT, "bigint"), + ValueTypeofTag::Symbol => get_cached(&TYPEOF_SYMBOL, "symbol"), } } @@ -818,11 +867,38 @@ mod rel_numeric_fastpath_tests { v.to_bits() == TAG_TRUE_BITS } - /// The early-out accepts exactly the operands for which `ToPrimitive` is - /// the identity and there is nothing to root; everything else must fall - /// through to the full abstract relational comparison. #[test] - fn fast_path_accepts_only_numbers() { + fn integer_typeof_classifier_covers_the_primitive_tag_families() { + let cases = [ + (f64::from_bits(UNDEF), ValueTypeofTag::Undefined), + (f64::from_bits(NULLV), ValueTypeofTag::Object), + (f64::from_bits(TRUEV), ValueTypeofTag::Boolean), + (42.5, ValueTypeofTag::Number), + (i32v(-123), ValueTypeofTag::Number), + ( + f64::from_bits(crate::value::SHORT_STRING_TAG | (1_u64 << 40) | b'x' as u64), + ValueTypeofTag::String, + ), + ( + f64::from_bits(crate::value::BIGINT_TAG | 1), + ValueTypeofTag::BigInt, + ), + ( + f64::from_bits(crate::value::POINTER_TAG | 42), + ValueTypeofTag::Object, + ), + ]; + for (value, expected) in cases { + assert_eq!(classify_value_typeof(value), expected); + assert_eq!(js_value_typeof_tag(value), expected as u32); + } + } + + /// The early-out accepts exactly the operands whose ToPrimitive/ToNumber + /// result is fixed and which have nothing to root; everything else must + /// fall through to the full abstract relational comparison. + #[test] + fn fast_path_accepts_numbers_and_simple_singletons() { assert!(rel_numeric_operand(1.5).is_some()); assert!(rel_numeric_operand(-0.0).is_some()); assert!(rel_numeric_operand(f64::INFINITY).is_some()); @@ -830,12 +906,31 @@ mod rel_numeric_fastpath_tests { assert!(rel_numeric_operand(f64::NAN).is_some()); assert_eq!(rel_numeric_operand(i32v(7)), Some(7.0)); assert_eq!(rel_numeric_operand(i32v(-7)), Some(-7.0)); - for tag in [UNDEF, NULLV, FALSEV, TRUEV] { - assert!( - rel_numeric_operand(f64::from_bits(tag)).is_none(), - "tag {tag:#x} must not take the numeric fast path" - ); + assert!(rel_numeric_operand(f64::from_bits(UNDEF)).unwrap().is_nan()); + assert_eq!(rel_numeric_operand(f64::from_bits(NULLV)), Some(0.0)); + assert_eq!(rel_numeric_operand(f64::from_bits(FALSEV)), Some(0.0)); + assert_eq!(rel_numeric_operand(f64::from_bits(TRUEV)), Some(1.0)); + } + + #[test] + fn simple_singleton_relational_comparisons_match_to_number() { + let undefined = f64::from_bits(UNDEF); + let null = f64::from_bits(NULLV); + let false_value = f64::from_bits(FALSEV); + let true_value = f64::from_bits(TRUEV); + + for got in [ + js_rel_lt(undefined, 1.0), + js_rel_gt(undefined, 1.0), + js_rel_le(undefined, 1.0), + js_rel_ge(undefined, 1.0), + ] { + assert!(!is_true(got), "undefined must compare as NaN"); } + assert!(is_true(js_rel_lt(null, 1.0))); + assert!(is_true(js_rel_ge(null, 0.0))); + assert!(is_true(js_rel_le(false_value, 0.0))); + assert!(is_true(js_rel_gt(true_value, 0.0))); } /// NaN makes all four operators false. This is the one way a naive `fcmp` diff --git a/crates/perry-runtime/src/gc/layout.rs b/crates/perry-runtime/src/gc/layout.rs index 712a5f5751..ca7b5f0cf3 100644 --- a/crates/perry-runtime/src/gc/layout.rs +++ b/crates/perry-runtime/src/gc/layout.rs @@ -950,6 +950,65 @@ pub(crate) fn layout_note_slot(parent_user: usize, slot_index: usize, value_bits } } +/// Existing-slot layout note with the value that was overwritten. +/// +/// The GC slot mask records one bit of information: whether a slot can carry a +/// heap edge. Replacing one pointer-bearing value with another cannot change +/// that bit. Arrays still have a second, independent invariant to maintain — +/// their homogeneous element-shape record — so the pointer-over-pointer path +/// runs that hook after validating/chasing the owner header and then stops +/// before the typed-layout and per-slot-mask machinery. Object-backed packed +/// numeric proofs are retired for the same reason as in [`layout_note_slot`]. +/// +/// Scalar-over-scalar keeps the historical fast return. A change in either +/// direction uses the complete note so pointer masks and typed descriptors are +/// updated exactly as before. +#[inline] +pub(crate) fn layout_note_slot_aware( + parent_user: usize, + slot_index: usize, + value_bits: u64, + old_bits: u64, +) { + let value_is_pointer = layout_pointer_bearing_bits(value_bits); + let old_is_pointer = layout_pointer_bearing_bits(old_bits); + if !value_is_pointer && !old_is_pointer { + return; + } + if value_is_pointer && old_is_pointer { + if slot_index > 16_000_000 { + return; + } + unsafe { + let Some(header) = layout_header_for_user(parent_user) else { + return; + }; + if (*header).gc_flags & GC_FLAG_FORWARDED != 0 { + let new_user = forwarding_address(header) as usize; + if new_user != 0 && new_user != parent_user { + layout_note_slot_aware(new_user, slot_index, value_bits, old_bits); + } + return; + } + if (*header).obj_type == GC_TYPE_ARRAY { + crate::array::note_element_store( + parent_user as *mut crate::array::ArrayHeader, + slot_index, + value_bits, + ); + } else if (*header).obj_type == GC_TYPE_OBJECT + && (*header)._reserved & OBJ_FLAG_PACKED_NUMERIC_PROOF != 0 + { + crate::array::clear_packed_subclass_numeric_proof( + parent_user as *mut crate::object::ObjectHeader, + ); + } + } + return; + } + layout_note_slot(parent_user, slot_index, value_bits); +} + /// True when `slot_index` of `parent_user` is a **raw-f64-masked slot of an /// intact typed-shape descriptor** — i.e. exactly the case where /// [`layout_note_slot`] would call `layout_set_typed_unknown` (permanently @@ -1001,14 +1060,13 @@ pub extern "C" fn js_gc_note_slot_layout(parent: u64, slot_index: u32, value_bit layout_note_slot(parent_user, slot_index as usize, value_bits); } -/// Scalar-aware variant of [`js_gc_note_slot_layout`]: `old_bits` is the value -/// previously held in the slot. When **neither** the new value nor the old -/// value is a heap pointer, the slot's pointer-ness is unchanged, so the -/// per-slot GC layout mask needs no update — the `SIDE_MASK`/typed path's -/// thread-local hashmap touch is skipped. The mask invariant ("bit set ⟺ slot -/// holds a pointer") is preserved because the full path still runs whenever a -/// pointer is involved on either side (`new` is a pointer → set; `old` was a -/// pointer → clear), which is exactly when the mask must change. This is the +/// Value-aware variant of [`js_gc_note_slot_layout`]: `old_bits` is the value +/// previously held in the slot. When old and new have the same heap-pointer +/// classification, the per-slot GC layout mask needs no update. The +/// pointer-over-pointer path still maintains Array element-shape metadata; +/// classification changes retain the full typed-layout and mask pipeline. +/// The mask invariant ("bit set ⟺ slot holds a pointer") is therefore +/// preserved while avoiding the thread-local hashmap on stable overwrites. This is the /// dominant per-write cost on heterogeneous `any[]` numeric write loops /// (stubbing `layout_note_slot` makes `bench_numeric_array_downgrade` 11× /// faster). `layout_pointer_bearing_bits` is the same predicate the layout @@ -1021,11 +1079,8 @@ pub extern "C" fn js_gc_note_slot_layout_aware( value_bits: u64, old_bits: u64, ) { - if !layout_pointer_bearing_bits(value_bits) && !layout_pointer_bearing_bits(old_bits) { - return; - } let parent_user = strip_nanbox_user_ptr(parent); - layout_note_slot(parent_user, slot_index as usize, value_bits); + layout_note_slot_aware(parent_user, slot_index as usize, value_bits, old_bits); } pub(super) unsafe fn layout_rebuild_from_slots_with_policy( diff --git a/crates/perry-runtime/src/gc/telemetry.rs b/crates/perry-runtime/src/gc/telemetry.rs index 79d91c6a0e..af1c0acd58 100644 --- a/crates/perry-runtime/src/gc/telemetry.rs +++ b/crates/perry-runtime/src/gc/telemetry.rs @@ -49,6 +49,9 @@ impl GcStats { /// collection path can update one without the others. /// pub(super) fn record_collection(&mut self, freed_bytes: u64, elapsed_us: u64) { + // Generated Symbol-property ICs are weak raw-bit caches. Invalidate + // them before the mutator can observe any relocated/reused address. + crate::symbol::symbol_property_ic_epoch_bump(); self.collection_count += 1; self.total_freed_bytes = self.total_freed_bytes.saturating_add(freed_bytes); self.last_pause_us = elapsed_us; diff --git a/crates/perry-runtime/src/object/array_tail_transition.rs b/crates/perry-runtime/src/object/array_tail_transition.rs new file mode 100644 index 0000000000..ee4dbac80d --- /dev/null +++ b/crates/perry-runtime/src/object/array_tail_transition.rs @@ -0,0 +1,489 @@ +//! Exact bidirectional transitions for dense numeric object tails. +//! +//! Perry represents `class X extends Array` instances as shaped objects. A +//! generic tail delete therefore clones and compacts the complete ordered-key +//! array even when the runtime has already observed the inverse append. This +//! cache retains that learned `(predecessor, numeric key, successor)` edge in +//! both directions. It does not authorize mutation by itself: the Array +//! subclass fast path separately proves the receiver brand, dense layout, +//! descriptors, prototype state, live length, and physical value slot. + +use crate::object::shapes; + +pub(crate) const ARRAY_TAIL_TRANSITION_CACHE_SIZE: usize = 8192; +const ARRAY_TAIL_TRANSITION_CACHE_MASK: usize = ARRAY_TAIL_TRANSITION_CACHE_SIZE - 1; +const ARRAY_TAIL_DIRECT_INDEX_MISS: u16 = u16::MAX; + +/// Compact exact-shape accelerator into the authoritative rooted transition +/// tables. One ShapeId can simultaneously be the successor of one numeric +/// append and the predecessor of the next, hence the two independent indices. +/// A collision only evicts this accelerator entry; the open-addressed tables +/// remain complete and are the semantics-preserving fallback. +#[derive(Clone, Copy)] +pub(crate) struct ArrayTailDirectIndex { + shape_id: u32, + forward: u16, + reverse: u16, +} + +impl ArrayTailDirectIndex { + pub(crate) const EMPTY: Self = Self { + shape_id: 0, + forward: ARRAY_TAIL_DIRECT_INDEX_MISS, + reverse: ARRAY_TAIL_DIRECT_INDEX_MISS, + }; +} + +#[derive(Clone, Copy)] +pub(crate) struct ArrayTailTransitionEntry { + pub(crate) predecessor_keys: usize, + pub(crate) successor_keys: usize, + pub(crate) predecessor_shape_id: u32, + pub(crate) successor_shape_id: u32, + pub(crate) slot: u32, + pub(crate) array_index: u32, + pub(crate) predecessor_live_inline_slots: u32, + pub(crate) successor_live_inline_slots: u32, +} + +impl ArrayTailTransitionEntry { + pub(crate) const EMPTY: Self = Self { + predecessor_keys: 0, + successor_keys: 0, + predecessor_shape_id: 0, + successor_shape_id: 0, + slot: 0, + array_index: 0, + predecessor_live_inline_slots: 0, + successor_live_inline_slots: 0, + }; + + /// Deleted entries cannot become `EMPTY` without breaking a later entry's + /// open-addressing probe chain. ShapeIds are nonzero, so this pointer-only + /// marker is unambiguous and is never visited as a GC root. + const TOMBSTONE: Self = Self { + predecessor_keys: usize::MAX, + successor_keys: 0, + predecessor_shape_id: 0, + successor_shape_id: 0, + slot: 0, + array_index: 0, + predecessor_live_inline_slots: 0, + successor_live_inline_slots: 0, + }; + + #[inline(always)] + fn is_empty(self) -> bool { + self.successor_shape_id == 0 && self.predecessor_keys == 0 + } + + #[inline(always)] + fn is_tombstone(self) -> bool { + self.successor_shape_id == 0 && self.predecessor_keys == usize::MAX + } +} + +#[inline(always)] +fn forward_slot(shape_id: u32, index: u32) -> usize { + let mixed = u64::from(shape_id).wrapping_mul(0x9E37_79B9_7F4A_7C15) + ^ u64::from(index).wrapping_mul(0xC6BC_2796_92B5_C323); + mixed as usize & ARRAY_TAIL_TRANSITION_CACHE_MASK +} + +#[inline(always)] +fn reverse_slot(shape_id: u32) -> usize { + let mixed = u64::from(shape_id).wrapping_mul(0xD6E8_FEB8_6659_FD93); + (mixed ^ (mixed >> 32)) as usize & ARRAY_TAIL_TRANSITION_CACHE_MASK +} + +#[inline] +unsafe fn publish_entry(target: &mut ArrayTailTransitionEntry, entry: ArrayTailTransitionEntry) { + target.predecessor_shape_id = entry.predecessor_shape_id; + target.successor_shape_id = entry.successor_shape_id; + target.slot = entry.slot; + target.array_index = entry.array_index; + target.predecessor_live_inline_slots = entry.predecessor_live_inline_slots; + target.successor_live_inline_slots = entry.successor_live_inline_slots; + crate::gc::runtime_store_root_usize_slot(&mut target.predecessor_keys, entry.predecessor_keys); + crate::gc::runtime_store_root_usize_slot(&mut target.successor_keys, entry.successor_keys); +} + +unsafe fn insert_forward( + table: *mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE], + entry: ArrayTailTransitionEntry, +) -> Option { + let start = forward_slot(entry.predecessor_shape_id, entry.array_index); + let mut tombstone = None; + for offset in 0..ARRAY_TAIL_TRANSITION_CACHE_SIZE { + let index = (start + offset) & ARRAY_TAIL_TRANSITION_CACHE_MASK; + let candidate = (*table)[index]; + if candidate.predecessor_shape_id == entry.predecessor_shape_id + && candidate.array_index == entry.array_index + && candidate.successor_shape_id != 0 + { + publish_entry(&mut (*table)[index], entry); + return Some(index); + } + if candidate.is_tombstone() && tombstone.is_none() { + tombstone = Some(index); + } else if candidate.is_empty() { + let index = tombstone.unwrap_or(index); + publish_entry(&mut (*table)[index], entry); + return Some(index); + } + } + if let Some(index) = tombstone { + publish_entry(&mut (*table)[index], entry); + return Some(index); + } + None +} + +unsafe fn insert_reverse( + table: *mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE], + entry: ArrayTailTransitionEntry, +) -> Option { + let start = reverse_slot(entry.successor_shape_id); + let mut tombstone = None; + for offset in 0..ARRAY_TAIL_TRANSITION_CACHE_SIZE { + let index = (start + offset) & ARRAY_TAIL_TRANSITION_CACHE_MASK; + let candidate = (*table)[index]; + if candidate.successor_shape_id == entry.successor_shape_id { + publish_entry(&mut (*table)[index], entry); + return Some(index); + } + if candidate.is_tombstone() && tombstone.is_none() { + tombstone = Some(index); + } else if candidate.is_empty() { + let index = tombstone.unwrap_or(index); + publish_entry(&mut (*table)[index], entry); + return Some(index); + } + } + if let Some(index) = tombstone { + publish_entry(&mut (*table)[index], entry); + return Some(index); + } + None +} + +#[inline] +fn with_forward( + f: impl FnOnce(*mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE]) -> R, +) -> R { + unsafe { + let boxed = &mut *crate::state::state().object_hot.array_tail_forward.get(); + f(boxed.as_mut_ptr() as *mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE]) + } +} + +#[inline] +fn with_reverse( + f: impl FnOnce(*mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE]) -> R, +) -> R { + unsafe { + let boxed = &mut *crate::state::state().object_hot.array_tail_reverse.get(); + f(boxed.as_mut_ptr() as *mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE]) + } +} + +/// Resolve the transition tables through an Array-subclass receiver after its +/// first learned edge. `RuntimeState` is heap allocated and stable until this +/// thread exits, while ObjectHeaders never cross agents (worker inputs are +/// deep-copied), so the native pointer can move with ObjectMeta without GC +/// tracing or rewriting. +#[inline(always)] +fn object_hot_for_owner( + owner: *const crate::object::ObjectHeader, +) -> &'static crate::object::ObjectHotTables { + unsafe { + if !owner.is_null() { + let meta = (*owner).meta; + if !meta.is_null() { + let cached = + (*meta).array_tail_object_hot as usize as *const crate::object::ObjectHotTables; + if !cached.is_null() { + return &*cached; + } + let hot = &crate::state::state().object_hot; + // GC_STORE_AUDIT(NATIVE_POINTER): RuntimeState storage, not a + // managed heap edge; ObjectMeta's GC descriptors intentionally + // visit only prototype, spill, and private brand. + (*meta).array_tail_object_hot = hot as *const _ as usize as u64; + return hot; + } + } + } + &crate::state::state().object_hot +} + +#[inline(always)] +fn with_forward_for_owner( + owner: *const crate::object::ObjectHeader, + f: impl FnOnce(*mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE]) -> R, +) -> R { + unsafe { + let boxed = &mut *object_hot_for_owner(owner).array_tail_forward.get(); + f(boxed.as_mut_ptr() as *mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE]) + } +} + +#[inline(always)] +fn with_reverse_for_owner( + owner: *const crate::object::ObjectHeader, + f: impl FnOnce(*mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE]) -> R, +) -> R { + unsafe { + let boxed = &mut *object_hot_for_owner(owner).array_tail_reverse.get(); + f(boxed.as_mut_ptr() as *mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE]) + } +} + +#[inline(always)] +fn direct_slot(shape_id: u32) -> usize { + shape_id as usize & ARRAY_TAIL_TRANSITION_CACHE_MASK +} + +#[inline] +fn publish_direct_index( + owner: *const crate::object::ObjectHeader, + shape_id: u32, + forward: Option, + reverse: Option, +) { + let hot = object_hot_for_owner(owner); + let direct = unsafe { &mut *hot.array_tail_direct.get() }; + let entry = &mut direct[direct_slot(shape_id)]; + if entry.shape_id != shape_id { + *entry = ArrayTailDirectIndex { + shape_id, + ..ArrayTailDirectIndex::EMPTY + }; + } + if let Some(index) = forward { + debug_assert!(index < ARRAY_TAIL_TRANSITION_CACHE_SIZE); + entry.forward = index as u16; + } + if let Some(index) = reverse { + debug_assert!(index < ARRAY_TAIL_TRANSITION_CACHE_SIZE); + entry.reverse = index as u16; + } +} + +#[inline] +fn canonical_index(key: *const crate::StringHeader) -> Option { + unsafe { + crate::object::has_own_helpers::str_from_string_header(key) + .and_then(crate::object::canonical_array_index) + } +} + +#[inline] +fn descriptor_pair_is_exact(entry: ArrayTailTransitionEntry) -> bool { + let Some(predecessor) = shapes::shape_descriptor_by_id(entry.predecessor_shape_id) else { + return false; + }; + let Some(successor) = shapes::shape_descriptor_by_id(entry.successor_shape_id) else { + return false; + }; + predecessor.keys as usize == entry.predecessor_keys + && successor.keys as usize == entry.successor_keys + && predecessor.object_kind == successor.object_kind + && predecessor.logical_key_count == entry.slot + && successor.logical_key_count == entry.slot.saturating_add(1) + && entry.predecessor_keys >= crate::gc::GC_HEADER_SIZE + && entry.successor_keys >= crate::gc::GC_HEADER_SIZE + && unsafe { + let predecessor_header = (entry.predecessor_keys as *const u8) + .sub(crate::gc::GC_HEADER_SIZE) + .cast::(); + let successor_header = (entry.successor_keys as *const u8) + .sub(crate::gc::GC_HEADER_SIZE) + .cast::(); + (*predecessor_header).obj_type == crate::gc::GC_TYPE_ARRAY + && (*successor_header).obj_type == crate::gc::GC_TYPE_ARRAY + } +} + +pub(crate) fn record_numeric_tail_transition( + owner: *const crate::object::ObjectHeader, + predecessor_shape_id: u32, + successor_shape_id: u32, + key: *const crate::StringHeader, + successor_keys: usize, + slot: u32, +) { + let Some(array_index) = canonical_index(key) else { + return; + }; + if owner.is_null() + || !crate::array::array_subclass_tail_descriptors_are_plain(owner, array_index) + { + return; + } + let Some(predecessor) = shapes::shape_descriptor_by_id(predecessor_shape_id) else { + return; + }; + let Some(successor) = shapes::shape_descriptor_by_id(successor_shape_id) else { + return; + }; + let entry = ArrayTailTransitionEntry { + predecessor_keys: predecessor.keys as usize, + successor_keys, + predecessor_shape_id, + successor_shape_id, + slot, + array_index, + predecessor_live_inline_slots: predecessor.live_inline_slot_count, + successor_live_inline_slots: successor.live_inline_slot_count, + }; + if successor.keys as usize != successor_keys + || predecessor.logical_key_count != slot + || successor.logical_key_count != slot.saturating_add(1) + || predecessor.object_kind != successor.object_kind + || !descriptor_pair_is_exact(entry) + { + return; + } + unsafe { + shapes::note_cache_carrier(Some(predecessor)); + shapes::note_cache_carrier(Some(successor)); + } + // A single lost edge poisons the remainder of a shrinking dense shape + // chain: the generic fallback mints a different predecessor, after which + // no historical edge can match. Preserve colliding entries with bounded + // open addressing instead of overwriting one direct-mapped slot. + let forward = with_forward_for_owner(owner, |table| unsafe { insert_forward(table, entry) }); + let reverse = with_reverse_for_owner(owner, |table| unsafe { insert_reverse(table, entry) }); + if let Some(index) = forward { + publish_direct_index(owner, predecessor_shape_id, Some(index), None); + } + if let Some(index) = reverse { + publish_direct_index(owner, successor_shape_id, None, Some(index)); + } +} + +#[inline] +pub(crate) fn lookup_forward_for_owner( + owner: *const crate::object::ObjectHeader, + predecessor_shape_id: u32, + array_index: u32, +) -> Option { + let hot = object_hot_for_owner(owner); + let direct = unsafe { &*hot.array_tail_direct.get() }; + let cached = direct[direct_slot(predecessor_shape_id)]; + let table = unsafe { &mut *hot.array_tail_forward.get() }; + if cached.shape_id == predecessor_shape_id && cached.forward != ARRAY_TAIL_DIRECT_INDEX_MISS { + let entry = table[cached.forward as usize]; + if entry.predecessor_shape_id == predecessor_shape_id + && entry.array_index == array_index + && entry.successor_shape_id != 0 + { + return Some(entry); + } + } + let start = forward_slot(predecessor_shape_id, array_index); + for offset in 0..ARRAY_TAIL_TRANSITION_CACHE_SIZE { + let entry = table[(start + offset) & ARRAY_TAIL_TRANSITION_CACHE_MASK]; + if entry.is_empty() { + return None; + } + if entry.predecessor_shape_id == predecessor_shape_id && entry.array_index == array_index { + return Some(entry); + } + } + None +} + +#[cfg(test)] +#[inline] +pub(crate) fn lookup_reverse(successor_shape_id: u32) -> Option { + with_reverse(|table| unsafe { + let start = reverse_slot(successor_shape_id); + for offset in 0..ARRAY_TAIL_TRANSITION_CACHE_SIZE { + let entry = (*table)[(start + offset) & ARRAY_TAIL_TRANSITION_CACHE_MASK]; + if entry.is_empty() { + return None; + } + if entry.successor_shape_id == successor_shape_id { + return Some(entry); + } + } + None + }) +} + +#[inline] +pub(crate) fn lookup_reverse_for_owner( + owner: *const crate::object::ObjectHeader, + successor_shape_id: u32, +) -> Option { + let hot = object_hot_for_owner(owner); + let direct = unsafe { &*hot.array_tail_direct.get() }; + let cached = direct[direct_slot(successor_shape_id)]; + let table = unsafe { &mut *hot.array_tail_reverse.get() }; + if cached.shape_id == successor_shape_id && cached.reverse != ARRAY_TAIL_DIRECT_INDEX_MISS { + let entry = table[cached.reverse as usize]; + if entry.successor_shape_id == successor_shape_id { + return Some(entry); + } + } + let start = reverse_slot(successor_shape_id); + for offset in 0..ARRAY_TAIL_TRANSITION_CACHE_SIZE { + let entry = table[(start + offset) & ARRAY_TAIL_TRANSITION_CACHE_MASK]; + if entry.is_empty() { + return None; + } + if entry.successor_shape_id == successor_shape_id { + return Some(entry); + } + } + None +} + +unsafe fn scan_table( + table: *mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE], + visitor: &mut crate::gc::RuntimeRootVisitor<'_>, +) { + for index in 0..ARRAY_TAIL_TRANSITION_CACHE_SIZE { + let entry = &mut (*table)[index]; + if entry.successor_shape_id != 0 { + visitor.visit_usize_slot(&mut entry.predecessor_keys); + visitor.visit_usize_slot(&mut entry.successor_keys); + } + } +} + +pub(crate) fn scan_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + with_forward(|table| unsafe { scan_table(table, visitor) }); + with_reverse(|table| unsafe { scan_table(table, visitor) }); +} + +unsafe fn prune_table(table: *mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE]) { + for index in 0..ARRAY_TAIL_TRANSITION_CACHE_SIZE { + let entry = &mut (*table)[index]; + if entry.successor_shape_id != 0 && !descriptor_pair_is_exact(*entry) { + *entry = ArrayTailTransitionEntry::TOMBSTONE; + } + } +} + +#[cold] +pub(crate) fn prune_invalid_entries() { + with_forward(|table| unsafe { prune_table(table) }); + with_reverse(|table| unsafe { prune_table(table) }); +} + +#[cfg(test)] +pub(crate) fn test_clear() { + with_forward(|table| unsafe { + for entry in (*table).iter_mut() { + *entry = ArrayTailTransitionEntry::EMPTY; + } + }); + with_reverse(|table| unsafe { + for entry in (*table).iter_mut() { + *entry = ArrayTailTransitionEntry::EMPTY; + } + }); +} diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index 89c5a87662..48d38edef0 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -291,8 +291,9 @@ pub(crate) use ic_miss::{ }; pub use ic_miss::{ js_class_field_add, js_object_get_field_by_name_f64, js_object_get_field_by_property_id_f64, - js_object_get_field_ic_miss, js_object_set_field_by_property_id, js_private_brand_add, - js_private_brand_check, js_private_field_add, js_private_guard, PicCache, PIC_CACHE_WORDS, + js_object_get_field_ic, js_object_get_field_ic_miss, js_object_set_field_by_property_id, + js_private_brand_add, js_private_brand_check, js_private_field_add, js_private_guard, PicCache, + PIC_CACHE_WORDS, }; #[cfg(test)] diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index a0aa5f574f..b3a5323abf 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -232,7 +232,7 @@ pub const PIC_CACHE_WORDS: usize = 12; /// |---|---| /// | 0 | `tok0` — most-recently-used ShapeId token | /// | 1 | `slot0` — its resolved field slot | -/// | 2 | reserved non-identity scratch | +/// | 2 | optional Array-subclass class-declared named-prefix token | /// | 3,4 / 5,6 / 7,8 / 9,10 | `(tok, slot)` ways | /// | 11 | round-robin victim index for the ways | pub type PicCache = [i64; PIC_CACHE_WORDS]; @@ -698,16 +698,14 @@ pub extern "C" fn js_object_get_field_ic_miss( let is_regular = shape.is_some_and(|shape| { shape.object_kind == crate::object::shapes::ShapeObjectKind::Ordinary }); - // Descriptor-bearing receivers must not prime this PIC: its generated - // hit path is a raw slot load and would bypass their getter / property - // semantics. This per-object bit replaces the old process-wide - // `accessors_in_use` gate. `note_descriptor_target` sets the bit and - // transitions the receiver's ShapeId before an installed descriptor is - // observable, while unrelated accessors cannot affect an own data - // property on this receiver. The emitted hit path independently checks - // the same bit, so both cache population and cache use fail closed. - // Gate-neutral builtin accessors set the owner bit as well. - if is_regular && !has_own_descriptors { + // Descriptor-bearing receivers ordinarily must not prime a raw-load + // PIC. One narrow exception is an object-backed Array subclass whose + // complete class-declared prefix has been proved data-only: its + // unrelated `length` descriptor must not make `arch.sset` / `mask` / + // `change` permanently generic. The proof below is class-wide but + // owner-authorized, and every descriptor/structural transition clears + // the owner's token before publication. + if is_regular { let Some(shape) = shape else { let value = js_object_get_field_by_name(obj, key); return f64::from_bits(value.bits()); @@ -749,6 +747,22 @@ pub extern "C" fn js_object_get_field_ic_miss( // shape id — no second probe. let stamp = crate::object::shapes::object_shape_stamp(obj); let token = (stamp as u64 | crate::object::shapes::PIC_ID_TOKEN_BIT) as i64; + // Word 2 carries an optional class-declared named-prefix + // identity for object-backed Array subclasses. Their + // numeric tail changes ShapeId on every push/pop while + // declared fields keep the same slots. The proof builder + // is gated by an existing ObjectMeta pointer so ordinary + // objects retain the old miss cost; it validates the + // complete prefix before publishing a nonzero token. + let named_prefix_token = if !(*obj).meta.is_null() { + crate::array::array_subclass_named_prefix_token_for_slot(obj, i) as i64 + } else { + 0 + }; + if has_own_descriptors && named_prefix_token == 0 { + break; + } + (*cache)[2] = named_prefix_token; pic_prime_get(cache, token, i as i64); let field_ptr = (obj as *const u8) .add(std::mem::size_of::() + i * 8) diff --git a/crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs b/crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs index 2b6164e8cf..c9001ca497 100644 --- a/crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs +++ b/crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs @@ -399,7 +399,14 @@ mod tests { ) .expect("shape range unexpectedly exhausted"); let key = key_handle.get_raw_const_ptr::(); - super::super::transition_cache_insert(predecessor, key, next_keys as usize, slot, target); + super::super::transition_cache_insert( + std::ptr::null(), + predecessor, + key, + next_keys as usize, + slot, + target, + ); assert!( super::super::transition_cache_lookup(predecessor, key).is_some(), "test premise: the synthetic transition must be cache-resident" diff --git a/crates/perry-runtime/src/object/field_set_by_name/tail.rs b/crates/perry-runtime/src/object/field_set_by_name/tail.rs index 3a835ad3c7..4dbacea616 100644 --- a/crates/perry-runtime/src/object/field_set_by_name/tail.rs +++ b/crates/perry-runtime/src/object/field_set_by_name/tail.rs @@ -416,6 +416,7 @@ pub(crate) fn set_field_by_name_object_tail( let is_frozen = obj_flags & crate::gc::OBJ_FLAG_FROZEN != 0; let is_sealed_or_no_extend = obj_flags & (crate::gc::OBJ_FLAG_SEALED | crate::gc::OBJ_FLAG_NO_EXTEND) != 0; + let record_array_tail = crate::array::is_array_subclass_class_id((*obj).class_id); let keys = crate::object::object_keys_array(obj); @@ -577,6 +578,11 @@ pub(crate) fn set_field_by_name_object_tail( // fast path above instead of allocating a fresh 4-elem // keys_array here. transition_cache_insert( + if record_array_tail { + obj as *const ObjectHeader + } else { + std::ptr::null() + }, prev_shape_id, interned_key, new_keys as usize, @@ -761,6 +767,11 @@ pub(crate) fn set_field_by_name_object_tail( refresh_roots_after_alloc!(); mirror_class_object_static_write(obj, key, value); transition_cache_insert( + if record_array_tail { + obj as *const ObjectHeader + } else { + std::ptr::null() + }, prev_shape_id, interned_key, new_keys as usize, @@ -804,6 +815,11 @@ pub(crate) fn set_field_by_name_object_tail( refresh_roots_after_alloc!(); mirror_class_object_static_write(obj, key, value); transition_cache_insert( + if record_array_tail { + obj as *const ObjectHeader + } else { + std::ptr::null() + }, prev_shape_id, interned_key, new_keys as usize, @@ -983,6 +999,11 @@ pub(crate) fn set_field_by_name_object_tail( // `transition_cache_insert`, which triggers clone-on-extend // on either object if someone later appends past this key. transition_cache_insert( + if record_array_tail { + obj as *const ObjectHeader + } else { + std::ptr::null() + }, prev_shape_id, interned_key, new_keys as usize, @@ -1024,6 +1045,11 @@ pub(crate) fn set_field_by_name_object_tail( mirror_class_object_static_write(obj, key, value); // Record the shape transition — see above for semantics. transition_cache_insert( + if record_array_tail { + obj as *const ObjectHeader + } else { + std::ptr::null() + }, prev_shape_id, interned_key, new_keys as usize, diff --git a/crates/perry-runtime/src/object/mod.rs b/crates/perry-runtime/src/object/mod.rs index 985c9c7902..5d9d722603 100644 --- a/crates/perry-runtime/src/object/mod.rs +++ b/crates/perry-runtime/src/object/mod.rs @@ -443,6 +443,12 @@ pub(crate) struct ObjectHotTables { /// and keys_array == null. pub(crate) shape_inline_cache: std::cell::UnsafeCell<[ShapeCacheEntry; SHAPE_INLINE_CACHE_SIZE]>, + /// Pointer-free direct cache for immutable ShapeId object-kind facts. + /// ShapeIds are monotone and never reused; descriptor retirement clears a + /// matching entry. Keeping this beside the other per-agent shape tables + /// avoids borrowing the descriptor HashMap on repeated regular-object + /// checks (notably homogeneous Array element stores). + pub(crate) shape_kind_cache: std::cell::UnsafeCell>, /// Overflow map for shape_ids that collide in the inline cache. Values /// are `(keys_array, runtime_shape_id)` — see [`ShapeCacheEntry`]. pub(crate) shape_cache_overflow: RefCell>, @@ -452,6 +458,18 @@ pub(crate) struct ObjectHotTables { /// TLS layout when this lived in a `thread_local!`, and keeping it /// boxed inside the heap-allocated `RuntimeState` preserves that. pub(crate) transition_cache: std::cell::UnsafeCell>, + /// Bidirectional index over learned sequential numeric property appends. + /// Array-subclass `push`/`pop` uses it to restore an exact historical + /// ShapeId without cloning or compacting the ordered-keys array. + pub(crate) array_tail_forward: + std::cell::UnsafeCell>, + pub(crate) array_tail_reverse: + std::cell::UnsafeCell>, + /// Exact-ShapeId -> authoritative forward/reverse table indices. This is + /// an accelerator only: collisions and stale indices revalidate the full + /// entry and fall back to the complete open-addressed tables. + pub(crate) array_tail_direct: + std::cell::UnsafeCell>, } impl ObjectHotTables { @@ -466,6 +484,9 @@ impl ObjectHotTables { keys_array: std::ptr::null_mut(), }; SHAPE_INLINE_CACHE_SIZE], ), + shape_kind_cache: std::cell::UnsafeCell::new( + vec![0; shapes::SHAPE_KIND_CACHE_SIZE].into_boxed_slice(), + ), shape_cache_overflow: RefCell::new(HashMap::new()), transition_cache: std::cell::UnsafeCell::new( vec![ @@ -481,6 +502,27 @@ impl ObjectHotTables { ] .into_boxed_slice(), ), + array_tail_forward: std::cell::UnsafeCell::new( + vec![ + array_tail_transition::ArrayTailTransitionEntry::EMPTY; + array_tail_transition::ARRAY_TAIL_TRANSITION_CACHE_SIZE + ] + .into_boxed_slice(), + ), + array_tail_reverse: std::cell::UnsafeCell::new( + vec![ + array_tail_transition::ArrayTailTransitionEntry::EMPTY; + array_tail_transition::ARRAY_TAIL_TRANSITION_CACHE_SIZE + ] + .into_boxed_slice(), + ), + array_tail_direct: std::cell::UnsafeCell::new( + vec![ + array_tail_transition::ArrayTailDirectIndex::EMPTY; + array_tail_transition::ARRAY_TAIL_TRANSITION_CACHE_SIZE + ] + .into_boxed_slice(), + ), } } } @@ -565,6 +607,7 @@ fn keys_index_insert( shapes::shape_note_append(keys, new_count, key_hash, slot); } +pub(crate) mod array_tail_transition; mod call_method_depth; use call_method_depth::CallMethodDepthGuard; pub(crate) use call_method_depth::{call_method_depth_restore, call_method_depth_savepoint}; @@ -907,6 +950,7 @@ unsafe fn transition_cache_stamp_shape_shared(next_keys: usize) -> bool { } fn transition_cache_insert( + array_tail_owner: *const ObjectHeader, prev_shape_id: u32, interned_key: *const crate::StringHeader, next_keys: usize, @@ -940,6 +984,16 @@ fn transition_cache_insert( entry.slot_idx = slot_idx; entry.target_len = target_len; }); + if !array_tail_owner.is_null() { + array_tail_transition::record_numeric_tail_transition( + array_tail_owner, + prev_shape_id, + target_shape_id, + interned_key, + next_keys, + slot_idx, + ); + } // Small dynamic shapes are stabilized eagerly because otherwise // the original builder can grow the cached target in place and // force future lookups to reject it. Large one-off dictionaries @@ -981,6 +1035,7 @@ pub fn scan_transition_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisit } } }); + array_tail_transition::scan_roots_mut(visitor); } /// #8192: death pruning for the transition cache. @@ -1023,6 +1078,7 @@ pub(crate) fn prune_dead_transition_cache_entries(is_dead_owner: &dyn Fn(usize) } } }); + array_tail_transition::prune_invalid_entries(); } #[cfg(test)] @@ -1552,6 +1608,39 @@ pub struct ObjectMeta { /// property: private branding must not consume a user field slot, alter /// the ShapeId/key order, or become visible to enumeration. pub private_evaluation_brand: u64, + /// Exact class-declared named-prefix identity for an Array-subclass + /// receiver. Numeric tail mutations change the ordinary ShapeId on every + /// push/pop even though the named slots before that tail remain fixed. + /// Property-read PICs may use this nonzero scalar as a second identity + /// only after `array_subclass_named_prefix_token` has proved the current + /// keys against the class's registered allocation keys. Generic shape or + /// semantic transitions clear it; the exact learned numeric-tail + /// transition is the only publisher that deliberately preserves it. + pub array_subclass_named_prefix_token: u64, + /// Native pointer to this receiver's per-thread [`ObjectHotTables`]. + /// Array-subclass tail transitions are agent-local: their ShapeIds and + /// rooted key arrays belong to the same thread that owns the object. Once + /// a transition is learned, caching that stable heap allocation here lets + /// every later push/pop reach the full historical shape lattice without a + /// Darwin TLS/TSD lookup first. + /// + /// This is NOT a managed-heap edge and the ObjectMeta slot visitors must + /// deliberately ignore it. Perry workers deep-copy values into independent + /// arenas rather than sharing ObjectHeaders, so an object cannot carry the + /// pointer into another agent. The RuntimeState allocation outlives every + /// object in that thread. + pub array_tail_object_hot: u64, + /// Move-stable, receiver-local cache of the Array-subclass dense layout. + /// `array_subclass_dense_key` is `(class_id << 32) | ShapeId`; the two + /// payload words use the same packing as `array::subclass`'s global + /// collision cache. They contain scalar slot indices only, never managed + /// pointers. A generic semantic/structural mutation publishes a new + /// ShapeId before it becomes observable, so a stale payload misses by key + /// without a pointer-side-table invalidation walk. Exact learned numeric + /// tail transitions update these words directly. + pub array_subclass_dense_key: u64, + pub array_subclass_dense_slots: u64, + pub array_subclass_dense_bounds: u64, } pub(crate) const OBJECT_META_FLAG_PROTO_OVERRIDE: u64 = 1; @@ -1572,8 +1661,8 @@ pub(crate) unsafe fn object_is_regular(obj: *const ObjectHeader) -> bool { }; header.obj_type == crate::gc::GC_TYPE_OBJECT && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 - && shapes::object_shape_descriptor(obj) - .is_some_and(|shape| shape.object_kind == shapes::ShapeObjectKind::Ordinary) + && shapes::shape_object_kind_by_id((*obj).parent_class_id) + == Some(shapes::ShapeObjectKind::Ordinary) } #[inline] @@ -1593,6 +1682,8 @@ pub(crate) unsafe fn object_is_shaped(obj: *const ObjectHeader) -> bool { // ObjectMeta record and buffer elements one word past the ArrayHeader. Keep // codegen and these structs in lock-step. const _: () = assert!(std::mem::offset_of!(ObjectMeta, spill) == 32); +const _: () = assert!(std::mem::offset_of!(ObjectMeta, array_subclass_named_prefix_token) == 48); +const _: () = assert!(std::mem::offset_of!(ObjectMeta, array_tail_object_hot) == 56); const _: () = assert!(std::mem::size_of::() == 8); /// Fetch-or-allocate the per-object meta record. Caller must have already @@ -1626,6 +1717,11 @@ pub(crate) unsafe fn object_meta_ensure(obj: *mut ObjectHeader) -> *mut ObjectMe (*meta).flags = 0; (*meta).spill = 0; (*meta).private_evaluation_brand = 0; + (*meta).array_subclass_named_prefix_token = 0; + (*meta).array_tail_object_hot = 0; + (*meta).array_subclass_dense_key = 0; + (*meta).array_subclass_dense_slots = 0; + (*meta).array_subclass_dense_bounds = 0; // GC_STORE_AUDIT(BARRIERED): meta-record edge is a header-slot store // followed by an object-slot barrier, mirroring `set_object_keys_array`. (*obj).meta = meta; diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index ac8bdf412d..c76c2c48a7 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -87,6 +87,11 @@ pub(crate) struct ShapeDescriptor { /// Notes accumulated since the last full trace; adopted into `old_carrier` /// by [`rotate_old_carrier_epoch_after_full_trace`]. pub(crate) old_carrier_seen: bool, + /// A runtime optimization cache can reinstall this historical shape even + /// while no live object currently carries it. Such a cache is an explicit + /// strong metadata owner, so collection must root and rewrite `keys` before + /// weak descriptor pruning. + pub(crate) cache_carrier: bool, pub(crate) logical_key_count: u32, pub(crate) live_inline_slot_count: u32, /// Zero for ordinary structural shapes. Descriptor/prototype mutations @@ -127,6 +132,61 @@ pub(crate) enum ShapeObjectKind { Class, } +/// Per-agent direct cache for the immutable `object_kind` half of a ShapeId. +/// A collision only falls back to the descriptor table. Entries contain no +/// managed address, and descriptor retirement clears a matching id before it +/// can be observed without the authoritative table record. +pub(crate) const SHAPE_KIND_CACHE_SIZE: usize = 16_384; +const SHAPE_KIND_CACHE_MASK: usize = SHAPE_KIND_CACHE_SIZE - 1; +const SHAPE_KIND_ORDINARY: u64 = 1; +const SHAPE_KIND_CLASS: u64 = 2; + +#[inline(always)] +fn shape_kind_cache_slot(shape_id: u32) -> usize { + let mixed = u64::from(shape_id).wrapping_mul(0x9E37_79B9_7F4A_7C15); + (mixed ^ (mixed >> 32)) as usize & SHAPE_KIND_CACHE_MASK +} + +#[inline] +fn cached_shape_object_kind(shape_id: u32) -> Option { + let cache = unsafe { &mut *crate::state::state().object_hot.shape_kind_cache.get() }; + let packed = cache[shape_kind_cache_slot(shape_id)]; + if (packed >> 32) as u32 != shape_id { + return None; + } + match packed & 0xFFFF_FFFF { + SHAPE_KIND_ORDINARY => Some(ShapeObjectKind::Ordinary), + SHAPE_KIND_CLASS => Some(ShapeObjectKind::Class), + _ => None, + } +} + +#[inline] +fn publish_shape_object_kind(shape_id: u32, kind: ShapeObjectKind) { + let cache = unsafe { &mut *crate::state::state().object_hot.shape_kind_cache.get() }; + let tag = match kind { + ShapeObjectKind::Ordinary => SHAPE_KIND_ORDINARY, + ShapeObjectKind::Class => SHAPE_KIND_CLASS, + }; + cache[shape_kind_cache_slot(shape_id)] = (u64::from(shape_id) << 32) | tag; +} + +#[inline] +fn retire_cached_shape_object_kind(shape_id: u32) { + let cache = unsafe { &mut *crate::state::state().object_hot.shape_kind_cache.get() }; + let entry = &mut cache[shape_kind_cache_slot(shape_id)]; + if (*entry >> 32) as u32 == shape_id { + *entry = 0; + } +} + +#[cfg(test)] +#[inline] +fn clear_shape_object_kind_cache() { + let cache = unsafe { &mut *crate::state::state().object_hot.shape_kind_cache.get() }; + cache.fill(0); +} + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] struct ShapeFacts { keys: u64, @@ -279,6 +339,7 @@ fn remove_descriptor_and_reverse_indices(inner: &mut ShapeTableInner, id: u32) { let Some(descriptor) = inner.descriptors.remove(&id) else { return; }; + retire_cached_shape_object_kind(id); let facts = descriptor_facts_with_keys(*descriptor, descriptor.indexed_keys); remove_descriptor_id_from_facts_index(inner, facts, id); remove_descriptor_id_from_keys_index(inner, descriptor.indexed_keys, id); @@ -394,6 +455,7 @@ fn shape_descriptor_ensure_with_generation( record: 0, old_carrier: false, old_carrier_seen: false, + cache_carrier: false, logical_key_count, live_inline_slot_count, semantic_generation, @@ -471,6 +533,19 @@ pub(crate) fn shape_descriptor_by_id(shape_id: u32) -> Option { .map(|record| lift_descriptor(record)) } +/// Immutable ordinary-vs-class fact with a pointer-free, per-agent direct +/// cache. The first observation remains the authoritative descriptor lookup; +/// subsequent observations avoid the hot ShapeId HashMap borrow. +#[inline] +pub(crate) fn shape_object_kind_by_id(shape_id: u32) -> Option { + if let Some(kind) = cached_shape_object_kind(shape_id) { + return Some(kind); + } + let kind = shape_descriptor_by_id(shape_id)?.object_kind; + publish_shape_object_kind(shape_id, kind); + Some(kind) +} + /// Box a descriptor and stamp the record with its OWN address (#8112). /// /// Self-referential on purpose. The alternative — deriving the address in @@ -525,6 +600,22 @@ pub(crate) unsafe fn note_old_generation_carrier(descriptor: Option) { + let Some(descriptor) = descriptor else { + return; + }; + if descriptor.record == 0 { + return; + } + let record = descriptor.record as *mut ShapeDescriptor; + // GC_STORE_AUDIT(POINTER_FREE): liveness bookkeeping byte, never a heap reference. + (*record).cache_carrier = true; +} + /// Recompute the old-carrier gate from the trace that just finished. /// /// A FULL trace enumerates every live object, so the notes it accumulated are @@ -595,6 +686,7 @@ fn install_external_shape_id( record: 0, old_carrier: false, old_carrier_seen: false, + cache_carrier: false, logical_key_count, live_inline_slot_count, semantic_generation: 0, @@ -673,6 +765,78 @@ pub(crate) unsafe fn install_cached_object_shape_transition( expected_predecessor_shape_id: u32, target_shape_id: u32, _target_keys: *mut ArrayHeader, +) -> bool { + let target_key_count = if _target_keys.is_null() { + 0 + } else { + crate::array::keys_array_len_capped_to_capacity(_target_keys) as u32 + }; + install_cached_object_shape_version( + obj, + expected_predecessor_shape_id, + target_shape_id, + _target_keys, + target_key_count, + ) +} + +/// Install an exact historical shape version whose authoritative keys array +/// may have grown in place since the descriptor was minted. Reflection and +/// field tracing use the descriptor's logical bound, not the backing array's +/// later physical length. +#[inline] +pub(crate) unsafe fn install_cached_object_shape_version( + obj: *mut crate::object::ObjectHeader, + expected_predecessor_shape_id: u32, + target_shape_id: u32, + _target_keys: *mut ArrayHeader, + _target_key_count: u32, +) -> bool { + install_cached_object_shape_version_impl( + obj, + expected_predecessor_shape_id, + target_shape_id, + _target_keys, + _target_key_count, + false, + ) +} + +/// Install a historical shape held by an optimization cache that permanently +/// owns the target descriptor and roots its keys array. +/// +/// Unlike the general cached-shape entry, this does not need to probe the +/// shape table merely to note an old-generation carrier: `cache_carrier` +/// already keeps the descriptor and keys live for the lifetime of the cache, +/// which is strictly stronger than the epoch-scoped old-carrier note. The +/// Array-subclass tail cache establishes that ownership before publishing an +/// edge and never returns an unowned entry. +#[inline] +pub(crate) unsafe fn install_cache_carried_object_shape_version( + obj: *mut crate::object::ObjectHeader, + expected_predecessor_shape_id: u32, + target_shape_id: u32, + _target_keys: *mut ArrayHeader, + _target_key_count: u32, +) -> bool { + install_cached_object_shape_version_impl( + obj, + expected_predecessor_shape_id, + target_shape_id, + _target_keys, + _target_key_count, + true, + ) +} + +#[inline] +unsafe fn install_cached_object_shape_version_impl( + obj: *mut crate::object::ObjectHeader, + expected_predecessor_shape_id: u32, + target_shape_id: u32, + _target_keys: *mut ArrayHeader, + _target_key_count: u32, + target_is_cache_carried: bool, ) -> bool { if obj.is_null() || !shape_word_is_writable(obj) @@ -688,13 +852,10 @@ pub(crate) unsafe fn install_cached_object_shape_transition( // and the cache's rooted target edge keeps its descriptor live. #[cfg(debug_assertions)] { - let key_count = if _target_keys.is_null() { - 0 - } else { - crate::array::keys_array_len_capped_to_capacity(_target_keys) as u32 - }; if !shape_descriptor_by_id(target_shape_id).is_some_and(|descriptor| { - descriptor.keys == _target_keys as u64 && descriptor.logical_key_count == key_count + descriptor.keys == _target_keys as u64 + && descriptor.logical_key_count == _target_key_count + && (!target_is_cache_carried || descriptor.cache_carrier) }) { return false; } @@ -704,12 +865,16 @@ pub(crate) unsafe fn install_cached_object_shape_transition( // invalidated while the predecessor stamp is still authoritative. super::mark_object_dynamic_shape_unknown(obj); (*obj).parent_class_id = target_shape_id; - if !crate::arena::pointer_in_nursery(obj as usize) { + if !target_is_cache_carried && !crate::arena::pointer_in_nursery(obj as usize) { note_old_generation_carrier(shape_descriptor_by_id(target_shape_id)); } #[cfg(debug_assertions)] - debug_assert_object_shape_parity_for_keys(obj, _target_keys); + if !_target_keys.is_null() + && crate::array::keys_array_len_capped_to_capacity(_target_keys) as u32 == _target_key_count + { + debug_assert_object_shape_parity_for_keys(obj, _target_keys); + } #[cfg(test)] TEST_CACHED_TRANSITION_WATCH.with(|watch| { if watch.get() == obj as usize { @@ -751,6 +916,7 @@ pub(crate) unsafe fn stamp_object_shape( return 0; } let Some(lineage) = object_shape_descriptor(obj) else { + crate::array::clear_array_subclass_named_prefix_token(obj); let id = shape_descriptor_ensure(keys, key_count, live_inline_slot_count) .unwrap_or_else(|error| shape_descriptor_error_abort(error)); (*obj).parent_class_id = id; @@ -764,6 +930,13 @@ pub(crate) unsafe fn stamp_object_shape( lineage.semantic_generation, lineage.object_kind, )); + if id != (*obj).parent_class_id { + // Read-side lookup also calls `stamp_object_shape` to populate its + // field cache. Preserve a proved Array-subclass prefix when that call + // merely republishes the exact current descriptor; retire it only for + // an actual structural identity change. + crate::array::clear_array_subclass_named_prefix_token(obj); + } (*obj).parent_class_id = id; debug_assert_object_shape_parity(obj); id @@ -921,6 +1094,10 @@ pub(crate) unsafe fn publish_object_shape_from( if obj.is_null() || !shape_word_is_writable(obj) { return 0; } + // Generic structural publication may add/delete/reorder a named field. + // The learned exact numeric-tail installer has its own entry point and + // intentionally preserves this Array-subclass family proof. + crate::array::clear_array_subclass_named_prefix_token(obj); let key_count = if keys.is_null() { 0 } else { @@ -991,6 +1168,7 @@ pub(crate) unsafe fn transition_object_shape_semantics( if obj.is_null() || !shape_word_is_writable(obj) { return 0; } + crate::array::clear_array_subclass_named_prefix_token(obj); let current = object_shape_descriptor(obj).unwrap_or_else(|| { synchronize_object_shape_descriptor(obj); object_shape_descriptor(obj).expect("shape synchronization must publish a descriptor") @@ -1022,6 +1200,7 @@ pub(crate) unsafe fn transition_object_shape_to_class( if obj.is_null() || !shape_word_is_writable(obj) { return 0; } + crate::array::clear_array_subclass_named_prefix_token(obj); let current = object_shape_descriptor(obj).unwrap_or_else(|| { synchronize_object_shape_descriptor(obj); object_shape_descriptor(obj).expect("shape synchronization must publish a descriptor") @@ -1435,7 +1614,7 @@ pub(crate) fn scan_shape_table_rekey_mut(visitor: &mut crate::gc::RuntimeRootVis // rooting them from the table would make every keys array ever minted // immortal and turn `prune_dead_shape_keys`'s "is the keys array // dead?" into a question it asks of itself. - let moved = if descriptor.old_carrier { + let moved = if descriptor.old_carrier || descriptor.cache_carrier { visitor.visit_usize_slot(&mut addr) } else { visitor.visit_metadata_usize_slot(&mut addr) @@ -1595,6 +1774,8 @@ pub(crate) fn test_clear_shape_table() { inner.descriptors.clear(); inner.ids_by_facts.clear(); inner.ids_by_keys.clear(); + drop(inner); + clear_shape_object_kind_cache(); } #[cfg(test)] diff --git a/crates/perry-runtime/src/object/shapes_tests.rs b/crates/perry-runtime/src/object/shapes_tests.rs index ab79895049..7b67539990 100644 --- a/crates/perry-runtime/src/object/shapes_tests.rs +++ b/crates/perry-runtime/src/object/shapes_tests.rs @@ -430,6 +430,27 @@ mod descriptor_tests_8067 { test_drop_shape_descriptors(fake_keys); } + #[test] + fn object_kind_direct_cache_is_agent_local_and_retires_with_descriptor() { + let _lock = crate::gc::global_side_table_test_lock(); + let keys = 0x8067_0000_0000_1400usize; + let id = shape_descriptor_ensure(keys as *const ArrayHeader, 2, 2) + .expect("shape range unexpectedly exhausted"); + assert_eq!(shape_object_kind_by_id(id), Some(ShapeObjectKind::Ordinary)); + assert_eq!( + shape_object_kind_by_id(id), + Some(ShapeObjectKind::Ordinary), + "the direct-cache hit must preserve the immutable descriptor fact" + ); + + test_drop_shape_descriptors(keys); + assert_eq!( + shape_object_kind_by_id(id), + None, + "retiring the authoritative descriptor must retire its direct-cache entry" + ); + } + #[test] fn process_global_module_shape_id_installs_with_agent_local_keys() { let _lock = crate::gc::global_side_table_test_lock(); diff --git a/crates/perry-runtime/src/object/tests.rs b/crates/perry-runtime/src/object/tests.rs index 2f2e90a351..84a168911d 100644 --- a/crates/perry-runtime/src/object/tests.rs +++ b/crates/perry-runtime/src/object/tests.rs @@ -947,7 +947,7 @@ fn transition_cache_lookup_rejects_mutated_edge_target() { let keys = crate::array::js_array_push(keys, JSValue::string_ptr(key)); let keys = crate::array::js_array_push(keys, JSValue::string_ptr(key)); - transition_cache_insert(0, key, keys as usize, 0, 0); + transition_cache_insert(std::ptr::null(), 0, key, keys as usize, 0, 0); assert!( transition_cache_lookup(0, key).is_none(), @@ -977,7 +977,7 @@ fn transition_cache_requires_exact_predecessor_shape_id() { const OTHER_PREDECESSOR: u32 = 102; const TARGET: u32 = 201; - transition_cache_insert(PREDECESSOR, key, keys as usize, 0, TARGET); + transition_cache_insert(std::ptr::null(), PREDECESSOR, key, keys as usize, 0, TARGET); assert!( transition_cache_lookup(OTHER_PREDECESSOR, key).is_none(), "equal keys edges with different semantic ShapeIds must not alias" @@ -1008,7 +1008,14 @@ fn transition_cache_prunes_a_descriptorless_target_shape() { let target = crate::object::shapes::shape_descriptor_ensure(next_keys, 0, 0) .expect("shape range unexpectedly exhausted"); let occupancy_before = test_transition_cache_occupancy(); - transition_cache_insert(predecessor, std::ptr::null(), next_keys as usize, 0, target); + transition_cache_insert( + std::ptr::null(), + predecessor, + std::ptr::null(), + next_keys as usize, + 0, + target, + ); assert_eq!(test_transition_cache_occupancy(), occupancy_before + 1); crate::object::shapes::test_drop_shape_descriptors(next_keys as usize); @@ -1036,7 +1043,7 @@ fn transition_cache_lookup_rejects_slot_key_mismatch() { // Insert an edge keyed on (prev=0, `alpha`) but targeting the `beta` shape, // mirroring a recycled-address false match (target_len is set because the // length matches slot_idx+1, so only the content check can catch it). - transition_cache_insert(0, want, keys as usize, 0, 0); + transition_cache_insert(std::ptr::null(), 0, want, keys as usize, 0, 0); assert!( transition_cache_lookup(0, want).is_none(), @@ -1046,7 +1053,7 @@ fn transition_cache_lookup_rejects_slot_key_mismatch() { // Sanity: an edge whose target slot DOES hold the key still hits. let good_keys = crate::array::js_array_alloc(4); let good_keys = crate::array::js_array_push(good_keys, JSValue::string_ptr(want)); - transition_cache_insert(0, want, good_keys as usize, 0, 0); + transition_cache_insert(std::ptr::null(), 0, want, good_keys as usize, 0, 0); assert!( transition_cache_lookup(0, want).is_some(), "a genuine edge (target slot holds the key) must still hit (#6006)" @@ -1080,7 +1087,7 @@ fn transition_cache_lookup_rejects_grown_shared_target() { // A 1-key target with spare capacity, cached as a slot-0 edge (target_len=1). let keys = crate::array::js_array_alloc(4); let keys = crate::array::js_array_push(keys, JSValue::string_ptr(key)); - transition_cache_insert(0, key, keys as usize, 0, 0); + transition_cache_insert(std::ptr::null(), 0, key, keys as usize, 0, 0); assert!( transition_cache_lookup(0, key).is_some(), "sanity: a genuine 1-key edge hits before the target grows (#6006)" diff --git a/crates/perry-runtime/src/symbol.rs b/crates/perry-runtime/src/symbol.rs index 8edf6e0d4d..d00c864e7f 100644 --- a/crates/perry-runtime/src/symbol.rs +++ b/crates/perry-runtime/src/symbol.rs @@ -55,8 +55,11 @@ pub use properties::{ }; // Symbol-keyed property reads. -pub use get::js_object_get_symbol_property; pub(crate) use get::{has_own_symbol_property, inherited_symbol_property, own_symbol_property}; +pub use get::{ + js_object_get_symbol_property, js_object_get_symbol_property_ic_miss, + js_object_get_symbol_then_field_ic_miss, +}; // Iterator protocol, getOwnPropertySymbols, ToPrimitive. pub(crate) use iterator::class_ref_resolves_iterator; @@ -83,6 +86,7 @@ pub(crate) use gc_roots::{ use crate::string::StringHeader; use std::cell::RefCell; use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Mutex; // NaN-boxing tags (must match value.rs) @@ -91,6 +95,22 @@ const POINTER_TAG: u64 = 0x7FFD_0000_0000_0000; const STRING_TAG: u64 = 0x7FFF_0000_0000_0000; const POINTER_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; +/// Invalidates generated weak Symbol-property inline caches. +/// +/// The caches deliberately hold raw NaN-boxed bits without registering them as +/// roots: making a receiver/value reachable only through an optimization cache +/// survive collection would change WeakRef/finalization behavior. Every +/// symbol-data mutation and every completed collection advances this epoch, so +/// cached raw addresses are observed only while the heap and the corresponding +/// own data property are unchanged. +#[no_mangle] +pub static PERRY_SYMBOL_PROPERTY_IC_EPOCH: AtomicU64 = AtomicU64::new(1); + +#[inline] +pub(crate) fn symbol_property_ic_epoch_bump() { + PERRY_SYMBOL_PROPERTY_IC_EPOCH.fetch_add(1, Ordering::Release); +} + /// Magic number distinguishing SymbolHeader from other GC_TYPE_STRING objects. /// Placed at offset 0 so `js_is_symbol` can cheaply detect symbols. pub const SYMBOL_MAGIC: u32 = 0x5359_4D42; // "SYMB" @@ -824,12 +844,14 @@ pub(crate) fn store_object_symbol_property_root( entry.1 = value_bits; drop(guard); publish_symbol_side_table_root_edges(sym_key, value_bits); + symbol_property_ic_epoch_bump(); return false; } } entries.push((sym_key, value_bits)); } publish_symbol_side_table_root_edges(sym_key, value_bits); + symbol_property_ic_epoch_bump(); true } diff --git a/crates/perry-runtime/src/symbol/accessors.rs b/crates/perry-runtime/src/symbol/accessors.rs index 9a579e3e7a..27f26099b8 100644 --- a/crates/perry-runtime/src/symbol/accessors.rs +++ b/crates/perry-runtime/src/symbol/accessors.rs @@ -21,7 +21,9 @@ per_test_global! { pub(super) fn clear_symbol_accessor_property(obj_key: usize, sym_key: usize) { let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_ACCESSOR_PROPERTIES); if let Some(map) = guard.as_mut() { - map.remove(&(obj_key, sym_key)); + if map.remove(&(obj_key, sym_key)).is_some() { + crate::symbol::symbol_property_ic_epoch_bump(); + } } } @@ -31,7 +33,11 @@ pub(super) fn clear_symbol_accessor_property(obj_key: usize, sym_key: usize) { pub(super) fn clear_all_symbol_accessor_properties_for_object(obj_key: usize) { let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_ACCESSOR_PROPERTIES); if let Some(map) = guard.as_mut() { + let before = map.len(); map.retain(|(o, _), _| *o != obj_key); + if map.len() != before { + crate::symbol::symbol_property_ic_epoch_bump(); + } } } @@ -121,6 +127,7 @@ pub(crate) unsafe fn set_symbol_accessor_property( }, ); } + crate::symbol::symbol_property_ic_epoch_bump(); if get_bits != 0 { publish_symbol_side_table_root_edges(sym_key, get_bits); } diff --git a/crates/perry-runtime/src/symbol/get.rs b/crates/perry-runtime/src/symbol/get.rs index 44dd490707..43ab532c1b 100644 --- a/crates/perry-runtime/src/symbol/get.rs +++ b/crates/perry-runtime/src/symbol/get.rs @@ -88,6 +88,139 @@ pub(crate) unsafe fn own_symbol_property(obj_f64: f64, sym_f64: f64) -> Option f64 { + let obj_bits = obj_f64.to_bits(); + if !cache.is_null() && (obj_bits >> 48) == 0x7FFD { + let obj_key = (obj_bits & POINTER_MASK) as usize; + if let Some(header) = crate::value::addr_class::try_read_gc_header(obj_key) { + if header.obj_type == crate::gc::GC_TYPE_OBJECT { + let sym_key = sym_key_from_f64(sym_f64); + if sym_key != 0 + && accessors::symbol_accessor_property_by_key(obj_key, sym_key).is_none() + { + let value_bits = { + let guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); + guard.as_ref().and_then(|map| { + map.get(&obj_key).and_then(|entries| { + entries + .iter() + .find(|(entry_sym, _)| *entry_sym == sym_key) + .map(|(_, value_bits)| *value_bits) + }) + }) + }; + if let Some(value_bits) = value_bits { + let epoch = PERRY_SYMBOL_PROPERTY_IC_EPOCH + .load(std::sync::atomic::Ordering::Acquire); + // Publish identity/value first and epoch last. The + // generated hit path acquire-loads this first word. + *cache.add(1) = obj_bits; + *cache.add(2) = sym_f64.to_bits(); + *cache.add(3) = value_bits; + (&*(cache as *const std::sync::atomic::AtomicU64)) + .store(epoch, std::sync::atomic::Ordering::Release); + return f64::from_bits(value_bits); + } + } + } + } + } + js_object_get_symbol_property(obj_f64, sym_f64) +} + +/// Miss path for a generated `owner[symbol].field` composed inline cache. +/// +/// The first cache is the ordinary weak Symbol cache above. The second is the +/// ordinary named-field [`crate::object::PicCache`]. Keeping their established +/// miss handlers is important: accessors, prototypes, Proxies, primitive +/// receivers, and nullish throws all retain the canonical property semantics. +/// This helper merely publishes the Symbol cache as hit-eligible when the +/// intermediate value also proved to be an exact, descriptor-free object with +/// a cacheable inline field. The generated hit can then safely validate its +/// live ShapeId and reload the field's current slot value directly. +#[no_mangle] +pub unsafe extern "C" fn js_object_get_symbol_then_field_ic_miss( + obj_f64: f64, + sym_f64: f64, + key: *const crate::StringHeader, + feedback_site_id: u64, + symbol_cache: *mut u64, + field_cache: *mut crate::object::PicCache, +) -> f64 { + if key.is_null() { + if !symbol_cache.is_null() { + (&*(symbol_cache as *const std::sync::atomic::AtomicU64)) + .store(0, std::sync::atomic::Ordering::Release); + } + return f64::from_bits(TAG_UNDEFINED); + } + + // The pooled key is a moving StringHeader. Root it before the Symbol miss, + // whose accessor/prototype fallback may allocate before the named read. + let scope = crate::gc::RuntimeHandleScope::new(); + let key_handle = scope.root_nanbox_f64(crate::value::js_nanbox_string(key as i64)); + let intermediate = js_object_get_symbol_property_ic_miss(obj_f64, sym_f64, symbol_cache); + let intermediate_handle = scope.root_nanbox_f64(intermediate); + + // Do not mistake an entry retained from the previously cached + // intermediate object for a prime performed by this miss. + if !field_cache.is_null() { + (*field_cache)[0] = 0; + } + + let current = intermediate_handle.get_nanbox_f64(); + let key_now_bits = key_handle.get_nanbox_u64(); + let key_now = (key_now_bits & POINTER_MASK) as *const crate::StringHeader; + let (result, intermediate_after) = intermediate_handle.across_nanbox(|| { + crate::object::js_object_get_field_ic( + current.to_bits() as i64, + key_now, + feedback_site_id, + field_cache, + ) + }); + + // `cache[3]` is dereferenced by generated code only after cache[0]'s + // acquire-load succeeds. Leave that epoch published solely when both miss + // handlers proved the exact composed fast path. A collection during the + // named read already makes the epoch unequal; the checks below also cover + // uncacheable primitives, inherited/accessor fields, and null caches. + let intermediate_bits = intermediate_after.to_bits(); + let raw = (intermediate_bits & POINTER_MASK) as usize; + let is_plain_object = (intermediate_bits >> 48) == 0x7FFD + && crate::value::addr_class::try_read_gc_header(raw) + .is_some_and(|header| header.obj_type == crate::gc::GC_TYPE_OBJECT); + let field_primed = !field_cache.is_null() + && ((*field_cache)[0] as u64 & crate::object::shapes::PIC_ID_TOKEN_BIT) != 0; + let symbol_primed = !symbol_cache.is_null() + && *symbol_cache.add(3) == intermediate_bits + && (&*(symbol_cache as *const std::sync::atomic::AtomicU64)) + .load(std::sync::atomic::Ordering::Acquire) + == PERRY_SYMBOL_PROPERTY_IC_EPOCH.load(std::sync::atomic::Ordering::Acquire); + if !(is_plain_object && field_primed && symbol_primed) && !symbol_cache.is_null() { + (&*(symbol_cache as *const std::sync::atomic::AtomicU64)) + .store(0, std::sync::atomic::Ordering::Release); + } + + result +} + /// #5437: resolve a symbol-keyed read against the underlying native handle a /// request wrapper aliases via its `_req` field. Returns `None` unless the /// receiver is a heap object whose `_req` is a small handle (POINTER-tagged, @@ -1159,3 +1292,148 @@ mod handle_meta_share_tests { } } } + +#[cfg(test)] +mod own_data_ic_tests { + use super::*; + use std::sync::atomic::Ordering; + + #[test] + fn own_data_miss_primes_cache_and_mutation_invalidates_epoch() { + let _global = crate::gc::global_side_table_test_lock(); + unsafe { + let obj_ptr = crate::object::js_object_alloc(0, 0); + assert!(!obj_ptr.is_null()); + let obj = crate::value::js_nanbox_pointer(obj_ptr as i64); + let sym = super::constructors::js_symbol_new_empty(); + let first = 41.0_f64; + super::properties::js_object_set_symbol_property(obj, sym, first); + + let mut cache = [0_u64; 12]; + let got = js_object_get_symbol_property_ic_miss(obj, sym, cache.as_mut_ptr()); + assert_eq!(got.to_bits(), first.to_bits()); + assert_eq!(cache[1], obj.to_bits()); + assert_eq!(cache[2], sym.to_bits()); + assert_eq!(cache[3], first.to_bits()); + assert_eq!( + cache[0], + PERRY_SYMBOL_PROPERTY_IC_EPOCH.load(Ordering::Acquire) + ); + + let cached_epoch = cache[0]; + let second = 99.0_f64; + super::properties::js_object_set_symbol_property(obj, sym, second); + assert_ne!( + cached_epoch, + PERRY_SYMBOL_PROPERTY_IC_EPOCH.load(Ordering::Acquire), + "a Symbol data write must make the generated hit guard fail" + ); + let got = js_object_get_symbol_property_ic_miss(obj, sym, cache.as_mut_ptr()); + assert_eq!(got.to_bits(), second.to_bits()); + assert_eq!(cache[3], second.to_bits()); + } + } + + #[test] + fn non_object_receiver_never_primes_own_data_cache() { + let _global = crate::gc::global_side_table_test_lock(); + unsafe { + let sym = super::constructors::js_symbol_new_empty(); + let mut cache = [0_u64; 12]; + let got = js_object_get_symbol_property_ic_miss(7.0, sym, cache.as_mut_ptr()); + assert_eq!(got.to_bits(), TAG_UNDEFINED); + assert_eq!(cache, [0_u64; 12]); + } + } + + #[test] + fn composed_symbol_field_cache_reloads_mutated_final_slot() { + let _global = crate::gc::global_side_table_test_lock(); + unsafe { + crate::gc::gc_suppress(); + let owner_ptr = crate::object::js_object_alloc(0, 0); + let metadata_ptr = crate::object::js_object_alloc(0, 0); + assert!(!owner_ptr.is_null() && !metadata_ptr.is_null()); + let owner = crate::value::js_nanbox_pointer(owner_ptr as i64); + let metadata = crate::value::js_nanbox_pointer(metadata_ptr as i64); + let sym = super::constructors::js_symbol_new_empty(); + let key = js_string_from_bytes(b"id".as_ptr(), 2); + crate::object::js_object_set_field_by_name(metadata_ptr, key, 41.0); + super::properties::js_object_set_symbol_property(owner, sym, metadata); + + let mut symbol_cache = [0_u64; 12]; + let mut field_cache: crate::object::PicCache = [0; crate::object::PIC_CACHE_WORDS]; + let first = js_object_get_symbol_then_field_ic_miss( + owner, + sym, + key, + 0, + symbol_cache.as_mut_ptr(), + &mut field_cache, + ); + let epoch_before_named_write = symbol_cache[0]; + let slot = field_cache[1] as usize; + let token = field_cache[0] as u64; + + // A write to the final named property must not require a Symbol + // epoch bump. The generated hit reloads this slot rather than + // caching `41`, so it must immediately observe `99`. + crate::object::js_object_set_field_by_name(metadata_ptr, key, 99.0); + let direct_after = *((metadata_ptr as *const u8) + .add(std::mem::size_of::() + slot * 8) + as *const f64); + let second = js_object_get_symbol_then_field_ic_miss( + owner, + sym, + key, + 0, + symbol_cache.as_mut_ptr(), + &mut field_cache, + ); + let epoch_after_named_write = PERRY_SYMBOL_PROPERTY_IC_EPOCH.load(Ordering::Acquire); + crate::gc::gc_unsuppress(); + + assert_eq!(first.to_bits(), 41.0_f64.to_bits()); + assert_ne!(epoch_before_named_write, 0); + assert_ne!( + token & crate::object::shapes::PIC_ID_TOKEN_BIT, + 0, + "the named miss must publish an exact ShapeId token" + ); + assert_eq!(epoch_before_named_write, epoch_after_named_write); + assert_eq!(direct_after.to_bits(), 99.0_f64.to_bits()); + assert_eq!(second.to_bits(), 99.0_f64.to_bits()); + } + } + + #[test] + fn composed_cache_never_publishes_a_primitive_intermediate() { + let _global = crate::gc::global_side_table_test_lock(); + unsafe { + crate::gc::gc_suppress(); + let owner_ptr = crate::object::js_object_alloc(0, 0); + let owner = crate::value::js_nanbox_pointer(owner_ptr as i64); + let sym = super::constructors::js_symbol_new_empty(); + let key = js_string_from_bytes(b"id".as_ptr(), 2); + super::properties::js_object_set_symbol_property(owner, sym, 7.0); + let mut symbol_cache = [0_u64; 12]; + let mut field_cache: crate::object::PicCache = [0; crate::object::PIC_CACHE_WORDS]; + let got = js_object_get_symbol_then_field_ic_miss( + owner, + sym, + key, + 0, + symbol_cache.as_mut_ptr(), + &mut field_cache, + ); + crate::gc::gc_unsuppress(); + + assert_eq!(got.to_bits(), TAG_UNDEFINED); + assert_eq!( + symbol_cache[0], 0, + "generated code must never dereference a primitive cached value" + ); + assert_eq!(field_cache[0], 0); + } + } +} diff --git a/crates/perry-runtime/src/symbol/properties.rs b/crates/perry-runtime/src/symbol/properties.rs index 25417651c0..95d4aa85cb 100644 --- a/crates/perry-runtime/src/symbol/properties.rs +++ b/crates/perry-runtime/src/symbol/properties.rs @@ -133,6 +133,7 @@ pub(crate) unsafe fn js_object_delete_symbol_property(obj_f64: f64, sym_f64: f64 map.remove(&(obj_key, sym_key)); } } + crate::symbol::symbol_property_ic_epoch_bump(); 1 } @@ -159,6 +160,7 @@ pub(crate) fn clear_all_symbol_properties_for_object(obj_key: usize) { map.retain(|(o, _), _| *o != obj_key); } } + crate::symbol::symbol_property_ic_epoch_bump(); } /// #6710: clear every per-handle JS-property side table for a recycled handle diff --git a/crates/perry-runtime/src/value/dynamic_object.rs b/crates/perry-runtime/src/value/dynamic_object.rs index b544879a71..3fd7c910f8 100644 --- a/crates/perry-runtime/src/value/dynamic_object.rs +++ b/crates/perry-runtime/src/value/dynamic_object.rs @@ -252,6 +252,18 @@ pub extern "C" fn js_value_length_f64(value: f64) -> f64 { /// lookup rather than a second `.length` implementation. #[no_mangle] pub extern "C" fn js_value_length_property_f64(value: f64) -> f64 { + value_length_property_with_cache(value, std::ptr::null_mut()) +} + +/// Property-semantic `.length` read whose Array-subclass arm primes the +/// generated scalar IC. All non-subclass cases deliberately share the exact +/// implementation used by `js_value_length_property_f64`. +#[no_mangle] +pub extern "C" fn js_value_length_property_ic_f64(value: f64, cache: *mut u64) -> f64 { + value_length_property_with_cache(value, cache) +} + +fn value_length_property_with_cache(value: f64, cache: *mut u64) -> f64 { let jsval = JSValue::from_bits(value.to_bits()); if jsval.is_undefined() || jsval.is_null() { crate::error::js_throw_type_error_property_access( @@ -266,7 +278,7 @@ pub extern "C" fn js_value_length_property_f64(value: f64) -> f64 { crate::builtins::boxed_primitive_to_string_tag(value), Some("String") ) { - return js_value_length_property_f64(payload); + return value_length_property_with_cache(payload, cache); } } @@ -275,7 +287,7 @@ pub extern "C" fn js_value_length_property_f64(value: f64) -> f64 { return crate::string::js_string_length(string) as f64; } - if let Some(length) = crate::array::array_subclass_fast_length(value) { + if let Some(length) = crate::array::array_subclass_fast_length_with_ic(value, cache) { return length; } @@ -858,11 +870,13 @@ mod length_handle_band_tests { #[test] fn property_length_preserves_missing_and_non_numeric_values() { + let mut cache = [0_u64; 3]; assert_eq!( - js_value_length_property_f64(42.0).to_bits(), + js_value_length_property_ic_f64(42.0, cache.as_mut_ptr()).to_bits(), crate::value::TAG_UNDEFINED, "a number has no length property" ); + assert_eq!(cache, [0; 3], "a primitive must not prime the object IC"); let obj = crate::object::js_object_alloc(0, 1); let length_key = crate::string::js_string_from_bytes(b"length".as_ptr(), 6); @@ -872,10 +886,14 @@ mod length_handle_band_tests { let boxed_obj = crate::value::js_nanbox_pointer(obj as i64); assert_eq!( - js_value_length_property_f64(boxed_obj).to_bits(), + js_value_length_property_ic_f64(boxed_obj, cache.as_mut_ptr()).to_bits(), seven_value.to_bits(), "a source-level property read must not coerce its value" ); + assert_eq!( + cache, [0; 3], + "an unrelated ordinary object must retain the generic property path" + ); } /// #7930: `TypedArrayHeader::length` shares payload offset zero with an diff --git a/crates/perry-runtime/src/value/mod.rs b/crates/perry-runtime/src/value/mod.rs index bb14298f70..74115f4180 100644 --- a/crates/perry-runtime/src/value/mod.rs +++ b/crates/perry-runtime/src/value/mod.rs @@ -145,4 +145,5 @@ pub use dynamic_array::{ pub use dynamic_object::{ js_collection_method_dispatch, js_dynamic_object_get_property, js_dynamic_object_keys, js_get_property, js_value_length_f64, js_value_length_property_f64, + js_value_length_property_ic_f64, }; From 12d4cbaa4712f6a91e7280d6950b8d7bf356b048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 10:52:08 +0200 Subject: [PATCH 04/15] perf(codegen): follow one growth-forwarding edge in the guarded array store `this.vals[i] = v` has no writeback slot: once the array grows past its initial capacity the object field keeps the pre-grow forwarding stub, and the guarded property-receiver STORE tier rejected the stub on every later store (`!GC_FLAG_FORWARDED`), sending the whole store out of line through the extend helper and the allocator/registry resolver. The READ tier already followed one edge inline; mirror it: `deref` selects the stub's forwarding word (heap-band checked), a new `deref.live` block re-validates the destination header, and the fast arm stores into the live head. wolf-ecs (Mac mini, 11 pairs): add/remove -9.15% (11/11), entity-cycle -13.67% (11/11). Test: index_set_barrier_tests::the_guarded_property_receiver_store_follows_one_forwarding_edge_inline Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- .../src/expr/index_set_barrier_tests.rs | 64 +++++++++++++++++++ .../src/expr/index_set_guarded.rs | 61 ++++++++++++++++-- 2 files changed, 118 insertions(+), 7 deletions(-) diff --git a/crates/perry-codegen/src/expr/index_set_barrier_tests.rs b/crates/perry-codegen/src/expr/index_set_barrier_tests.rs index 920d96e0e1..1dba4079df 100644 --- a/crates/perry-codegen/src/expr/index_set_barrier_tests.rs +++ b/crates/perry-codegen/src/expr/index_set_barrier_tests.rs @@ -410,3 +410,67 @@ fn runtime_array_setter_is_not_followed_by_a_duplicate_opaque_barrier() { inline store's precise slot barrier:\n{barrier_body}" ); } + +/// The guarded property-receiver store follows ONE growth-forwarding edge +/// inline, exactly like the guarded read tier, instead of failing its +/// `!GC_FLAG_FORWARDED` test and going out of line on every store. +/// +/// `this.vals[i] = v` has no writeback slot: once `vals` has grown past its +/// initial capacity the object field keeps the pre-grow stub forever, so the +/// old guard rejected the receiver on EVERY later store and the whole store +/// paid the extend helper plus the allocator/registry forwarding resolver +/// (the wolf-ecs `_ent`/`_updateTo`/`sparse` hot path). The heal is pinned +/// three ways: the `deref` block selects the forwarding target, the selected +/// live head is re-validated in `deref.live`, and the fast arm's element +/// address is derived from that live head rather than from the original box. +#[test] +fn the_guarded_property_receiver_store_follows_one_forwarding_edge_inline() { + let ir = ir(); + let deref = + block_body(&ir, "idxset.recv_prop.deref.").expect("guarded store emits its `deref` block"); + let live = block_body(&ir, "idxset.recv_prop.deref.live.") + .expect("guarded store emits its `deref.live` block"); + let fast = + block_body(&ir, "idxset.recv_prop.fast.").expect("guarded store emits its `fast` block"); + + // (1) `deref` reads the stub's first payload word and selects it as the + // live handle when the header says ARRAY + FORWARDED. + let select_line = deref + .lines() + .map(str::trim) + .find(|line| line.contains("select i1") && line.contains("i64")) + .expect("`deref` selects between the forwarding target and the receiver"); + let live_handle = select_line + .split(" = ") + .next() + .expect("select defines a register") + .to_string(); + let target = operand(select_line, 2).expect("select's taken operand"); + let target_def = def_of(&deref, &target).expect("forwarding target is defined in `deref`"); + assert!( + target_def.contains("load i64"), + "the forwarding target must be the stub's first payload word, got `{target_def}`" + ); + assert!( + deref.contains("br i1") && deref.contains("idxset.recv_prop.deref.live."), + "`deref` must branch into `deref.live` after the heap-band test of the live handle" + ); + + // (2) `deref.live` re-reads the ARRAY brand and the FORWARDED bit from the + // LIVE handle (not from the original box) before admitting the fast arm. + assert!( + live.contains(&format!("sub i64 {live_handle}, 8")) + && live.contains(&format!("sub i64 {live_handle}, 7")), + "`deref.live` must re-validate the header of the selected live head" + ); + assert!( + live.contains("idxset.recv_prop.fast."), + "`deref.live` is the fast arm's predecessor" + ); + + // (3) The fast arm's element address is computed from the live head. + assert!( + fast.contains(&format!("add i64 {live_handle}, ")), + "the fast arm must address the element relative to the live head, not the stub" + ); +} diff --git a/crates/perry-codegen/src/expr/index_set_guarded.rs b/crates/perry-codegen/src/expr/index_set_guarded.rs index fc96ecfa7a..977332084e 100644 --- a/crates/perry-codegen/src/expr/index_set_guarded.rs +++ b/crates/perry-codegen/src/expr/index_set_guarded.rs @@ -94,7 +94,9 @@ pub(super) fn emit_guarded_inbounds_array_store( } ctx.current_block = deref_idx; - { + let live_deref_idx = ctx.new_block(&format!("{}.deref.live", block_prefix)); + let live_deref_label = ctx.block_label(live_deref_idx); + let live_handle = { let blk = ctx.block(); let arr_bits = blk.bitcast_double_to_i64(arr_box); let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64); @@ -104,6 +106,50 @@ pub(super) fn emit_guarded_inbounds_array_store( let gc_type = blk.load(I8, &gc_type_ptr); let is_array = blk.icmp_eq(I8, &gc_type, "1"); // GC_TYPE_ARRAY + let gc_flags_addr = blk.sub(I64, &arr_handle, "7"); + let gc_flags_ptr = blk.inttoptr(I64, &gc_flags_addr); + let gc_flags = blk.load(I8, &gc_flags_ptr); + let forwarded_bits = blk.and(I8, &gc_flags, "128"); + let is_forwarded = blk.icmp_ne(I8, &forwarded_bits, "0"); + + // Array growth (and GC evacuation) leave the live user address in the + // first payload word of a forwarded array stub. Follow one edge inline + // and re-brand/re-check the destination below, exactly as the guarded + // read tier does. This matters most for receivers that are NOT stack + // locals: `this.vals[i] = v` has no writeback slot, so once the array + // has grown past its initial capacity the object field keeps the + // pre-grow stub forever. Before this the stub failed `not_forwarded` + // on EVERY later store and the whole store went out of line through + // the extend helper and the allocator/registry resolver (the ECS + // add/remove hot path: `this._ent[id] = arch`, `this.sparse[x] = n`). + // Longer or corrupt chains still take the slow arm. + let original_arr_ptr = blk.inttoptr(I64, &arr_handle); + let forwarding_target = blk.load(I64, &original_arr_ptr); + let follow_forwarding = blk.and(I1, &is_array, &is_forwarded); + let live_handle = blk.select(I1, &follow_forwarding, I64, &forwarding_target, &arr_handle); + + let live_top = blk.lshr(I64, &live_handle, "48"); + let live_top_clear = blk.icmp_eq(I64, &live_top, "0"); + let live_above_handle_band = blk.icmp_ugt(I64, &live_handle, "1048575"); + let live_below_heap_limit = blk.icmp_ult(I64, &live_handle, "140737488355328"); + let mut live_heap_candidate = blk.and(I1, &live_top_clear, &live_above_handle_band); + live_heap_candidate = blk.and(I1, &live_heap_candidate, &live_below_heap_limit); + // A forwarding word is not trusted until its address is in the heap + // band: never read the destination header speculatively. + blk.cond_br(&live_heap_candidate, &live_deref_label, &slow_label); + live_handle + }; + + ctx.current_block = live_deref_idx; + { + let blk = ctx.block(); + let arr_handle = live_handle.clone(); + + let gc_type_addr = blk.sub(I64, &arr_handle, "8"); + let gc_type_ptr = blk.inttoptr(I64, &gc_type_addr); + let gc_type = blk.load(I8, &gc_type_ptr); + let is_array = blk.icmp_eq(I8, &gc_type, "1"); // GC_TYPE_ARRAY + let gc_flags_addr = blk.sub(I64, &arr_handle, "7"); let gc_flags_ptr = blk.inttoptr(I64, &gc_flags_addr); let gc_flags = blk.load(I8, &gc_flags_ptr); @@ -160,8 +206,9 @@ pub(super) fn emit_guarded_inbounds_array_store( // invariant, which is why that half of #7511's argument does not transfer. let (arr_handle, element_addr, value_bits) = { let blk = ctx.block(); - let arr_bits = blk.bitcast_double_to_i64(arr_box); - let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64); + // The live (possibly forwarded-once) head proved by `deref.live`, + // which is this block's only predecessor. + let arr_handle = live_handle.clone(); let idx_i64 = blk.zext(I32, idx_i32, I64); let byte_offset = blk.shl(I64, &idx_i64, "3"); let with_header = blk.add(I64, &byte_offset, "8"); @@ -186,10 +233,10 @@ pub(super) fn emit_guarded_inbounds_array_store( (arr_handle, element_addr, value_bits) }; if write_barrier_needed { - // `arr_handle` reached this block through the guard's own - // `obj_type == GC_TYPE_ARRAY` / `!GC_FLAG_FORWARDED` header reads, so - // it is a live, non-forwarded GC array user pointer — the precondition - // for reading its header byte. (LLVM CSEs that byte load with the + // `arr_handle` is the live head `deref.live` just proved through its + // own `obj_type == GC_TYPE_ARRAY` / `!GC_FLAG_FORWARDED` header reads, + // so it is a live, non-forwarded GC array user pointer — the + // precondition for reading its header byte. (LLVM CSEs that byte load with the // guard's, so the gate costs the test and the branch, not a reload.) emit_write_barrier_slot_value_and_generation_tested( ctx, From 6de5299edd83ce5a4bfd51296f2c757b2253cd69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 10:52:29 +0200 Subject: [PATCH 05/15] perf(array): gate the raw-f64 downgrade note inline; typed-array pre-dispatch - index_set_guarded.rs: the fast arm only calls js_array_note_numeric_write when the live head's `_reserved` word (already loaded by `deref.live`) has a raw-f64 bit set; the note is exactly "clear those bits if the value is not a Number" and was re-resolving the receiver through the tracked resolver on every pointer store. - header.rs: js_array_note_numeric_write returns early for Number values and for already-clear live headers before paying clean_arr_ptr. - indexing.rs: js_array_get_f64 dispatches a GC_TYPE_TYPED_ARRAY-tagged, registered receiver to js_typed_array_get before clean_arr_ptr (a guaranteed tracked miss for a typed array). wolf-ecs (Mac mini, 11 pairs): add/remove -4.86% (11/11), entity-cycle -5.50% (11/11). Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- .../src/expr/index_set_barrier_tests.rs | 50 +++++++++++++++++++ .../src/expr/index_set_guarded.rs | 42 +++++++++++++--- crates/perry-runtime/src/array/header.rs | 20 ++++++++ crates/perry-runtime/src/array/indexing.rs | 15 ++++++ .../typedarray/element_read_receiver_tests.rs | 19 +++++++ 5 files changed, 138 insertions(+), 8 deletions(-) diff --git a/crates/perry-codegen/src/expr/index_set_barrier_tests.rs b/crates/perry-codegen/src/expr/index_set_barrier_tests.rs index 1dba4079df..80802025d8 100644 --- a/crates/perry-codegen/src/expr/index_set_barrier_tests.rs +++ b/crates/perry-codegen/src/expr/index_set_barrier_tests.rs @@ -474,3 +474,53 @@ fn the_guarded_property_receiver_store_follows_one_forwarding_edge_inline() { "the fast arm must address the element relative to the live head, not the stub" ); } + +/// The fast arm's raw-f64 downgrade note is gated on the live head's +/// `_reserved` word rather than called unconditionally: `js_array_note_numeric_write` +/// is exactly "clear the raw-f64 bits if the value is not a Number", and it +/// re-resolves the receiver through the tracked resolver on every call, so a +/// pointer store into an array whose raw-f64 bits are already clear must not +/// reach it at all. +#[test] +fn the_fast_arm_numeric_note_is_gated_on_the_raw_f64_header_bits() { + let ir = ir(); + let live = block_body(&ir, "idxset.recv_prop.deref.live.") + .expect("guarded store emits its `deref.live` block"); + let reserved_line = live + .lines() + .map(str::trim) + .find(|line| line.contains("load i16")) + .expect("`deref.live` loads the live head's `_reserved` word"); + let reserved = reserved_line + .split(" = ") + .next() + .expect("load defines a register") + .to_string(); + + let (gate, gate_body) = branch_into_block(&ir, "idxset.recv_prop.numnote.") + .expect("the numeric note sits behind a conditional branch"); + let cond = operand(&gate, 0).expect("cond_br has a condition"); + let cond_def = def_of(&gate_body, &cond).expect("gate condition is defined in its block"); + assert!( + cond_def.contains("icmp ne i16"), + "gate must test the raw-f64 bits for non-zero, got `{cond_def}`" + ); + let masked = operand(cond_def, 0).expect("icmp operand"); + let masked_def = def_of(&gate_body, &masked).expect("masked bits are defined in the block"); + assert!( + masked_def.contains(&format!("and i16 {reserved}, 4224")), + "gate must mask GC_ARRAY_RAW_F64_LAYOUT|GC_ARRAY_RAW_F64_HOLES (0x1080) out of the \\ + live head's `_reserved`, got `{masked_def}`" + ); + + let note = block_body(&ir, "idxset.recv_prop.numnote.").expect("the numeric note block exists"); + assert!( + note.contains("call void @js_array_note_numeric_write("), + "the note call must live inside the gated block:\n{note}" + ); + let fast = block_body(&ir, "idxset.recv_prop.fast.").expect("fast block"); + assert!( + !fast.contains("js_array_note_numeric_write"), + "the fast arm must not call the note unconditionally:\n{fast}" + ); +} diff --git a/crates/perry-codegen/src/expr/index_set_guarded.rs b/crates/perry-codegen/src/expr/index_set_guarded.rs index 977332084e..ee43838bf6 100644 --- a/crates/perry-codegen/src/expr/index_set_guarded.rs +++ b/crates/perry-codegen/src/expr/index_set_guarded.rs @@ -141,7 +141,7 @@ pub(super) fn emit_guarded_inbounds_array_store( }; ctx.current_block = live_deref_idx; - { + let reserved = { let blk = ctx.block(); let arr_handle = live_handle.clone(); @@ -188,7 +188,10 @@ pub(super) fn emit_guarded_inbounds_array_store( guard_ok = blk.and(I1, &guard_ok, &capacity_sane); guard_ok = blk.and(I1, &guard_ok, &length_within_capacity); blk.cond_br(&guard_ok, &fast_label, &slow_label); - } + // The live head's `_reserved` word dominates the fast arm; the numeric + // write note below is gated on it. + reserved + }; ctx.current_block = fast_idx; // #7715 B3: the barrier is emitted separately, behind an inline live test @@ -247,15 +250,38 @@ pub(super) fn emit_guarded_inbounds_array_store( block_prefix, ); } - { - let blk = ctx.block(); - if !value_is_numeric { - // A non-numeric store into a raw-f64-flagged array downgrades the - // layout. Identical to the local-receiver arm. + if !value_is_numeric { + // A non-numeric store into a raw-f64-flagged array downgrades the + // layout. The runtime note is exactly "clear GC_ARRAY_RAW_F64_LAYOUT | + // GC_ARRAY_RAW_F64_HOLES if the value is not a Number", so an array + // whose header already has both bits clear has nothing to note. Test + // the `_reserved` word `deref.live` just loaded and skip the call — + // which re-resolved the receiver through the allocator/registry + // resolver on EVERY pointer store (the ECS archetype moves) — unless a + // raw-f64 bit is actually set. Nothing between that load and here can + // SET a raw-f64 bit: the slot store, string addref, layout note and + // write barrier only ever clear them; the bits are set solely by the + // explicit verify/rebuild paths. + let note_idx = ctx.new_block(&format!("{}.numnote", block_prefix)); + let note_done_idx = ctx.new_block(&format!("{}.numnote.done", block_prefix)); + let note_label = ctx.block_label(note_idx); + let note_done_label = ctx.block_label(note_done_idx); + { + let blk = ctx.block(); + // GC_ARRAY_RAW_F64_LAYOUT (0x80) | GC_ARRAY_RAW_F64_HOLES (0x1000). + let raw_bits = blk.and(I16, &reserved, "4224"); + let has_raw_layout = blk.icmp_ne(I16, &raw_bits, "0"); + blk.cond_br(&has_raw_layout, ¬e_label, ¬e_done_label); + } + ctx.current_block = note_idx; + { + let blk = ctx.block(); emit_array_numeric_write_note_on_block(blk, &arr_handle, &value_bits); + blk.br(¬e_done_label); } - blk.br(&merge_label); + ctx.current_block = note_done_idx; } + ctx.block().br(&merge_label); ctx.current_block = slow_idx; fallback(ctx)?; diff --git a/crates/perry-runtime/src/array/header.rs b/crates/perry-runtime/src/array/header.rs index d83a7aa059..06f6f2fb4c 100644 --- a/crates/perry-runtime/src/array/header.rs +++ b/crates/perry-runtime/src/array/header.rs @@ -1651,6 +1651,26 @@ pub extern "C" fn js_array_clear_numeric_layout(arr: *mut ArrayHeader) { #[no_mangle] pub extern "C" fn js_array_note_numeric_write(arr: *mut ArrayHeader, value_bits: u64) { + // A Number never downgrades the raw-f64 layout: nothing to clear, so do + // not pay the receiver resolver for it. + if value_bits_are_numeric(value_bits) { + return; + } + // Exact, non-forwarded ordinary Array whose raw-f64 bits are already + // clear: the note is a no-op. Answer from the magnitude-checked live + // header probe (the same discipline as the generated guards) instead of + // the tracked-allocation resolver. Forwarding stubs and every other brand + // keep the complete resolver below. + let raw_bits = crate::gc::GC_ARRAY_RAW_F64_LAYOUT | crate::gc::GC_ARRAY_RAW_F64_HOLES; + let already_clear = unsafe { crate::value::addr_class::try_read_gc_header(arr as usize) } + .is_some_and(|header| { + header.obj_type == crate::gc::GC_TYPE_ARRAY + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + && header._reserved & raw_bits == 0 + }); + if already_clear { + return; + } let arr = clean_arr_ptr_mut(arr); unsafe { note_array_numeric_write(arr, value_bits); diff --git a/crates/perry-runtime/src/array/indexing.rs b/crates/perry-runtime/src/array/indexing.rs index f42f18a3a3..69f6610c5a 100644 --- a/crates/perry-runtime/src/array/indexing.rs +++ b/crates/perry-runtime/src/array/indexing.rs @@ -858,6 +858,21 @@ pub extern "C" fn js_array_get_f64(arr: *const ArrayHeader, index: u32) -> f64 { } } + // A %TypedArray% receiver reaching the generic element read (an untyped + // `mask[i]` on a `Uint32Array` field) used to pay `clean_arr_ptr`'s + // tracked-allocation resolver — a guaranteed miss for a typed array — + // before the registry probe below could route it. The managed header tag + // already read above selects the typed authority first; the registry + // remains the liveness/layout proof, exactly as for Map/Set. + if receiver_tag.0 == crate::gc::GC_TYPE_TYPED_ARRAY + && crate::typedarray::lookup_typed_array_kind(raw_ptr as usize).is_some() + { + return crate::typedarray::js_typed_array_get( + raw_ptr as *const crate::typedarray::TypedArrayHeader, + index as i32, + ); + } + let cleaned = clean_arr_ptr(arr); if cleaned.is_null() { // #7574: `a[i]` on a `class X extends Array` instance held in a diff --git a/crates/perry-runtime/src/typedarray/element_read_receiver_tests.rs b/crates/perry-runtime/src/typedarray/element_read_receiver_tests.rs index c895d50d5f..7767181c48 100644 --- a/crates/perry-runtime/src/typedarray/element_read_receiver_tests.rs +++ b/crates/perry-runtime/src/typedarray/element_read_receiver_tests.rs @@ -509,3 +509,22 @@ fn js_typed_array_index_get_dynamic_still_reads_a_uint8array_buffer_owner() { assert_eq!(js_typed_array_index_get_dynamic(recv, 2.0), 7.0); assert!(is_undefined(js_typed_array_index_get_dynamic(recv, 3.0))); } + +/// The generic Array element read routes a registered %TypedArray% receiver +/// off its managed header tag BEFORE the tracked-allocation resolver (which +/// can only miss for a typed array), and answers exactly what the typed read +/// answers: the lane value in range, `undefined` out of range. +#[test] +fn generic_array_element_read_dispatches_a_typed_array_off_its_header_tag() { + let ta = typed(crate::typedarray::KIND_UINT32, &[7.0, 9.0, 4_000_000_000.0]); + let as_array = ta as *const crate::array::ArrayHeader; + assert_eq!(crate::array::js_array_get_f64(as_array, 0), 7.0); + assert_eq!(crate::array::js_array_get_f64(as_array, 1), 9.0); + assert_eq!(crate::array::js_array_get_f64(as_array, 2), 4_000_000_000.0); + assert!(is_undefined(crate::array::js_array_get_f64(as_array, 3))); + // The NaN-boxed form generated code hands the helper must agree. + let boxed = (crate::value::POINTER_TAG | (ta as u64 & POINTER_MASK)) + as *const crate::array::ArrayHeader; + assert_eq!(crate::array::js_array_get_f64(boxed, 1), 9.0); + assert!(is_undefined(crate::array::js_array_get_f64(boxed, 3))); +} From 45be16c16546c0b2851c86923efabd6d099f6192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 10:52:41 +0200 Subject: [PATCH 06/15] perf(codegen): exact inline typeof-number compare; header-branded typed-array reads - compare.rs: `typeof local === "number"` / `!==` decides the definitely-Number cases inline (top 16 bits outside 0x7FF9..=0x7FFF, not the untagged raw typed-array pointer shape, outside the Web Streams id band) and keeps js_value_typeof_tag on the slow arm, so the two routes can never disagree. A 33-kind differential probe matches Node byte-for-byte. - index_get/inline_dyn_typed_array.rs: the inline dynamic typed-array read brands the receiver off its GC_TYPE_TYPED_ARRAY header and reads the element kind from the TypedArrayHeader instead of probing the 64-slot direct-mapped PERRY_TA_KIND_CACHE, which every ordinary-array registry miss also writes negative entries into (hot typed arrays kept being evicted and missed the tier). PERRY_TA_VIEW_GUARD still gates the whole tier. wolf-ecs (Mac mini, 11 pairs): add/remove -1.28% (11/11), entity-cycle -0.73% (11/11). Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- crates/perry-codegen/src/expr/compare.rs | 92 +++++++++++++++++-- .../perry-codegen/src/expr/compare_tests.rs | 44 +++++++++ .../expr/index_get/inline_dyn_typed_array.rs | 82 ++++++++++------- .../src/expr/index_get_claim_tests.rs | 57 ++++++++++++ 4 files changed, 235 insertions(+), 40 deletions(-) diff --git a/crates/perry-codegen/src/expr/compare.rs b/crates/perry-codegen/src/expr/compare.rs index 40826f145d..2666a0995e 100644 --- a/crates/perry-codegen/src/expr/compare.rs +++ b/crates/perry-codegen/src/expr/compare.rs @@ -823,6 +823,78 @@ fn lower_string_strict_eq_inline( ) } +/// `typeof v === "number"` (or `!==`) with the definitely-a-Number cases +/// decided inline and everything else deferred to the shared classifier. +/// +/// The classifier answers `Number` only on its final fallthrough, after every +/// NaN-box tag has been excluded and two raw-bit exceptions have been checked: +/// an untagged typed-array pointer (`top16 == 0 && bits >= 0x10000`, #654) and +/// a Web Streams handle id, which is a positive whole number inside +/// `[STREAM_ID_BAND_START, STREAM_ID_BAND_END)` (#1545/#1650). A value is +/// therefore a Number by construction when its top 16 bits are outside the +/// tag range `0x7FF9..=0x7FFF` (that covers every negative double as well), +/// it is not that raw-pointer shape, and it lies outside the stream id band. +/// INT32-tagged values (which may be class references) and everything inside +/// the two exception windows keep the complete classifier, so the two routes +/// can never disagree. The inline arm is the ECS `_validID` entity-id check. +fn lower_typeof_number_inline(ctx: &mut FnCtx<'_>, value: &str, negate: bool) -> String { + let fast_idx = ctx.new_block("typeof.num.fast"); + let slow_idx = ctx.new_block("typeof.num.slow"); + let merge_idx = ctx.new_block("typeof.num.merge"); + let fast_label = ctx.block_label(fast_idx); + let slow_label = ctx.block_label(slow_idx); + let merge_label = ctx.block_label(merge_idx); + { + let blk = ctx.block(); + let bits = blk.bitcast_double_to_i64(value); + let top16 = blk.lshr(I64, &bits, "48"); + // Outside the tag range `0x7FF9..=0x7FFF` in one unsigned compare. + let tag_offset = blk.sub(I64, &top16, "32761"); + let untagged = blk.icmp_ugt(I64, &tag_offset, "6"); + // Not an untagged raw typed-array pointer. + let top_nonzero = blk.icmp_ne(I64, &top16, "0"); + let below_pointer_floor = blk.icmp_ult(I64, &bits, "65536"); + let not_raw_pointer = blk.or(I1, &top_nonzero, &below_pointer_floor); + // Not inside the stream handle id band (a positive whole number + // there is the one Number-looking value the classifier may call an + // object). Ordered compares are false for NaN, which is a Number. + let below_band = blk.fcmp("olt", value, "1048576.0"); + let above_band = blk.fcmp("oge", value, "2097152.0"); + let outside_band = blk.or(I1, &below_band, &above_band); + let is_nan = blk.fcmp("uno", value, value); + let nan_or_outside_band = blk.or(I1, &outside_band, &is_nan); + let mut definitely_number = blk.and(I1, &untagged, ¬_raw_pointer); + definitely_number = blk.and(I1, &definitely_number, &nan_or_outside_band); + blk.cond_br(&definitely_number, &fast_label, &slow_label); + } + ctx.current_block = fast_idx; + let fast_end = { + let blk = ctx.block(); + let label = blk.label.clone(); + blk.br(&merge_label); + label + }; + ctx.current_block = slow_idx; + let (slow_bit, slow_end) = { + let tag = ctx + .block() + .call(I32, "js_value_typeof_tag", &[(crate::types::DOUBLE, value)]); + let bit = if negate { + ctx.block().icmp_ne(I32, &tag, "3") + } else { + ctx.block().icmp_eq(I32, &tag, "3") + }; + let blk = ctx.block(); + let label = blk.label.clone(); + blk.br(&merge_label); + (bit, label) + }; + ctx.current_block = merge_idx; + let fast_bit = if negate { "false" } else { "true" }; + ctx.block() + .phi(I1, &[(fast_bit, &fast_end), (&slow_bit, &slow_end)]) +} + pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { match expr { Expr::Compare { op, left, right } => { @@ -835,15 +907,19 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // classifier's integer result instead of materializing a // heap string and entering string equality. let value = lower_expr(ctx, operand)?; - let tag = ctx.block().call( - I32, - "js_value_typeof_tag", - &[(crate::types::DOUBLE, &value)], - ); - let bit = if matches!(op, CompareOp::Ne) { - ctx.block().icmp_ne(I32, &tag, &expected_tag.to_string()) + let bit = if expected_tag == 3 { + lower_typeof_number_inline(ctx, &value, matches!(op, CompareOp::Ne)) } else { - ctx.block().icmp_eq(I32, &tag, &expected_tag.to_string()) + let tag = ctx.block().call( + I32, + "js_value_typeof_tag", + &[(crate::types::DOUBLE, &value)], + ); + if matches!(op, CompareOp::Ne) { + ctx.block().icmp_ne(I32, &tag, &expected_tag.to_string()) + } else { + ctx.block().icmp_eq(I32, &tag, &expected_tag.to_string()) + } }; let tagged = ctx.block().select( I1, diff --git a/crates/perry-codegen/src/expr/compare_tests.rs b/crates/perry-codegen/src/expr/compare_tests.rs index 36d0bd5d7a..4e51625792 100644 --- a/crates/perry-codegen/src/expr/compare_tests.rs +++ b/crates/perry-codegen/src/expr/compare_tests.rs @@ -661,3 +661,47 @@ fn loose_equality_against_number_keeps_coercion() { "dynamic == number incorrectly bypassed coercion:\n{ir}" ); } + +/// `typeof x === "number"` decides the definitely-a-Number cases inline and +/// keeps the integer classifier only on the slow arm: a value whose top 16 +/// bits are outside the NaN-box tag range, that is not an untagged raw +/// typed-array pointer, and that lies outside the Web Streams id band is a +/// Number by construction. Everything else still asks `js_value_typeof_tag`, +/// so the two routes can never disagree. +#[test] +fn local_typeof_number_literal_is_decided_inline_for_plain_doubles() { + let ir = cmp_ir( + "typeof_local_eq_number_inline", + CompareOp::Eq, + Expr::TypeOf(Box::new(Expr::LocalGet(X))), + Expr::String("number".to_string()), + ); + assert!( + ir.contains("typeof.num.fast") && ir.contains("typeof.num.slow"), + "the inline Number test must fork a fast and a slow arm:\n{ir}" + ); + let slow = super::class_field_barrier_tests::block_body(&ir, "typeof.num.slow.") + .expect("slow arm exists"); + assert!( + slow.contains("call i32 @js_value_typeof_tag("), + "the classifier call must live on the slow arm:\n{slow}" + ); + let calls = ir.matches("call i32 @js_value_typeof_tag(").count(); + assert_eq!( + calls, 1, + "exactly one classifier call, on the slow arm:\n{ir}" + ); + assert!( + ir.contains("icmp ugt i64") && ir.contains(", 6\n") || ir.contains(", 6 "), + "the tag-range test must be the single unsigned range compare:\n{ir}" + ); + assert!( + ir.contains("fcmp olt double") && ir.contains("1048576.0") + || ir.contains("0x4130000000000000"), + "the stream id band must be excluded inline:\n{ir}" + ); + assert!( + !ir.contains("call i64 @js_value_typeof(") && !ir.contains("call i32 @js_string_equals("), + "no typeof string or string equality may be materialized:\n{ir}" + ); +} diff --git a/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs b/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs index 89b38a4272..7211892e12 100644 --- a/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs +++ b/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs @@ -28,9 +28,9 @@ use super::FnCtx; /// 1. receiver-is-pointer NaN-box guard, /// 2. a read of the process-global `PERRY_TA_VIEW_GUARD` (must be 0 → every /// live typed array uses inline storage, so `data_ptr == header + 16`), -/// 3. a probe of the `PERRY_TA_KIND_CACHE` slot for the receiver address -/// (matches the cached `(addr << 8) | tag` word; the tag is the element -/// kind and must be a non-BigInt kind ≤ `KIND_UINT8_CLAMPED`), +/// 3. a `GC_TYPE_TYPED_ARRAY` brand read from the receiver's managed header +/// (`obj_type == 11`) plus the element kind read from the +/// `TypedArrayHeader` (must be a non-BigInt kind ≤ `KIND_UINT8_CLAMPED`), /// 4. an index validity + bounds check against the header `length`, /// 5. a direct per-kind element load + int↔f64 widen, /// and falls back to the existing `js_dyn_index_get` slow path on ANY guard @@ -81,22 +81,46 @@ pub(super) fn lower_inline_dyn_typed_array_get( // view guard must be 0 (all typed arrays inline-storage) let vg = blk.load(I64, "@PERRY_TA_VIEW_GUARD"); let vg_zero = blk.icmp_eq(I64, &vg, "0"); - // cache slot = (raw >> 3) & 63 - let slot = blk.lshr(I64, &raw, "3"); - let slot = blk.and(I64, &slot, "63"); - let entry_ptr = blk.gep( - "[64 x i64]", - "@PERRY_TA_KIND_CACHE", - &[(I64, "0"), (I64, &slot)], - ); - let entry_val = blk.load(I64, &entry_ptr); - // addr match: (entry_val u>> 8) == raw (also rejects empty slot = 0) - let entry_addr = blk.lshr(I64, &entry_val, "8"); - let addr_match = blk.icmp_eq(I64, &entry_addr, &raw); - // kind = entry_val & 0xFF; loadable numeric kind = kind <= 8 - // (KIND_INT8=0 .. KIND_UINT8_CLAMPED=8; rejects BigInt 9/10, - // Float16 11, and the 0xFF "not a typed array" sentinel). - let kind = blk.and(I64, &entry_val, "255"); + // Heap-band magnitude before any dereference: the same floor and + // ceiling the guarded Array tiers apply (`is_plausible_heap_addr`). + let above_handle_band = blk.icmp_ugt(I64, &raw, "1048575"); + let below_heap_limit = blk.icmp_ult(I64, &raw, "140737488355328"); + let heap_candidate = blk.and(I1, &above_handle_band, &below_heap_limit); + let g0 = blk.and(I1, &is_ptr, &vg_zero); + blk.and(I1, &g0, &heap_candidate) + }; + let brand_idx = ctx.new_block("tav.get.brand"); + let brand_label = ctx.block_label(brand_idx); + ctx.block().cond_br(&entry_guard, &brand_label, &slow_label); + + // ---- brand: managed-header tag + header kind -> fast | slow ---- + // + // Every typed array carries a real `GC_TYPE_TYPED_ARRAY` GcHeader (the + // 2026-07-09 audit) whose payload starts with `TypedArrayHeader` + // {length u32, capacity u32, kind u8, ...}. Reading the brand and the kind + // from the object itself replaces the 64-slot direct-mapped + // `PERRY_TA_KIND_CACHE` probe, which every ordinary-array registry miss + // also writes NEGATIVE entries into: a hot typed array whose slot kept + // being evicted (the wolf-ecs archetype `mask` reads) missed this tier on + // every access and paid the complete dynamic read. The header tag is + // ABA-proof for a value held by live code: the arena rewrites `obj_type` + // before it hands the address out again, and a live reference keeps the + // typed array alive. + ctx.current_block = brand_idx; + let entry_guard = { + let blk = ctx.block(); + let obj_bits = blk.bitcast_double_to_i64(obj_box); + let raw = blk.and(I64, &obj_bits, pointer_mask); + let gc_type_addr = blk.sub(I64, &raw, "8"); + let gc_type_ptr = blk.inttoptr(I64, &gc_type_addr); + let gc_type = blk.load(I8, &gc_type_ptr); + let is_typed_array = blk.icmp_eq(I8, &gc_type, "11"); // GC_TYPE_TYPED_ARRAY + let kind_addr = blk.add(I64, &raw, "8"); + let kind_ptr = blk.inttoptr(I64, &kind_addr); + let kind_i8 = blk.load(I8, &kind_ptr); + let kind = blk.zext(I8, &kind_i8, I64); + // loadable numeric kind = kind <= 8 (KIND_INT8=0 .. KIND_UINT8_CLAMPED=8; + // rejects BigInt 9/10 and Float16 11). let kind_ok = blk.icmp_ule(I64, &kind, "8"); // index float-range pre-checks (well-defined on NaN → false): the // fptosi in the load block is only reached when these hold, so its @@ -104,9 +128,7 @@ pub(super) fn lower_inline_dyn_typed_array_get( let idx_ge0 = blk.fcmp("oge", idx_d, "0.0"); let idx_lt = blk.fcmp("olt", idx_d, "4294967296.0"); // AND-reduce all guards. - let g = blk.and(I1, &is_ptr, &vg_zero); - let g = blk.and(I1, &g, &addr_match); - let g = blk.and(I1, &g, &kind_ok); + let g = blk.and(I1, &is_typed_array, &kind_ok); let g = blk.and(I1, &g, &idx_ge0); blk.and(I1, &g, &idx_lt) }; @@ -118,16 +140,12 @@ pub(super) fn lower_inline_dyn_typed_array_get( let blk = ctx.block(); let obj_bits = blk.bitcast_double_to_i64(obj_box); let raw = blk.and(I64, &obj_bits, pointer_mask); - // kind re-read from cache (cheap; keeps the fast block self-contained). - let slot = blk.lshr(I64, &raw, "3"); - let slot = blk.and(I64, &slot, "63"); - let entry_ptr = blk.gep( - "[64 x i64]", - "@PERRY_TA_KIND_CACHE", - &[(I64, "0"), (I64, &slot)], - ); - let entry_val = blk.load(I64, &entry_ptr); - let kind = blk.and(I64, &entry_val, "255"); + // kind re-read from the header (cheap; keeps the fast block + // self-contained). + let kind_addr = blk.add(I64, &raw, "8"); + let kind_ptr = blk.inttoptr(I64, &kind_addr); + let kind_i8 = blk.load(I8, &kind_ptr); + let kind = blk.zext(I8, &kind_i8, I64); // idx is in [0, 2^32) (entry guard) so fptosi i64 is well-defined. let idx_i64 = blk.fptosi(DOUBLE, idx_d, I64); (raw, idx_i64, kind) diff --git a/crates/perry-codegen/src/expr/index_get_claim_tests.rs b/crates/perry-codegen/src/expr/index_get_claim_tests.rs index 2bbfc20396..346908512d 100644 --- a/crates/perry-codegen/src/expr/index_get_claim_tests.rs +++ b/crates/perry-codegen/src/expr/index_get_claim_tests.rs @@ -278,3 +278,60 @@ fn erased_symbol_annotation_does_not_bypass_runtime_validation() { "a TypeScript Symbol annotation without initializer provenance must not enter the exact-Symbol IC:\n{ir}" ); } + +/// The inline dynamic typed-array read brands the receiver off its managed +/// `GC_TYPE_TYPED_ARRAY` header and reads the element kind from the +/// `TypedArrayHeader` itself, instead of probing the 64-slot direct-mapped +/// `PERRY_TA_KIND_CACHE` that every ordinary-array registry miss also writes +/// negative entries into (a hot typed array kept getting evicted and missed +/// the tier on every access). +#[test] +fn unknown_numeric_read_brands_typed_arrays_off_the_header_not_the_kind_cache() { + let ir = ir_for( + "unknown_typed_array_read_brand", + vec![ + Stmt::Let { + id: ITEMS, + name: "items".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::PropertyGet { + object: Box::new(Expr::Object(vec![( + "value".to_string(), + Expr::Array(vec![Expr::Number(7.0)]), + )])), + property: "value".to_string(), + byte_offset: 0, + }), + }, + Stmt::Let { + id: RESULT, + name: "result".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::IndexGet { + object: Box::new(Expr::LocalGet(ITEMS)), + index: Box::new(Expr::Integer(0)), + }), + }, + ], + ); + assert!( + ir.contains("tav.get.brand"), + "the inline typed-array tier must brand the receiver off its header:\n{ir}" + ); + let brand = super::class_field_barrier_tests::block_body(&ir, "tav.get.brand.") + .expect("brand block exists"); + assert!( + brand.contains("icmp eq i8") && brand.contains(", 11"), + "the brand block must test GC_TYPE_TYPED_ARRAY (11):\n{brand}" + ); + assert!( + brand.contains("load i8"), + "the element kind must be read from the TypedArrayHeader:\n{brand}" + ); + assert!( + !ir.contains("@PERRY_TA_KIND_CACHE"), + "the inline read must no longer depend on the kind cache:\n{ir}" + ); +} From ec3b4d8ff19abcb32b41451f3392a6e571afb9b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 10:52:56 +0200 Subject: [PATCH 07/15] perf(array): route object receivers to the subclass fast read before clean_arr_ptr An ordinary-object receiver (the object-backed `class X extends Array` instance behind wolf-ecs' `packed[sparse[x]]`) can never be an ArrayHeader, so clean_arr_ptr's tracked-allocation resolver was a guaranteed miss on every js_array_get_f64 call for it. Ask array_subclass_fast_index_get_raw first when the header tag already read for the Map/Set probes says GC_TYPE_OBJECT; every rejected case still reaches the complete resolver and spec-generic Get. wolf-ecs (Mac mini, 11 pairs): add/remove -2.03% (11/11), entity-cycle -2.39% (11/11). Cumulative vs v74: -16.5% / -20.9% (0.4645 / 0.3944 ms/op). Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- crates/perry-runtime/src/array/indexing.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/perry-runtime/src/array/indexing.rs b/crates/perry-runtime/src/array/indexing.rs index 69f6610c5a..8a18044ab3 100644 --- a/crates/perry-runtime/src/array/indexing.rs +++ b/crates/perry-runtime/src/array/indexing.rs @@ -873,6 +873,19 @@ pub extern "C" fn js_array_get_f64(arr: *const ArrayHeader, index: u32) -> f64 { ); } + // An ordinary-object receiver (the object-backed `class X extends Array` + // instance — the wolf-ecs `Archetype` behind `packed[sparse[x]]`) can + // never be an `ArrayHeader`, so `clean_arr_ptr`'s tracked-allocation + // resolver is a guaranteed miss for it. Ask the exact dense-subclass + // proof first; it re-validates the object header itself. Every rejected + // case (holes, descriptors, prototype overrides, spilled/unknown layouts) + // still reaches the complete resolver and spec-generic `Get` below. + if receiver_tag.0 == crate::gc::GC_TYPE_OBJECT { + if let Some(value) = crate::array::subclass::array_subclass_fast_index_get_raw(arr, index) { + return value; + } + } + let cleaned = clean_arr_ptr(arr); if cleaned.is_null() { // #7574: `a[i]` on a `class X extends Array` instance held in a From 4bf2c84a7ca2792efa854ec0b8ada01e754519e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 11:19:59 +0200 Subject: [PATCH 08/15] perf(codegen): give integer-valued dynamic keys the inline numeric read tiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A declared-array receiver read with an `Any`-typed key (`packed[sparse[x]]` in the wolf-ecs SparseSet, `a[b[i]]` in general) always took the out-of-line `js_array_get_index_or_string` route because the key carried no integer array-index proof. Test the key inline — nonnegative, below 2^32, and equal to its own fptosi/sitofp round trip — and on a hit take exactly the tiers a statically proven index takes: the inline typed-array read, the dense Array-subclass `arrlike.ic` shape cache, then the complete `js_packed_arraylike_index_get` → `js_dyn_index_get` dispatcher. Fractional, negative, NaN and out-of-range keys keep the previous route. wolf-ecs (Mac mini, 11 pairs): add/remove -2.37% (11/11), entity-cycle -2.89% (11/11); the js_array_get_index_or_string → js_array_get_f64 → array_subclass_fast_index_get_raw chain (4.4% of the add/remove profile) is gone. Cumulative vs v74: -18.5% / -23.1% (0.4531 / 0.3836 ms/op). Test: index_get_claim_tests::any_typed_dynamic_key_takes_the_numeric_tiers_when_it_is_an_array_index Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- crates/perry-codegen/src/expr/index_get.rs | 45 ++++++++++- .../src/expr/index_get_claim_tests.rs | 77 +++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/crates/perry-codegen/src/expr/index_get.rs b/crates/perry-codegen/src/expr/index_get.rs index 98fb37c6e6..1ff86d13c8 100644 --- a/crates/perry-codegen/src/expr/index_get.rs +++ b/crates/perry-codegen/src/expr/index_get.rs @@ -384,7 +384,46 @@ fn lower_claimable_array_string_key_get( let string_end = ctx.block().label.clone(); ctx.block().br(&merge_label); + // A dynamic key that is an integer-valued double in `[0, 2^32)` IS an + // array index, so the receiver-unknown numeric tiers apply to it exactly + // as they do to a statically proven index: the inline typed-array read, + // then the dense Array-subclass `arrlike.ic` shape cache, then the + // complete `js_packed_arraylike_index_get` → `js_dyn_index_get` + // dispatcher. Before this, an `Any`-typed key (`packed[sparse[x]]` in the + // wolf-ecs SparseSet, `a[b[i]]` in general) always took the out-of-line + // `js_array_get_index_or_string` route below. Fractional, negative, NaN + // and out-of-range keys keep that route unchanged; `-0` round-trips to + // index 0, which is what ToPropertyKey gives it too. + let int_idx = ctx.new_block("aidxkey.int"); + let int_label = ctx.block_label(int_idx); + let generic_idx = ctx.new_block("aidxkey.generic"); + let generic_label = ctx.block_label(generic_idx); ctx.current_block = array_idx; + { + let blk = ctx.block(); + let nonnegative = blk.fcmp("oge", idx_double, "0.0"); + let below_limit = blk.fcmp("olt", idx_double, "4294967296.0"); + let in_range = blk.and(I1, &nonnegative, &below_limit); + blk.cond_br(&in_range, &int_label, &generic_label); + } + ctx.current_block = int_idx; + let int_label_checked = ctx.new_block("aidxkey.int.exact"); + let int_label_checked_label = ctx.block_label(int_label_checked); + { + let blk = ctx.block(); + // In range, so `fptosi` is well-defined; the round trip rejects + // fractional keys. + let idx_i64 = blk.fptosi(DOUBLE, idx_double, I64); + let idx_back = blk.sitofp(I64, &idx_i64, DOUBLE); + let is_integer = blk.fcmp("oeq", &idx_back, idx_double); + blk.cond_br(&is_integer, &int_label_checked_label, &generic_label); + } + ctx.current_block = int_label_checked; + let index_value = lower_inline_dyn_typed_array_get(ctx, arr_box, idx_double, false); + let index_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = generic_idx; let arr_handle = unbox_to_i64(ctx.block(), arr_box); let array_value = ctx.block().call( DOUBLE, @@ -397,7 +436,11 @@ fn lower_claimable_array_string_key_get( ctx.current_block = merge_idx; ctx.block().phi( DOUBLE, - &[(&string_value, &string_end), (&array_value, &array_end)], + &[ + (&string_value, &string_end), + (&index_value, &index_end), + (&array_value, &array_end), + ], ) } diff --git a/crates/perry-codegen/src/expr/index_get_claim_tests.rs b/crates/perry-codegen/src/expr/index_get_claim_tests.rs index 346908512d..297ce6f172 100644 --- a/crates/perry-codegen/src/expr/index_get_claim_tests.rs +++ b/crates/perry-codegen/src/expr/index_get_claim_tests.rs @@ -335,3 +335,80 @@ fn unknown_numeric_read_brands_typed_arrays_off_the_header_not_the_kind_cache() "the inline read must no longer depend on the kind cache:\n{ir}" ); } + +/// An `Any`-typed dynamic key (`packed[sparse[x]]`, `a[b[i]]`) on a +/// declared-array receiver is tested inline for "integer-valued double in +/// [0, 2^32)"; a hit takes the same receiver-unknown numeric tiers a +/// statically proven index takes (inline typed-array read → dense +/// Array-subclass `arrlike.ic` → complete dispatcher), while every other key +/// keeps the out-of-line `js_array_get_index_or_string` route. +#[test] +fn any_typed_dynamic_key_takes_the_numeric_tiers_when_it_is_an_array_index() { + const SPARSE: u32 = 41; + let ir = ir_for( + "any_key_index_read", + vec![ + Stmt::Let { + id: ITEMS, + name: "packed".to_string(), + ty: Type::Array(Box::new(Type::Any)), + mutable: false, + init: Some(Expr::PropertyGet { + object: Box::new(Expr::Object(vec![( + "value".to_string(), + Expr::Array(vec![Expr::Number(7.0)]), + )])), + property: "value".to_string(), + byte_offset: 0, + }), + }, + Stmt::Let { + id: SPARSE, + name: "sparse".to_string(), + ty: Type::Array(Box::new(Type::Any)), + mutable: false, + init: Some(Expr::PropertyGet { + object: Box::new(Expr::Object(vec![( + "value".to_string(), + Expr::Array(vec![Expr::Number(0.0)]), + )])), + property: "value".to_string(), + byte_offset: 0, + }), + }, + Stmt::Let { + id: RESULT, + name: "result".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::IndexGet { + object: Box::new(Expr::LocalGet(ITEMS)), + index: Box::new(Expr::IndexGet { + object: Box::new(Expr::LocalGet(SPARSE)), + index: Box::new(Expr::Integer(0)), + }), + }), + }, + ], + ); + assert!( + ir.contains("aidxkey.int.exact") && ir.contains("aidxkey.generic"), + "the dynamic key must be classified inline before choosing a route:\n{ir}" + ); + let exact = super::class_field_barrier_tests::block_body(&ir, "aidxkey.int.") + .expect("the range-checked key block exists"); + assert!( + exact.contains("fptosi double") + && exact.contains("sitofp i64") + && exact.contains("fcmp oeq"), + "the integer test must be the fptosi/sitofp round trip:\n{exact}" + ); + assert!( + ir.contains("tav.get.brand") && ir.contains("arrlike.ic.family_token"), + "an integer key must reach the inline typed-array and dense-subclass tiers:\n{ir}" + ); + assert!( + ir.contains("call double @js_array_get_index_or_string("), + "non-index keys must keep the complete key route:\n{ir}" + ); +} From 0c00774bd4f661c7ff00b5911aed046afcc15a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 11:24:00 +0200 Subject: [PATCH 09/15] test(codegen): update the proven-number strict-eq rooting test to the inline lowering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `x === {…}` with a proven-Number left operand now lowers to an inline `fcmp oeq` (every non-Number NaN-box reads as a NaN double, so the object compares unequal exactly as `js_eq` answered), leaving no `js_eq` call for the test to find. Keep the test's actual claim — the non-pointer left operand stays in the register produced above the right operand's allocation instead of being rooted/re-read — on the fcmp operands, and pin that no runtime equality call remains. Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- crates/perry-codegen/src/expr/compare_tests.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/perry-codegen/src/expr/compare_tests.rs b/crates/perry-codegen/src/expr/compare_tests.rs index 4e51625792..97086ddc1e 100644 --- a/crates/perry-codegen/src/expr/compare_tests.rs +++ b/crates/perry-codegen/src/expr/compare_tests.rs @@ -318,8 +318,20 @@ fn strict_eq_reuses_a_non_pointer_left_operand_across_an_allocating_right_operan }, ], ); - let left = call_operand_of(&ir, "js_eq", 0); - let right = call_operand_of(&ir, "js_eq", 1); + // A proven-Number left operand lowers the whole comparison inline: every + // non-Number NaN-box reads as a NaN double, so `fcmp oeq` answers `false` + // for the object exactly as `js_eq` would, and no helper call remains. + assert!( + !ir.contains("@js_eq(") && !ir.contains("@js_strict_eq("), + "a proven-Number left operand must not pay a runtime equality call:\n{ir}" + ); + let fcmp = ir + .lines() + .map(str::trim) + .find(|line| line.contains("fcmp oeq double")) + .unwrap_or_else(|| panic!("no inline numeric strict-equality compare in:\n{ir}")); + let left = super::class_field_barrier_tests::operand(fcmp, 1).expect("fcmp left operand"); + let right = super::class_field_barrier_tests::operand(fcmp, 2).expect("fcmp right operand"); let left_producer = producer_line(&ir, &left); let right_producer = producer_line(&ir, &right); assert!( From c41343c2d99a1f1406d29a04fe7d0db890bde500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 12:27:50 +0200 Subject: [PATCH 10/15] ci: ratchet baselines for the file splits, census gate for cache carriers, changelog fragment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - addr-class ratchet/allowlist and raw-handle debt ceilings: the sites that the 2,000-line split moved from `array/indexing.rs` into `array/indexing_keyed.rs` keep their existing justification under the new path (indexing 4→3 / 13→7, indexing_keyed 1 / 6); lower the stale `field_set_by_name/fast_paths.rs` handle-floor count 3→2. - shape-descriptor census: refresh the exact call-site multiset for the moved `property_get/composed_ics.rs` sites and the new `stmt/cached_field_index_return.rs` / `generic_dispatch.rs` header-size reads, and pin the scanner's rooting gate as `descriptor.old_carrier || descriptor.cache_carrier` — a runtime optimization cache that can reinstall a historical shape is a strong metadata owner a minor cannot enumerate (see `ShapeDescriptor:: cache_carrier`), so its keys array must be rooted and rewritten before weak pruning. The sabotage self-test is updated to the new gate. - changelog.d/8876 fragment. Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- .../8876-ecs-forwarded-store-inline-tiers.md | 22 +++++++++++++++++++ scripts/addr_class_allowlist.txt | 1 + scripts/addr_class_ratchet_baseline.txt | 5 +++-- scripts/raw_handle_debt_files.txt | 3 ++- scripts/shape_descriptor_census.py | 12 +++++----- scripts/shape_descriptor_census_baseline.json | 7 +++++- 6 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 changelog.d/8876-ecs-forwarded-store-inline-tiers.md diff --git a/changelog.d/8876-ecs-forwarded-store-inline-tiers.md b/changelog.d/8876-ecs-forwarded-store-inline-tiers.md new file mode 100644 index 0000000000..4e33edb0d8 --- /dev/null +++ b/changelog.d/8876-ecs-forwarded-store-inline-tiers.md @@ -0,0 +1,22 @@ +Array/ECS performance: guarded property-receiver stores now follow one +growth-forwarding edge inline (the read tier already did), so a field that +kept a pre-grow forwarding stub (`this.ents[id] = arch`) no longer pays the +out-of-line extend helper and allocator resolver on every store; the raw-f64 +downgrade note is gated on the header word already loaded; `typeof x === +"number"` decides the definitely-Number cases inline (exactly, deferring +INT32/class refs, raw typed-array pointers and the Web Streams id band to the +classifier); inline dynamic typed-array reads brand off the +`GC_TYPE_TYPED_ARRAY` header instead of the evictable 64-slot kind cache; +`js_array_get_f64` routes object-backed Array-subclass receivers to their +dense fast read before the tracked resolver; and an `Any`-typed key that is an +integer array index takes the inline numeric read tiers (`a[b[i]]`). + +Also carries the accumulated Array-subclass dense-tail work (validated +prototype-override reads, pre-statepoint inlining of compact guarded +specializations by lowered IR size) and splits six oversized source files into +child modules. + +wolf-ecs (noctjs/ecs-benchmark) on the Mac mini reference box, versus the +previous retained build: add/remove -18.5% (0.556 → 0.453 ms/op), entity-cycle +-23.1% (0.499 → 0.384 ms/op); each step 11/11 paired wins, semantics probes +byte-identical to Node. diff --git a/scripts/addr_class_allowlist.txt b/scripts/addr_class_allowlist.txt index 55b95c8d47..d9f317b916 100644 --- a/scripts/addr_class_allowlist.txt +++ b/scripts/addr_class_allowlist.txt @@ -33,6 +33,7 @@ crates/perry-runtime/src/array/flat_clone.rs | * | pre-existing GcHeader probe p crates/perry-runtime/src/array/generic.rs | * | pre-existing GcHeader probe predating addr_class; address validated by call-site guards (magnitude/registry/is_valid_obj_ptr) -- migrate to addr_class::try_read_gc_header in a follow-up crates/perry-runtime/src/array/header.rs | * | pre-existing GcHeader probe predating addr_class; address validated by call-site guards (magnitude/registry/is_valid_obj_ptr) -- migrate to addr_class::try_read_gc_header in a follow-up crates/perry-runtime/src/array/indexing.rs | * | pre-existing GcHeader probe predating addr_class; address validated by call-site guards (magnitude/registry/is_valid_obj_ptr) -- migrate to addr_class::try_read_gc_header in a follow-up +crates/perry-runtime/src/array/indexing_keyed.rs | * | same pre-existing GcHeader probe, moved from indexing.rs by the 2,000-line file split (js_array_set_string_key); migrate to addr_class::try_read_gc_header in a follow-up crates/perry-runtime/src/array/is_array.rs | * | pre-existing GcHeader probe predating addr_class; address validated by call-site guards (magnitude/registry/is_valid_obj_ptr) -- migrate to addr_class::try_read_gc_header in a follow-up crates/perry-runtime/src/array/iter_methods.rs | * | pre-existing GcHeader probe predating addr_class; address validated by call-site guards (magnitude/registry/is_valid_obj_ptr) -- migrate to addr_class::try_read_gc_header in a follow-up crates/perry-runtime/src/array/iter_object.rs | * | pre-existing GcHeader probe predating addr_class; address validated by call-site guards (magnitude/registry/is_valid_obj_ptr) -- migrate to addr_class::try_read_gc_header in a follow-up diff --git a/scripts/addr_class_ratchet_baseline.txt b/scripts/addr_class_ratchet_baseline.txt index be65a3e608..f476ad620e 100644 --- a/scripts/addr_class_ratchet_baseline.txt +++ b/scripts/addr_class_ratchet_baseline.txt @@ -32,7 +32,8 @@ handle-floor | crates/perry-runtime/src/array/concat_reverse.rs | 1 handle-floor | crates/perry-runtime/src/array/flat_clone.rs | 4 handle-floor | crates/perry-runtime/src/array/generic.rs | 4 handle-floor | crates/perry-runtime/src/array/header.rs | 3 -handle-floor | crates/perry-runtime/src/array/indexing.rs | 4 +handle-floor | crates/perry-runtime/src/array/indexing.rs | 3 +handle-floor | crates/perry-runtime/src/array/indexing_keyed.rs | 1 handle-floor | crates/perry-runtime/src/array/iter_object.rs | 1 handle-floor | crates/perry-runtime/src/array/iterator.rs | 2 handle-floor | crates/perry-runtime/src/array/push_pop.rs | 1 @@ -117,7 +118,7 @@ handle-floor | crates/perry-runtime/src/object/field_get_set/get_field_by_name_t handle-floor | crates/perry-runtime/src/object/field_get_set/has_property.rs | 2 handle-floor | crates/perry-runtime/src/object/field_get_set/ic_miss.rs | 4 handle-floor | crates/perry-runtime/src/object/field_set_by_name.rs | 1 -handle-floor | crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs | 3 +handle-floor | crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs | 2 handle-floor | crates/perry-runtime/src/object/field_set_by_name/tail.rs | 6 handle-floor | crates/perry-runtime/src/object/field_set_by_name/write_helpers.rs | 1 handle-floor | crates/perry-runtime/src/object/global_this/array_error.rs | 1 diff --git a/scripts/raw_handle_debt_files.txt b/scripts/raw_handle_debt_files.txt index 974f0d9d3a..c5fc1cce22 100644 --- a/scripts/raw_handle_debt_files.txt +++ b/scripts/raw_handle_debt_files.txt @@ -48,7 +48,8 @@ # # Format: 5 crates/perry-runtime/src/array/header.rs -13 crates/perry-runtime/src/array/indexing.rs +7 crates/perry-runtime/src/array/indexing.rs +6 crates/perry-runtime/src/array/indexing_keyed.rs 4 crates/perry-runtime/src/array/flat_clone.rs 2 crates/perry-runtime/src/array/iter_methods.rs 31 crates/perry-runtime/src/array/iterator.rs diff --git a/scripts/shape_descriptor_census.py b/scripts/shape_descriptor_census.py index 9e0d96b1ca..9c946127c1 100644 --- a/scripts/shape_descriptor_census.py +++ b/scripts/shape_descriptor_census.py @@ -315,6 +315,7 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: # nothing loses the keys array of a shape only OLD objects carry, which # a minor never enumerates. (r"pub\(crate\)\s+old_carrier\s*:\s*bool", "old-carrier ephemeron gate"), + (r"pub\(crate\)\s+cache_carrier\s*:\s*bool", "cache-carrier strong metadata owner"), (r"\bfn\s+rotate_old_carrier_epoch_after_full_trace\b", "old-carrier gate recomputed by a full trace"), (r"is_dead_owner\s*\(\s*descriptor\.keys\s+as\s+usize\s*\)", "dead descriptor pruning"), ): @@ -343,15 +344,16 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: # just the set of APIs called: a sabotage that widens the gate, or that # swaps the arms, has to be red. if not re.search( - r"if\s+descriptor\.old_carrier\s*\{\s*" + r"if\s+descriptor\.old_carrier\s*\|\|\s*descriptor\.cache_carrier\s*\{\s*" r"visitor\.visit_usize_slot\(&mut addr\)\s*\}\s*else\s*\{\s*" r"visitor\.visit_metadata_usize_slot\(&mut addr\)\s*\}", scanner, ): raise CensusError( - "descriptor rooting is not gated on `old_carrier`: the shape table " - "either roots unconditionally (every keys array immortal) or not at " - "all (a shape only old objects carry loses its keys array)" + "descriptor rooting is not gated on `old_carrier || cache_carrier`: " + "the shape table either roots unconditionally (every keys array " + "immortal) or not at all (a shape only old objects carry, or one a " + "runtime optimization cache can reinstall, loses its keys array)" ) scanner_slot_apis = set(re.findall(r"\b(visit_[A-Za-z0-9_]*slot)\s*\(", scanner)) if scanner_slot_apis != {"visit_metadata_usize_slot", "visit_usize_slot"}: @@ -739,7 +741,7 @@ def run_sabotage_selftests(sources: dict[str, str], baseline: dict[str, object]) ungated_root = dict(sources) ungated_root[shapes_path] = ungated_root[shapes_path].replace( - "let moved = if descriptor.old_carrier {", + "let moved = if descriptor.old_carrier || descriptor.cache_carrier {", "let moved = if true {", 1, ) diff --git a/scripts/shape_descriptor_census_baseline.json b/scripts/shape_descriptor_census_baseline.json index 27adb5bbcd..0f6db6d7fb 100644 --- a/scripts/shape_descriptor_census_baseline.json +++ b/scripts/shape_descriptor_census_baseline.json @@ -6,7 +6,10 @@ "crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs|let meta_offset = (crate::target_layout::object_header_size_bytes(ctx.target_triple)": 1, "crates/perry-codegen/src/expr/member_update.rs|let header_skip = crate::target_layout::object_header_size_bytes(": 1, "crates/perry-codegen/src/expr/property_get.rs|crate::target_layout::object_header_size_bytes(ctx.target_triple)": 3, + "crates/perry-codegen/src/expr/property_get/composed_ics.rs|crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string();": 1, + "crates/perry-codegen/src/expr/property_get/composed_ics.rs|let meta_offset = (crate::target_layout::object_header_size_bytes(ctx.target_triple)": 1, "crates/perry-codegen/src/expr/property_get/generic_dispatch.rs|crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string();": 1, + "crates/perry-codegen/src/expr/property_get/generic_dispatch.rs|let meta_offset = (crate::target_layout::object_header_size_bytes(ctx.target_triple)": 1, "crates/perry-codegen/src/expr/property_get/helpers.rs|crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string();": 3, "crates/perry-codegen/src/expr/property_get/helpers.rs|let header_skip = crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string();": 1, "crates/perry-codegen/src/expr/property_set.rs|crate::target_layout::object_header_size_bytes(": 3, @@ -18,6 +21,8 @@ "crates/perry-codegen/src/lower_call/property_get/imported_object.rs|let header_skip = crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string();": 1, "crates/perry-codegen/src/lower_call/scalar_method.rs|let header_skip = crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string();": 1, "crates/perry-codegen/src/lower_call/typed_shape_bake_tests.rs|8 + crate::target_layout::object_header_size_bytes( ) + 8 * slots;": 1, + "crates/perry-codegen/src/stmt/cached_field_index_return.rs|let header_size = crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string();": 1, + "crates/perry-codegen/src/stmt/cached_field_index_return.rs|let meta_offset = (crate::target_layout::object_header_size_bytes(ctx.target_triple)": 1, "crates/perry-codegen/src/stmt/loops.rs|let object_header_size = crate::target_layout::object_header_size_bytes(ctx.target_triple);": 1, "crates/perry-codegen/src/stmt/stable_packed_loop.rs|crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string();": 2, "crates/perry-codegen/src/stmt/stable_packed_loop.rs|let meta_offset = (crate::target_layout::object_header_size_bytes(ctx.target_triple)": 2, @@ -50,7 +55,7 @@ "crates/perry-runtime/src/object/object_ops.rs|keys_array|declaration|pub(crate) use keys_array::{": 1 }, "summary": { - "codegen_object_header_size_sites": 41, + "codegen_object_header_size_sites": 46, "raw_member_files": 7, "raw_member_sites": { "keys_array": 24 From a34c580ced2e765a66dc26717bfac79c5cbad9a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 13:19:01 +0200 Subject: [PATCH 11/15] perf(runtime): array-read fallback serves object-backed Array subclasses densely After merging main (#8878 / #8872's canonical-i32 read split), a declared-array receiver with a non-static key takes the guarded plain-array tier first. On an object-backed `class X extends Array` receiver (wolf-ecs `Archetype`, `packed[sparse[x]]` in SparseSet.has/remove) that guard always misses and `js_typed_feedback_array_index_get_fallback_boxed`'s GC_TYPE_OBJECT arm stringified every index into a by-name lookup (from_utf8 + string alloc + reflection ladder per read): both wolf-ecs benchmarks regressed ~2.2x. The fallback now asks `array_subclass_fast_index_get` for a canonical (plain or INT32-boxed) non-negative index before its registry probes and the by-name path; receivers without a dense proof keep the established route. Mac mini 11-pair screen vs the pre-merge build: add/remove +0.5%, entity-cycle -1.2% (from +126% / +121%); semantics probe byte-identical to Node. Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- .../8876-ecs-forwarded-store-inline-tiers.md | 6 ++- crates/perry-runtime/src/typed_feedback.rs | 14 ++++++ .../perry-runtime/src/typed_feedback/tests.rs | 47 +++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/changelog.d/8876-ecs-forwarded-store-inline-tiers.md b/changelog.d/8876-ecs-forwarded-store-inline-tiers.md index 4e33edb0d8..aba9db6467 100644 --- a/changelog.d/8876-ecs-forwarded-store-inline-tiers.md +++ b/changelog.d/8876-ecs-forwarded-store-inline-tiers.md @@ -9,7 +9,11 @@ classifier); inline dynamic typed-array reads brand off the `GC_TYPE_TYPED_ARRAY` header instead of the evictable 64-slot kind cache; `js_array_get_f64` routes object-backed Array-subclass receivers to their dense fast read before the tracked resolver; and an `Any`-typed key that is an -integer array index takes the inline numeric read tiers (`a[b[i]]`). +integer array index takes the inline numeric read tiers (`a[b[i]]`). The +typed-feedback array-read fallback answers a proven dense read on an +object-backed Array-subclass receiver before its registry probes and by-name +key path, so the canonical-i32 element tier's guard miss on such a receiver +no longer stringifies every index. Also carries the accumulated Array-subclass dense-tail work (validated prototype-override reads, pre-statepoint inlining of compact guarded diff --git a/crates/perry-runtime/src/typed_feedback.rs b/crates/perry-runtime/src/typed_feedback.rs index 5dffc05ee9..1a8d9762ba 100644 --- a/crates/perry-runtime/src/typed_feedback.rs +++ b/crates/perry-runtime/src/typed_feedback.rs @@ -2082,6 +2082,20 @@ pub extern "C" fn js_typed_feedback_array_index_get_fallback_boxed( return f64::from_bits(TAG_UNDEFINED); } + // #8876: an object-backed `class X extends Array` instance carries + // `GC_TYPE_OBJECT`, so the guarded element tier rejects it and every + // canonical index on such a receiver lands here. Answer the proven dense + // subclass read first — the registry probes below cannot classify it, and + // the `GC_TYPE_OBJECT` arm stringifies the index into a by-name lookup + // (wolf-ecs `packed[sparse[x]]` on an `Archetype` paid `from_utf8` + + // string allocation per read). A receiver without a dense proof keeps the + // established path. + if let Some(index) = finite_nonnegative_u32_index(index) { + if let Some(value) = crate::array::array_subclass_fast_index_get(receiver, index) { + return value; + } + } + if crate::typedarray::lookup_typed_array_kind(raw_addr).is_some() { return crate::typedarray::js_typed_array_index_get_dynamic( raw_addr as *const crate::typedarray::TypedArrayHeader, diff --git a/crates/perry-runtime/src/typed_feedback/tests.rs b/crates/perry-runtime/src/typed_feedback/tests.rs index 4c027f2e6a..06ce8302e7 100644 --- a/crates/perry-runtime/src/typed_feedback/tests.rs +++ b/crates/perry-runtime/src/typed_feedback/tests.rs @@ -476,6 +476,53 @@ fn typed_feedback_array_get_guard_failure_uses_jsvalue_object_fallback() { assert_eq!(site.fallback_calls, 1); } +#[test] +fn typed_feedback_array_get_fallback_reads_an_object_backed_array_subclass_densely() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + register(26, TypedFeedbackSiteKind::ArrayElement, "packed[i]"); + + // `class X extends Array` — the receiver carries GC_TYPE_OBJECT, so the + // guarded element tier rejects it and every canonical index lands in the + // fallback. #8876: the fallback answers the proven dense subclass read + // instead of stringifying the index into an `obj["1"]` by-name lookup. + const CLASS_ID_ARRAY: u32 = 0xFFFF_0024; + let class_id = 0x0074_8656; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let obj = crate::object::js_object_alloc(class_id, 2); + assert!(!obj.is_null()); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + for (index, value) in [11.0, 22.0, 33.0].into_iter().enumerate() { + crate::object::js_object_set_index_polymorphic(obj as i64, index as f64, value); + } + + let guard = js_typed_feedback_plain_array_index_get_guard(26, receiver, 1, 3); + assert_eq!( + guard, 0, + "an object-backed subclass must fail the plain-array guard" + ); + assert_eq!( + crate::array::array_subclass_fast_index_get(receiver, 1), + Some(22.0), + "fixture must carry a dense subclass proof" + ); + + let actual = js_typed_feedback_array_index_get_fallback_boxed(26, receiver, 1.0); + assert_eq!(actual.to_bits(), 22.0f64.to_bits()); + let boxed_index = f64::from_bits(crate::value::JSValue::int32(2).bits()); + let actual = js_typed_feedback_array_index_get_fallback_boxed(26, receiver, boxed_index); + assert_eq!(actual.to_bits(), 33.0f64.to_bits()); + + // Past the dense tail the established by-name path still answers. + let missing = js_typed_feedback_array_index_get_fallback_boxed(26, receiver, 7.0); + assert_eq!(missing.to_bits(), crate::value::TAG_UNDEFINED); + + let site = &typed_feedback_snapshot().sites[0]; + assert_eq!(site.guard_failures, 1); + assert_eq!(site.fallback_calls, 3); +} + #[test] fn typed_feedback_non_bounded_array_set_guard_failure_uses_jsvalue_object_fallback() { let _guard = typed_feedback_test_lock(); From 08b7e02e07cb052fbf6bc4aabf67c86a04829013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 13:28:38 +0200 Subject: [PATCH 12/15] runtime(array): keyed index paths reload the receiver via across_* (raw-handle debt -6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file split moved six bare `get_raw_{mut,const}_ptr` reads into `indexing_keyed.rs`, which the raw-handle ratchet rejects as a module that was not listed at the merge base. Every site had the sanctioned shape already — root the receiver, run the allocating stringify / symbol store, reload — so they now use `across_const` / `across_mut`. `indexing_keyed.rs` needs no ceiling; the baseline ratchets 970 -> 964. Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- .../perry-runtime/src/array/indexing_keyed.rs | 28 +++++++++---------- scripts/raw_handle_debt_baseline.txt | 2 +- scripts/raw_handle_debt_files.txt | 7 ++--- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/crates/perry-runtime/src/array/indexing_keyed.rs b/crates/perry-runtime/src/array/indexing_keyed.rs index 844bd78a27..b812fe4b6a 100644 --- a/crates/perry-runtime/src/array/indexing_keyed.rs +++ b/crates/perry-runtime/src/array/indexing_keyed.rs @@ -207,14 +207,12 @@ pub extern "C" fn js_array_get_index_or_string(arr: *const ArrayHeader, idx: f64 // `a[{toString(){...}}]`) runs user JS, allocates and can evacuate `arr`. let scope = crate::gc::RuntimeHandleScope::new(); let arr_handle = scope.root_raw_const_ptr(arr); - let key = crate::value::js_jsvalue_to_string(idx); + let (key, arr) = + arr_handle.across_const::(|| crate::value::js_jsvalue_to_string(idx)); if key.is_null() { return f64::from_bits(crate::value::TAG_UNDEFINED); } - array_get_property_by_key( - arr_handle.get_raw_const_ptr::(), - key as *const crate::StringHeader, - ) + array_get_property_by_key(arr, key as *const crate::StringHeader) } /// `arr[idx] = value` where idx may be a NaN-boxed string (numeric-string @@ -279,15 +277,16 @@ pub extern "C" fn js_array_set_index_or_string( let scope = crate::gc::RuntimeHandleScope::new(); let arr_handle = scope.root_raw_mut_ptr(arr); let value_handle = scope.root_nanbox_f64(value); - let key = crate::value::js_jsvalue_to_string(idx); + let (key, arr) = + arr_handle.across_mut::(|| crate::value::js_jsvalue_to_string(idx)); if !key.is_null() { return js_array_set_string_key( - arr_handle.get_raw_mut_ptr::(), + arr, key as *const crate::StringHeader, value_handle.get_nanbox_f64(), ); } - return arr_handle.get_raw_mut_ptr::(); + return arr; } // Symbol-keyed write: store through the symbol side table (keyed by the // header address), exactly like a plain-object receiver. This arm used to @@ -301,14 +300,14 @@ pub extern "C" fn js_array_set_index_or_string( // array), which can GC and evacuate the receiver. let scope = crate::gc::RuntimeHandleScope::new(); let arr_handle = scope.root_raw_mut_ptr(arr); - unsafe { + let ((), arr) = arr_handle.across_mut::(|| unsafe { crate::symbol::js_object_set_symbol_property( crate::value::js_nanbox_pointer(arr as i64), idx, value, ); - } - return arr_handle.get_raw_mut_ptr::(); + }); + return arr; } // Fallback for a NON-numeric key: a primitive (`a[null]`, `a[undefined]`, // `a[true]`, `a[10n]`) or a boxed object (`a[new Number(1)]`). Per @@ -327,15 +326,16 @@ pub extern "C" fn js_array_set_index_or_string( let scope = crate::gc::RuntimeHandleScope::new(); let arr_handle = scope.root_raw_mut_ptr(arr); let value_handle = scope.root_nanbox_f64(value); - let key = crate::value::js_jsvalue_to_string(idx); + let (key, arr) = + arr_handle.across_mut::(|| crate::value::js_jsvalue_to_string(idx)); if !key.is_null() { return js_array_set_string_key( - arr_handle.get_raw_mut_ptr::(), + arr, key as *const crate::StringHeader, value_handle.get_nanbox_f64(), ); } - return arr_handle.get_raw_mut_ptr::(); + return arr; } arr } diff --git a/scripts/raw_handle_debt_baseline.txt b/scripts/raw_handle_debt_baseline.txt index e59798702d..03b7719c61 100644 --- a/scripts/raw_handle_debt_baseline.txt +++ b/scripts/raw_handle_debt_baseline.txt @@ -1 +1 @@ -970 +964 diff --git a/scripts/raw_handle_debt_files.txt b/scripts/raw_handle_debt_files.txt index c5fc1cce22..1af48ace6f 100644 --- a/scripts/raw_handle_debt_files.txt +++ b/scripts/raw_handle_debt_files.txt @@ -47,10 +47,9 @@ # rule above covers 84% of the crate on day one. # # Format: +4 crates/perry-runtime/src/array/flat_clone.rs 5 crates/perry-runtime/src/array/header.rs 7 crates/perry-runtime/src/array/indexing.rs -6 crates/perry-runtime/src/array/indexing_keyed.rs -4 crates/perry-runtime/src/array/flat_clone.rs 2 crates/perry-runtime/src/array/iter_methods.rs 31 crates/perry-runtime/src/array/iterator.rs 12 crates/perry-runtime/src/array/push_pop.rs @@ -90,6 +89,7 @@ 4 crates/perry-runtime/src/node_submodules/mod.rs 4 crates/perry-runtime/src/node_submodules/test.rs 33 crates/perry-runtime/src/object/alloc.rs +2 crates/perry-runtime/src/object/array_object_ops.rs 2 crates/perry-runtime/src/object/bigint_dispatch.rs 3 crates/perry-runtime/src/object/class_registry/construct.rs 2 crates/perry-runtime/src/object/delete_rest.rs @@ -128,6 +128,7 @@ 3 crates/perry-runtime/src/promise/rejection.rs 3 crates/perry-runtime/src/promise/then.rs 8 crates/perry-runtime/src/proxy.rs +6 crates/perry-runtime/src/proxy/put_value.rs 6 crates/perry-runtime/src/regex.rs 19 crates/perry-runtime/src/regex/exec_array.rs 13 crates/perry-runtime/src/regex/match_all.rs @@ -157,5 +158,3 @@ 19 crates/perry-runtime/src/wasi.rs 6 crates/perry-runtime/src/weakref.rs 20 crates/perry-runtime/src/webassembly.rs -6 crates/perry-runtime/src/proxy/put_value.rs -2 crates/perry-runtime/src/object/array_object_ops.rs From 1dd37f9c04da96a6c2c76f3ec33cb767ed78dc7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 16:18:08 +0200 Subject: [PATCH 13/15] review: keep the fused u31 push non-reentrant, bound ECS columns, lifecycle the cache-carrier gate Review follow-ups on #8876: - `js_array_push_u31_with_length` stays allocate-but-never-reenter: it now answers null for receivers whose push can run user code (indexed descriptors / prototype indices, Proxy traps, foreign families) instead of calling the spec / public push itself; the generated caller takes the complete guarded push (`js_array_push_guard` + `js_array_push_f64`) in a new `apush.u31.generic` block. Test: the fused-push runtime test declines a typed-array receiver; the composed-clone IR test pins the hot path / fallback split. - `js_packed_ecs_u32_loop_guard` declines admission when a component column is shorter than the admitted bound the receiver guard published (`out[6]`), so the fused loop cannot read past a column's payload. Test added. - `object_hot_for_owner` validates the cached table pointer against the current thread's `RuntimeState` before reuse. - `cache_carrier` gets a lifecycle: noted only after an entry naming the pair was inserted, and recomputed from live table occupancy after every full trace (`recompute_cache_carriers_after_full_trace`, called beside the old-carrier rotation) so a descriptor whose entries were evicted stops being rooted. Test: carrier bits follow live occupancy across a recompute. - `js_object_get_symbol_then_field_ic_miss` is declared with the runtime's pointer parameter type. - Minor: parenthesized mixed `&&`/`||` assertion, unique test class ids, `function_this_safe` visited-key includes the terminal-`this` allowance, exhaustive `UnaryOp` match, changelog fragment restated as shipped behavior. Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- .../8876-ecs-forwarded-store-inline-tiers.md | 30 ++--- .../src/codegen/index_method_clone_tests.rs | 24 +++- .../perry-codegen/src/collectors/ptr_shape.rs | 11 +- .../src/collectors/ptr_shape_numeric.rs | 2 +- .../perry-codegen/src/expr/compare_tests.rs | 2 +- .../src/expr/property_get/composed_ics.rs | 3 +- .../native/native_instance_branch.rs | 34 +++++- .../src/runtime_decls/strings_part2.rs | 2 +- crates/perry-runtime/src/array/push_pop.rs | 40 +++---- .../src/array/subclass_loop_guard.rs | 8 ++ .../perry-runtime/src/array/subclass_tests.rs | 111 +++++++++++++++++- crates/perry-runtime/src/gc/dead_owner.rs | 3 + .../src/object/array_tail_transition.rs | 56 +++++++-- crates/perry-runtime/src/object/shapes.rs | 17 ++- 14 files changed, 279 insertions(+), 64 deletions(-) diff --git a/changelog.d/8876-ecs-forwarded-store-inline-tiers.md b/changelog.d/8876-ecs-forwarded-store-inline-tiers.md index aba9db6467..0fa36032a2 100644 --- a/changelog.d/8876-ecs-forwarded-store-inline-tiers.md +++ b/changelog.d/8876-ecs-forwarded-store-inline-tiers.md @@ -1,24 +1,18 @@ -Array/ECS performance: guarded property-receiver stores now follow one +Array and ECS performance. Guarded property-receiver element stores follow one growth-forwarding edge inline (the read tier already did), so a field that kept a pre-grow forwarding stub (`this.ents[id] = arch`) no longer pays the out-of-line extend helper and allocator resolver on every store; the raw-f64 -downgrade note is gated on the header word already loaded; `typeof x === -"number"` decides the definitely-Number cases inline (exactly, deferring -INT32/class refs, raw typed-array pointers and the Web Streams id band to the -classifier); inline dynamic typed-array reads brand off the -`GC_TYPE_TYPED_ARRAY` header instead of the evictable 64-slot kind cache; -`js_array_get_f64` routes object-backed Array-subclass receivers to their -dense fast read before the tracked resolver; and an `Any`-typed key that is an -integer array index takes the inline numeric read tiers (`a[b[i]]`). The -typed-feedback array-read fallback answers a proven dense read on an -object-backed Array-subclass receiver before its registry probes and by-name -key path, so the canonical-i32 element tier's guard miss on such a receiver -no longer stringifies every index. - -Also carries the accumulated Array-subclass dense-tail work (validated -prototype-override reads, pre-statepoint inlining of compact guarded -specializations by lowered IR size) and splits six oversized source files into -child modules. +downgrade note is gated on the header word already loaded. `typeof x === +"number"` decides the definitely-Number cases inline, deferring INT32/class +refs, raw typed-array pointers and the Web Streams id band to the classifier. +Inline dynamic typed-array reads brand off the `GC_TYPE_TYPED_ARRAY` header +instead of the evictable 64-slot kind cache. `js_array_get_f64` and the +typed-feedback array-read fallback route object-backed Array-subclass +receivers to their dense fast read before the tracked resolver and the by-name +key path, and an `Any`-typed key that is an integer array index takes the +inline numeric read tiers (`a[b[i]]`). Object-backed Array subclasses keep +validated prototype-override reads, and compact guarded specializations are +pre-statepoint inlined by lowered IR size. wolf-ecs (noctjs/ecs-benchmark) on the Mac mini reference box, versus the previous retained build: add/remove -18.5% (0.556 → 0.453 ms/op), entity-cycle diff --git a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs index 489d615b90..27d2964158 100644 --- a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs +++ b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs @@ -746,12 +746,26 @@ fn combined_receiver_and_u31_clone_fuses_property_array_push() { let fast = function_body(&ir, &format!("@{combined}(")); let generic = function_body(&ir, &format!("@{pshape}$generic(")); + // The fused entry is allocate-but-never-reenter and answers null for + // receivers whose push can run user code; the composed clone consumes the + // u31 proof in that one entry on its hot path and keeps the complete + // guarded push only behind the null test, in `apush.u31.generic`. + let (hot, cold) = fast + .split_once("apush.u31.generic") + .expect("the u31 push must carry its null-result fallback block"); assert!( - fast.contains("call i64 @js_array_push_u31_with_length") - && !fast.contains("call void @js_array_push_guard") - && !fast.contains("call i64 @js_array_push_f64") - && !fast.contains("call i32 @js_array_length"), - "the composed clone must consume the u31 proof in one push/length runtime entry:\n{fast}" + hot.contains("call i64 @js_array_push_u31_with_length") + && !hot.contains("call void @js_array_push_guard") + && !hot.contains("call i64 @js_array_push_f64") + && !hot.contains("call i32 @js_array_length"), + "the composed clone must consume the u31 proof in one push/length runtime entry on its hot path:\n{fast}" + ); + assert!( + cold.contains("call void @js_array_push_guard") + && cold.contains("call i64 @js_array_push_f64") + && cold.contains("call i32 @js_array_length") + && !cold.contains("js_array_push_u31_with_length"), + "the null-result fallback must perform the complete guarded push:\n{fast}" ); assert!( generic.contains("call void @js_array_push_guard") diff --git a/crates/perry-codegen/src/collectors/ptr_shape.rs b/crates/perry-codegen/src/collectors/ptr_shape.rs index f0fecb191b..b153919e2b 100644 --- a/crates/perry-codegen/src/collectors/ptr_shape.rs +++ b/crates/perry-codegen/src/collectors/ptr_shape.rs @@ -1445,7 +1445,7 @@ pub(super) struct ThisFlowAnalysis<'a, 'b> { chain: &'b [&'a Class], fields: &'b HashSet, methods: &'b HashMap, - visited: HashSet<(String, String)>, + visited: HashSet<(String, String, bool)>, store_records: Vec>, /// `super(...)` argument lists observed in chain constructors, keyed by /// the PARENT (callee) class name. Feeds the parent-ctor parameter @@ -1553,7 +1553,14 @@ impl<'a, 'b> ThisFlowAnalysis<'a, 'b> { func: &'a perry_hir::Function, allow_terminal_this_return: bool, ) -> bool { - let key = (owner.to_string(), name.to_string()); + // Keyed by the terminal-`this`-return allowance too: the strict + // (`false`) vetting of a nested `this.m()` / `super.m()` edge must not + // be satisfied by an earlier lenient (`true`) visit of the same method. + let key = ( + owner.to_string(), + name.to_string(), + allow_terminal_this_return, + ); if !self.visited.insert(key) { return true; // already vetted (or in-progress higher up the stack) } diff --git a/crates/perry-codegen/src/collectors/ptr_shape_numeric.rs b/crates/perry-codegen/src/collectors/ptr_shape_numeric.rs index c0122386fa..285eaa26c5 100644 --- a/crates/perry-codegen/src/collectors/ptr_shape_numeric.rs +++ b/crates/perry-codegen/src/collectors/ptr_shape_numeric.rs @@ -766,7 +766,7 @@ pub(super) fn expr_provably_not_bigint(e: &Expr, not_bigint_locals: &HashSet { expr_provably_not_bigint(operand, not_bigint_locals) } - _ => true, // !x, typeof x, … never produce BigInt + perry_hir::UnaryOp::Not => true, // `!x` is always a Boolean }, Expr::Binary { .. } => false, // handled structurally by the caller _ => false, diff --git a/crates/perry-codegen/src/expr/compare_tests.rs b/crates/perry-codegen/src/expr/compare_tests.rs index ec8db001a5..53df155181 100644 --- a/crates/perry-codegen/src/expr/compare_tests.rs +++ b/crates/perry-codegen/src/expr/compare_tests.rs @@ -728,7 +728,7 @@ fn local_typeof_number_literal_is_decided_inline_for_plain_doubles() { "the tag-range test must be the single unsigned range compare:\n{ir}" ); assert!( - ir.contains("fcmp olt double") && ir.contains("1048576.0") + (ir.contains("fcmp olt double") && ir.contains("1048576.0")) || ir.contains("0x4130000000000000"), "the stream id band must be excluded inline:\n{ir}" ); diff --git a/crates/perry-codegen/src/expr/property_get/composed_ics.rs b/crates/perry-codegen/src/expr/property_get/composed_ics.rs index 7c65631451..4ae3f0d7a1 100644 --- a/crates/perry-codegen/src/expr/property_get/composed_ics.rs +++ b/crates/perry-codegen/src/expr/property_get/composed_ics.rs @@ -126,13 +126,14 @@ pub(super) fn lower_symbol_then_named_property_ic( let key_box = ctx.block().load(DOUBLE, &key_global); let key_bits = ctx.block().bitcast_double_to_i64(&key_box); let key_handle = ctx.block().and(I64, &key_bits, POINTER_MASK_I64); + let key_ptr = ctx.block().inttoptr(I64, &key_handle); let miss_value = ctx.block().call( DOUBLE, "js_object_get_symbol_then_field_ic_miss", &[ (DOUBLE, &base_box), (DOUBLE, &symbol_box), - (I64, &key_handle), + (PTR, &key_ptr), (I64, &feedback_site_id), (PTR, &symbol_cache), (PTR, &field_cache), diff --git a/crates/perry-codegen/src/lower_call/native/native_instance_branch.rs b/crates/perry-codegen/src/lower_call/native/native_instance_branch.rs index 163b223c3c..e80bfd92a9 100644 --- a/crates/perry-codegen/src/lower_call/native/native_instance_branch.rs +++ b/crates/perry-codegen/src/lower_call/native/native_instance_branch.rs @@ -339,11 +339,43 @@ let orig_handle = arr_handle.clone(); let fused_length_slot = if let Some(value) = u31_value { let length_slot = blk.alloca(I32); - arr_handle = blk.call( + let fast_handle = blk.call( I64, "js_array_push_u31_with_length", &[(I64, &arr_handle), (I32, &value), (PTR, &length_slot)], ); + // The fused entry is allocate-but-never-reenter: it answers null + // for every receiver whose push can run user code (indexed + // descriptors, Proxy traps, foreign families). Those take the same + // complete guarded push the unfused lowering below performs. + let handled = blk.icmp_ne(I64, &fast_handle, "0"); + let fast_end = blk.label.clone(); + let generic_idx = ctx.new_block("apush.u31.generic"); + let merge_idx = ctx.new_block("apush.u31.merge"); + let generic_label = ctx.block_label(generic_idx); + let merge_label = ctx.block_label(merge_idx); + ctx.block().cond_br(&handled, &merge_label, &generic_label); + ctx.current_block = generic_idx; + let generic_handle = { + let blk = ctx.block(); + blk.call_void("js_array_push_guard", &[(I64, &orig_handle)]); + let value_double = blk.sitofp(I32, &value, DOUBLE); + let generic_handle = blk.call( + I64, + "js_array_push_f64", + &[(I64, &orig_handle), (DOUBLE, &value_double)], + ); + let generic_length = blk.call(I32, "js_array_length", &[(I64, &generic_handle)]); + blk.store(I32, &generic_length, &length_slot); + generic_handle + }; + let generic_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + ctx.current_block = merge_idx; + arr_handle = ctx.block().phi( + I64, + &[(&fast_handle, &fast_end), (&generic_handle, &generic_end)], + ); Some(length_slot) } else { // Spec §23.1.3.21: Set(O,"length",…,true) fires unconditionally — guard diff --git a/crates/perry-codegen/src/runtime_decls/strings_part2.rs b/crates/perry-codegen/src/runtime_decls/strings_part2.rs index ad5c80a007..c8d9417808 100644 --- a/crates/perry-codegen/src/runtime_decls/strings_part2.rs +++ b/crates/perry-codegen/src/runtime_decls/strings_part2.rs @@ -317,7 +317,7 @@ pub(crate) fn declare_phase_b_strings_part2(module: &mut LlModule) { module.declare_function( "js_object_get_symbol_then_field_ic_miss", DOUBLE, - &[DOUBLE, DOUBLE, I64, I64, PTR, PTR], + &[DOUBLE, DOUBLE, PTR, I64, PTR, PTR], ); module.add_external_global("PERRY_SYMBOL_PROPERTY_IC_EPOCH", I64); module.declare_function("js_object_create", DOUBLE, &[DOUBLE]); diff --git a/crates/perry-runtime/src/array/push_pop.rs b/crates/perry-runtime/src/array/push_pop.rs index c52f771bab..d8b1e5a83a 100644 --- a/crates/perry-runtime/src/array/push_pop.rs +++ b/crates/perry-runtime/src/array/push_pop.rs @@ -719,10 +719,12 @@ unsafe fn js_array_push_f64_resolved(arr: *mut ArrayHeader, value: f64) -> *mut /// this entry returns the semantic push result through `new_length`, so the /// caller does not immediately redispatch `js_array_length` on the receiver. /// -/// Every unproved receiver state retains the complete public fallback. In -/// particular Proxy traps, descriptor mutations, Array-subclass integrity -/// flags, and first-seen tail transitions all run the same generic algorithms -/// as `js_array_push_f64`. +/// Every unproved receiver state either retains the complete resolved +/// algorithm (Array-subclass integrity flags, first-seen tail transitions, +/// growth) or — for the receivers whose push can run user code (indexed +/// descriptors / prototype indices, Proxy traps, foreign families) — returns +/// null so the generated caller performs the complete public push itself. +/// That is what keeps this symbol allocate-but-never-reenter. #[no_mangle] pub extern "C" fn js_array_push_u31_with_length( arr: *mut ArrayHeader, @@ -781,24 +783,20 @@ pub extern "C" fn js_array_push_u31_with_length( } let cleaned = clean_arr_ptr_mut(arr); - if !cleaned.is_null() { - if crate::array::array_iteration_is_exotic(cleaned) { - let pushed = js_array_push_f64_spec(cleaned, number); - if !new_length.is_null() { - unsafe { *new_length = crate::array::js_array_length(pushed) }; - } - return pushed; - } - let pushed = unsafe { js_array_push_f64_resolved(cleaned, number) }; - if !new_length.is_null() { - unsafe { *new_length = (*pushed).length }; - } - return pushed; - } - - let pushed = js_array_push_f64(arr, number); + if cleaned.is_null() || crate::array::array_iteration_is_exotic(cleaned) { + // Not handled here. An exotic receiver (indexed descriptors, an indexed + // prototype property, a registered buffer / typed-array view) needs the + // observable `Set`, and a receiver the resolver does not own (a Proxy, + // a foreign family) needs the complete public push — both can run user + // code through accessors or traps. This entry is classified + // allocate-but-never-reenter in `gc_call_effects`, so it must never + // reach those paths; the generated caller takes its complete guarded + // push (`js_array_push_guard` + `js_array_push_f64`) on a null result. + return std::ptr::null_mut(); + } + let pushed = unsafe { js_array_push_f64_resolved(cleaned, number) }; if !new_length.is_null() { - unsafe { *new_length = crate::array::js_array_length(pushed) }; + unsafe { *new_length = (*pushed).length }; } pushed } diff --git a/crates/perry-runtime/src/array/subclass_loop_guard.rs b/crates/perry-runtime/src/array/subclass_loop_guard.rs index 8c46af2611..c170048f04 100644 --- a/crates/perry-runtime/src/array/subclass_loop_guard.rs +++ b/crates/perry-runtime/src/array/subclass_loop_guard.rs @@ -230,6 +230,14 @@ pub extern "C" fn js_packed_ecs_u32_loop_guard( common_length = Some(length); addresses[index] = address; } + // The admitted bound (`out[6]`, resolved for the live-length form too) is + // checked against the SOURCE receiver above; the fused loop also reads + // every column up to that bound, so a column shorter than it must decline + // admission rather than read past its payload. + let admitted_bound = unsafe { out.add(6).read() }; + if common_length.is_some_and(|common| u64::from(common) < admitted_bound) { + return 0; + } unsafe { for (index, address) in addresses .iter() diff --git a/crates/perry-runtime/src/array/subclass_tests.rs b/crates/perry-runtime/src/array/subclass_tests.rs index 3356ab895e..ba9d4b3ab5 100644 --- a/crates/perry-runtime/src/array/subclass_tests.rs +++ b/crates/perry-runtime/src/array/subclass_tests.rs @@ -483,6 +483,92 @@ fn fused_u31_push_reports_length_for_plain_and_subclass_arrays() { assert_eq!(returned, obj as *mut ArrayHeader); assert_eq!(length, 1); assert_eq!(array_subclass_fast_index_get(receiver, 0), Some(42.0)); + + // An exotic receiver (here: a typed-array view, which `push` must answer + // through the observable `Set` — a non-writable `length` throws) can run + // user code on the complete path. The fused entry is classified + // allocate-but-never-reenter, so it must decline with null and leave the + // receiver untouched; the generated caller then performs the complete push. + let exotic = + crate::typedarray::js_typed_array_new_empty(crate::typedarray::KIND_UINT32 as i32, 4); + length = u32::MAX; + let declined = js_array_push_u31_with_length(exotic as *mut ArrayHeader, 5, &mut length); + assert!( + declined.is_null(), + "an exotic receiver must be declined to the caller's complete push" + ); + assert_eq!(length, u32::MAX, "a declined push must not report a length"); + assert_eq!( + unsafe { (*exotic).length }, + 4, + "a declined push must not mutate the receiver" + ); +} + +/// `cache_carrier` follows live transition-cache occupancy: an inserted edge +/// marks both of its descriptors, and the post-full-trace recompute clears the +/// mark once no live entry names them (eviction / tombstone / test clear). +#[test] +fn transition_cache_carrier_bits_follow_live_occupancy_across_full_trace_recompute() { + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + let class_id = 0x0074_8695; + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let obj = js_object_alloc(class_id, 2); + assert!(!obj.is_null()); + let receiver = crate::value::js_nanbox_pointer(obj as i64); + crate::node_stream::js_array_subclass_init(receiver, 0.0); + let before = unsafe { (*obj).parent_class_id }; + assert_eq!( + js_array_push_f64(obj as *mut ArrayHeader, 11.0), + obj as *mut ArrayHeader + ); + // Pop warms the reverse edge; push it back to learn the forward edge. + assert_eq!(array_subclass_fast_pop(receiver), Some(11.0)); + assert_eq!( + js_array_push_f64(obj as *mut ArrayHeader, 11.0), + obj as *mut ArrayHeader + ); + let after = unsafe { (*obj).parent_class_id }; + assert_ne!(before, after); + let carrier = |id: u32| { + crate::object::shapes::shape_descriptor_by_id(id) + .expect("shape exists") + .cache_carrier + }; + assert!( + carrier(before) && carrier(after), + "descriptors named by a live transition entry must be cache carriers" + ); + + // With no live entry naming them, the recompute releases both. + crate::object::array_tail_transition::test_clear(); + crate::object::array_tail_transition::recompute_cache_carriers_after_full_trace(); + assert!( + !carrier(before) && !carrier(after), + "a full-trace recompute must release descriptors no live entry names" + ); + + // Relearning an edge marks its descriptors again, and a recompute keeps + // them. The cleared cache has no reverse edge for the fast pop, so the + // generic pop runs; it may mint a different predecessor shape, so the + // relearned pair is read back from the object rather than assumed. + assert_eq!( + crate::array::js_array_pop_f64(obj as *mut ArrayHeader), + 11.0 + ); + let relearned_predecessor = unsafe { (*obj).parent_class_id }; + assert_eq!( + js_array_push_f64(obj as *mut ArrayHeader, 11.0), + obj as *mut ArrayHeader + ); + let relearned_successor = unsafe { (*obj).parent_class_id }; + assert_ne!(relearned_predecessor, relearned_successor); + crate::object::array_tail_transition::recompute_cache_carriers_after_full_trace(); + assert!( + carrier(relearned_predecessor) && carrier(relearned_successor), + "a recompute must keep descriptors that a live entry still names" + ); } #[test] @@ -737,7 +823,7 @@ fn empty_array_subclass_named_prefix_token_survives_warm_tail_cycle() { fn dense_array_subclass_tail_cache_preserves_a_1024_shape_lattice() { let _global = crate::gc::global_side_table_test_lock(); crate::object::array_tail_transition::test_clear(); - let class_id = 0x0074_865a; + let class_id = 0x0074_8693; crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); let obj = js_object_alloc(class_id, 2); let receiver = crate::value::js_nanbox_pointer(obj as i64); @@ -821,7 +907,7 @@ fn dense_array_subclass_tail_transition_edges_survive_moving_gc() { crate::gc::gc_register_mutable_root_scanner(crate::object::scan_transition_cache_roots_mut); crate::object::array_tail_transition::test_clear(); - let class_id = 0x0074_8659; + let class_id = 0x0074_8694; crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); let obj = js_object_alloc(class_id, 2); let scope = crate::gc::RuntimeHandleScope::new(); @@ -1036,6 +1122,27 @@ fn fused_ecs_guard_requires_distinct_owning_u32_columns_and_exact_entity_ids() { 0, "non-Uint32 component columns must not borrow the direct clone" ); + // Columns shorter than the admitted bound: the fused loop reads every + // column up to that bound, so admission must be declined even though the + // columns agree with each other. + let tiny_a = + crate::typedarray::js_typed_array_new_empty(crate::typedarray::KIND_UINT32 as i32, 2); + let tiny_b = + crate::typedarray::js_typed_array_new_empty(crate::typedarray::KIND_UINT32 as i32, 2); + assert_eq!( + js_packed_ecs_u32_loop_guard( + receiver_h.get_nanbox_f64(), + 3.0, + crate::value::js_nanbox_pointer(tiny_a as i64), + crate::value::js_nanbox_pointer(tiny_b as i64), + 0.0, + 0.0, + 2, + facts.as_mut_ptr(), + ), + 0, + "a column shorter than the admitted bound must decline the fused loop" + ); assert_eq!( js_packed_ecs_u32_loop_guard( receiver_h.get_nanbox_f64(), diff --git a/crates/perry-runtime/src/gc/dead_owner.rs b/crates/perry-runtime/src/gc/dead_owner.rs index 194ddfda72..f22c6f7bd9 100644 --- a/crates/perry-runtime/src/gc/dead_owner.rs +++ b/crates/perry-runtime/src/gc/dead_owner.rs @@ -211,6 +211,9 @@ pub(super) fn prune_dead_owner_side_tables_post_trace(full_trace: bool) { // ever add notes, so without this the table's root set would grow // monotonically and no keys array would ever be reclaimed again. crate::object::shapes::rotate_old_carrier_epoch_after_full_trace(); + // The same rule for the array-tail transition caches: their carrier + // bits are exact only when rebuilt from live occupancy. + crate::object::array_tail_transition::recompute_cache_carriers_after_full_trace(); } let probe = PostTraceProbe::new(full_trace); fan_out( diff --git a/crates/perry-runtime/src/object/array_tail_transition.rs b/crates/perry-runtime/src/object/array_tail_transition.rs index ee4dbac80d..e168201d50 100644 --- a/crates/perry-runtime/src/object/array_tail_transition.rs +++ b/crates/perry-runtime/src/object/array_tail_transition.rs @@ -189,9 +189,12 @@ fn with_reverse( /// Resolve the transition tables through an Array-subclass receiver after its /// first learned edge. `RuntimeState` is heap allocated and stable until this -/// thread exits, while ObjectHeaders never cross agents (worker inputs are +/// thread exits, and ObjectHeaders never cross agents (worker inputs are /// deep-copied), so the native pointer can move with ObjectMeta without GC -/// tracing or rewriting. +/// tracing or rewriting. The cached pointer is still validated against the +/// CURRENT thread's tables on every use: a pump thread acting on an agent's +/// behalf (Android's UI-thread timer pump) must never reach another thread's +/// mutable tables through a cache the owning thread filled. #[inline(always)] fn object_hot_for_owner( owner: *const crate::object::ObjectHeader, @@ -200,12 +203,12 @@ fn object_hot_for_owner( if !owner.is_null() { let meta = (*owner).meta; if !meta.is_null() { + let hot = &crate::state::state().object_hot; let cached = (*meta).array_tail_object_hot as usize as *const crate::object::ObjectHotTables; - if !cached.is_null() { + if std::ptr::eq(cached, hot) { return &*cached; } - let hot = &crate::state::state().object_hot; // GC_STORE_AUDIT(NATIVE_POINTER): RuntimeState storage, not a // managed heap edge; ObjectMeta's GC descriptors intentionally // visit only prototype, spill, and private brand. @@ -345,16 +348,24 @@ pub(crate) fn record_numeric_tail_transition( { return; } - unsafe { - shapes::note_cache_carrier(Some(predecessor)); - shapes::note_cache_carrier(Some(successor)); - } // A single lost edge poisons the remainder of a shrinking dense shape // chain: the generic fallback mints a different predecessor, after which // no historical edge can match. Preserve colliding entries with bounded // open addressing instead of overwriting one direct-mapped slot. let forward = with_forward_for_owner(owner, |table| unsafe { insert_forward(table, entry) }); let reverse = with_reverse_for_owner(owner, |table| unsafe { insert_reverse(table, entry) }); + if forward.is_none() && reverse.is_none() { + // Nothing references the pair: it must not claim cache ownership. + return; + } + // Both descriptors are now named by a live entry. The bit is recomputed + // from table occupancy after every full trace + // (`recompute_cache_carriers_after_full_trace`), so an entry that is later + // evicted or tombstoned releases its descriptors at the next full trace. + unsafe { + shapes::note_cache_carrier(Some(predecessor)); + shapes::note_cache_carrier(Some(successor)); + } if let Some(index) = forward { publish_direct_index(owner, predecessor_shape_id, Some(index), None); } @@ -468,6 +479,35 @@ unsafe fn prune_table(table: *mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITI } } +/// Rebuild the `cache_carrier` gate from live cache occupancy. +/// +/// `note_cache_carrier` marks a descriptor when an entry naming it is inserted; +/// eviction and tombstoning do not unmark it, because several entries may name +/// the same descriptor. A full trace is the one point where the exact answer is +/// cheap: clear every bit, then re-note the descriptors of every live entry in +/// both tables. Between full traces the bit set is therefore a superset of the +/// live occupancy — never a subset — which is the direction the rooting gate in +/// `scan_shape_table_rekey_mut` requires. Mirrors +/// `rotate_old_carrier_epoch_after_full_trace` for the other carrier class. +pub(crate) fn recompute_cache_carriers_after_full_trace() { + shapes::clear_all_cache_carriers(); + with_forward(|table| unsafe { note_live_entries(table) }); + with_reverse(|table| unsafe { note_live_entries(table) }); +} + +unsafe fn note_live_entries( + table: *mut [ArrayTailTransitionEntry; ARRAY_TAIL_TRANSITION_CACHE_SIZE], +) { + for entry in (*table).iter() { + // `EMPTY` and `TOMBSTONE` both carry a zero successor ShapeId. + if entry.successor_shape_id == 0 { + continue; + } + shapes::note_cache_carrier(shapes::shape_descriptor_by_id(entry.predecessor_shape_id)); + shapes::note_cache_carrier(shapes::shape_descriptor_by_id(entry.successor_shape_id)); + } +} + #[cold] pub(crate) fn prune_invalid_entries() { with_forward(|table| unsafe { prune_table(table) }); diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 747454af89..fc4a20ca57 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -600,9 +600,12 @@ pub(crate) unsafe fn note_old_generation_carrier(descriptor: Option) { let Some(descriptor) = descriptor else { @@ -616,6 +619,14 @@ pub(crate) unsafe fn note_cache_carrier(descriptor: Option) { (*record).cache_carrier = true; } +/// Clear every `cache_carrier` bit ahead of the post-full-trace recompute. +pub(crate) fn clear_all_cache_carriers() { + let mut inner = crate::state::state().shapes.inner.borrow_mut(); + for record in inner.descriptors.values_mut() { + record.cache_carrier = false; + } +} + /// Recompute the old-carrier gate from the trace that just finished. /// /// A FULL trace enumerates every live object, so the notes it accumulated are From f2e2c8d609a00fb6921281d76f2b7b6a5b9acc75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 16:25:39 +0200 Subject: [PATCH 14/15] perf(shapes): drop SipHash from ids_by_facts, the last one on the shape path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ids_by_facts was the only shape-table map still on std's RandomState. Profiling `claude -p` showed RandomState::hash_one at 17 self-samples inside shapes:: alone (57 across the process) — pure hashing overhead on a lookup that runs on every descriptor install and retire. Its sibling maps already moved off SipHash (#8125). The standing comment argued only against PtrHasher, whose write_* methods OVERWRITE the accumulator — correct for a single-word key, and wrong for this five-field one, which would collapse to its last field. That objection does not apply to FastKeyHasher: it implements only `write`, so the derived Hash's write_u32/write_u64 calls all forward there and FOLD with FNV-1a, reaching every field. The key is internal shape state, never program input, so DoS-resistant hashing buys nothing — the same rationale already applied to the descriptor side tables. Test pins the folding property by varying one field at a time and requiring a distinct hash. Sabotage-checked against PtrHasher: it fails with 'changing keys alone must change the hash'. Suite 2717 passed. --- crates/perry-runtime/src/object/shapes.rs | 20 ++++- .../perry-runtime/src/object/shapes_tests.rs | 84 +++++++++++++++++++ 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index d348857534..09c6a8be28 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -167,7 +167,23 @@ struct ShapeTableInner { /// OVERWRITE the accumulator instead of folding it, which is exactly right /// for a single-word key and wrong for this five-field one — every /// `ShapeFacts` would hash to its last field alone. - ids_by_facts: HashMap>, + /// + /// It is a `FastKeyHashMap` rather than the SipHash default, though: that + /// objection is to `PtrHasher` specifically, and leaving std's + /// `RandomState` here made this the only SipHash map left on the shape + /// path. Profiling `claude -p` showed `RandomState::hash_one` at 17 + /// self-samples inside `shapes::` alone (57 across the process) — pure + /// hashing overhead on a lookup that runs on every descriptor + /// install/retire. + /// + /// `FastKeyHasher` is the right third option: it implements only `write`, + /// so every `write_u32` / `write_u64` from the derived `Hash` forwards + /// there and FOLDS with FNV-1a. All five fields reach the accumulator, + /// which is exactly the property `PtrHasher` lacks. The key is built from + /// internal shape state (never program input), so DoS-resistant hashing + /// buys nothing here — the same rationale already applied to the + /// descriptor side tables and to `indices` (#8125). + ids_by_facts: crate::fast_hash::FastKeyHashMap>, /// Keys-array address -> every descriptor id that currently names it. /// Same-address key-count retirement uses this index instead of scanning /// every shape ever observed by the agent. Single-word key, so `PtrHasher` @@ -185,7 +201,7 @@ impl ShapeTable { inner: RefCell::new(ShapeTableInner { indices: crate::fast_hash::new_ptr_hash_map(), descriptors: crate::fast_hash::new_ptr_hash_map(), - ids_by_facts: HashMap::new(), + ids_by_facts: crate::fast_hash::new_fast_key_hash_map(), ids_by_keys: crate::fast_hash::new_ptr_hash_map(), }), } diff --git a/crates/perry-runtime/src/object/shapes_tests.rs b/crates/perry-runtime/src/object/shapes_tests.rs index 30ea3ea124..8bf3279bb4 100644 --- a/crates/perry-runtime/src/object/shapes_tests.rs +++ b/crates/perry-runtime/src/object/shapes_tests.rs @@ -776,3 +776,87 @@ mod descriptor_tests_8067 { } } } + +/// `ids_by_facts` moved from std's SipHash `RandomState` to `FastKeyHasher`. +/// +/// The hazard that motivated the original "deliberately NOT a `PtrHashMap`" +/// note is real: `PtrHasher`'s `write_*` methods OVERWRITE the accumulator, so +/// a five-field `ShapeFacts` would collapse to its last field and every +/// descriptor sharing that field would collide into one bucket. +/// +/// `FastKeyHasher` avoids this by implementing only `write` — the derived +/// `Hash`'s `write_u32`/`write_u64` calls all forward there and FOLD with +/// FNV-1a. This test pins that property directly: vary ONE field at a time and +/// require a distinct hash each time. It fails loudly against any hasher that +/// overwrites instead of folding. +#[test] +fn shape_facts_hash_folds_every_field() { + use crate::fast_hash::FastKeyHasher; + use std::hash::{BuildHasher, Hash, Hasher}; + + fn h(f: &ShapeFacts) -> u64 { + let mut hasher = FastKeyHasher.build_hasher(); + f.hash(&mut hasher); + hasher.finish() + } + + let base = ShapeFacts { + keys: 0x1111_2222_3333_4444, + logical_key_count: 7, + live_inline_slot_count: 3, + semantic_generation: 9, + object_kind: ShapeObjectKind::Ordinary, + }; + + let variants = [ + ( + "keys", + ShapeFacts { + keys: 0x5555_6666_7777_8888, + ..base + }, + ), + ( + "logical_key_count", + ShapeFacts { + logical_key_count: 8, + ..base + }, + ), + ( + "live_inline_slot_count", + ShapeFacts { + live_inline_slot_count: 4, + ..base + }, + ), + ( + "semantic_generation", + ShapeFacts { + semantic_generation: 10, + ..base + }, + ), + ( + "object_kind", + ShapeFacts { + object_kind: ShapeObjectKind::Class, + ..base + }, + ), + ]; + + let base_hash = h(&base); + for (field, v) in &variants { + assert_ne!( + h(v), + base_hash, + "changing `{field}` alone must change the hash — a hasher that \ + overwrites instead of folding would collapse ShapeFacts to its \ + last field and collide every descriptor that shares it" + ); + } + + // Same facts must still hash the same, or lookups would miss. + assert_eq!(h(&base), h(&base.clone()), "hashing must be deterministic"); +} From 2c06e3f60cb2d5c5422b4f734377a6fc9cd36138 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 16:37:17 +0200 Subject: [PATCH 15/15] changelog: add fragment for the ids_by_facts hasher change --- changelog.d/8882-shape-facts-hasher.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 changelog.d/8882-shape-facts-hasher.md diff --git a/changelog.d/8882-shape-facts-hasher.md b/changelog.d/8882-shape-facts-hasher.md new file mode 100644 index 0000000000..50629d57ac --- /dev/null +++ b/changelog.d/8882-shape-facts-hasher.md @@ -0,0 +1,20 @@ +Dropped SipHash from `ids_by_facts`, the last shape-table map still using it. + +Profiling `claude -p` put `RandomState::hash_one` at 17 self-samples inside +`shapes::` alone (57 across the process) — pure hashing overhead on a lookup +that runs on every descriptor install and retire. + +Its sibling maps already moved off SipHash (#8125). The standing comment on +this one argued only against `PtrHasher`, whose `write_*` methods OVERWRITE the +accumulator — right for a single-word key, wrong for this five-field one, which +would collapse to its last field and collide every descriptor sharing it. That +objection does not apply to `FastKeyHasher`: it implements only `write`, so the +derived `Hash`'s `write_u32` / `write_u64` calls all forward there and FOLD with +FNV-1a, reaching every field. + +The key is internal shape state, never program input, so DoS-resistant hashing +buys nothing — the same rationale already applied to the descriptor side tables. + +The new test pins the folding property directly: vary one field at a time and +require a distinct hash each time. Sabotage-checked against `PtrHasher`, where +it fails with "changing `keys` alone must change the hash".