From 731513e8824f91407b737fb42ebd15fa7744ed58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 6 Sep 2026 22:15:49 +0200 Subject: [PATCH 01/27] perf(regex): remove the traced-source side table (cherry picked from commit d8daa4fd42e02bc1c8315abd6a1572255606271b) --- .../gc/tests/copying/survival_and_malloc.rs | 6 - crates/perry-runtime/src/hot_diag.rs | 26 +++- crates/perry-runtime/src/regex.rs | 147 +++++++----------- crates/perry-runtime/src/regex/compile.rs | 21 ++- crates/perry-runtime/src/regex/escape.rs | 79 +++++++++- crates/perry-runtime/src/regex/lazy.rs | 16 +- crates/perry-runtime/src/regex/program_key.rs | 5 +- crates/perry-runtime/src/regex/site_cache.rs | 9 +- crates/perry-runtime/src/regex/tests_part2.rs | 70 +++++++-- 9 files changed, 232 insertions(+), 147 deletions(-) diff --git a/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs b/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs index b6104a63f5..91cdf84d8f 100644 --- a/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs +++ b/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs @@ -930,7 +930,6 @@ fn test_movable_regexp_evacuation_migrates_all_address_owned_state() { let old_addr = re as usize; assert!(crate::arena::pointer_in_nursery(old_addr)); assert!(crate::regex::test_regex_pointer_entry_exists(old_addr)); - assert!(crate::regex::test_regex_source_entry_exists(old_addr)); crate::object::exotic_expando::test_seed_exotic_expando_entry( old_addr, @@ -948,8 +947,6 @@ fn test_movable_regexp_evacuation_migrates_all_address_owned_state() { assert!(crate::regex::test_regex_pointer_entry_exists(new_addr)); assert!(!crate::regex::test_regex_pointer_entry_exists(old_addr)); - assert!(crate::regex::test_regex_source_entry_exists(new_addr)); - assert!(!crate::regex::test_regex_source_entry_exists(old_addr)); assert!(crate::object::exotic_expando::test_exotic_expando_entry_exists(new_addr)); assert!(!crate::object::exotic_expando::test_exotic_expando_entry_exists(old_addr)); @@ -1045,7 +1042,6 @@ fn nursery_regexp_that_dies_young_is_finalized_by_the_copied_minor() { "the header must be nursery-allocated" ); assert!(crate::regex::test_regex_pointer_entry_exists(dead_addr)); - assert!(crate::regex::test_regex_source_entry_exists(dead_addr)); // Both headers share one program through the site cache. let count_before = crate::regex::test_regexp_std_program_strong_count(live); assert!(count_before >= 2); @@ -1059,13 +1055,11 @@ fn nursery_regexp_that_dies_young_is_finalized_by_the_copied_minor() { assert_ne!(live_new, live_addr, "the rooted RegExp must be evacuated"); assert!(crate::regex::regex_header_has_magic(live_new as *const _)); assert!(crate::regex::test_regex_pointer_entry_exists(live_new)); - assert!(crate::regex::test_regex_source_entry_exists(live_new)); assert!( !crate::regex::test_regex_pointer_entry_exists(dead_addr), "a nursery RegExp that died must be removed from REGEX_POINTERS by the copied minor" ); - assert!(!crate::regex::test_regex_source_entry_exists(dead_addr)); assert_eq!( crate::regex::test_regexp_std_program_strong_count(live_new as *const _), count_before - 1, diff --git a/crates/perry-runtime/src/hot_diag.rs b/crates/perry-runtime/src/hot_diag.rs index 4695e6743b..dcfd44a6dc 100644 --- a/crates/perry-runtime/src/hot_diag.rs +++ b/crates/perry-runtime/src/hot_diag.rs @@ -179,11 +179,21 @@ pub struct RegexDiag { /// or missed: this is the `memcmp` volume alone, which is what a 12 KB /// emoji pattern makes expensive and a 60-byte one does not. pub new_site_verify_bytes: u64, - /// Address-keyed side-table inserts performed per construction - /// (`REGEX_POINTERS` and `REGEX_SOURCE_TABLE`) — two per header, each a - /// `PtrHasher` hash plus a hashbrown insert, mirrored by two removals at - /// death and two rekeys per evacuation. + /// Address-keyed side-table inserts performed per construction. This was + /// two (`REGEX_POINTERS` plus the source table) before the header's string + /// slots became traced edges; only `REGEX_POINTERS` remains. pub new_side_table_inserts: u64, + /// Split of the above by table. The source counters are retained as zeroed + /// before/after controls for the #9908 measurement; `REGEX_POINTERS` is + /// still the registry the copied-minor finaliser enumerates. + pub pointer_table_inserts: u64, + pub source_table_inserts: u64, + /// The death side. `source_table_removals` is the zeroed after-control; + /// `regex_header_clear_dead_for_gc` now removes only `REGEX_POINTERS`. + pub pointer_table_removals: u64, + pub source_table_removals: u64, + /// Evacuation rekeys of the remaining pointer registry. + pub side_table_rekeys: u64, /// Constructions answered from the LITERAL-SITE table — identity by the /// compiler-emitted site global's address, so neither the pattern's /// fingerprint nor its byte compare ran. `site_hit` counts the @@ -336,7 +346,8 @@ impl RegexDiag { match={} replace={} replace_matches={} split={} flags_alloc={} \ desc_regexp_probes={} desc_regexp_meta_negative={} \ barrier_taken={} barrier_gated={} header_bytes={} site_verify_bytes={} \ - side_table_inserts={} site_key_hit={}", + side_table_inserts={} site_key_hit={} ptr_ins={} src_ins={} \ + ptr_rm={} src_rm={} rekeys={}", self.new_calls, self.new_validated_hit, self.new_site_hit, @@ -366,6 +377,11 @@ impl RegexDiag { self.new_site_verify_bytes, self.new_side_table_inserts, self.new_site_key_hit, + self.pointer_table_inserts, + self.source_table_inserts, + self.pointer_table_removals, + self.source_table_removals, + self.side_table_rekeys, ); // Merge by content (prefix, len, flags): distinct literal sites with // the same pattern are one row. diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 2a7c5461e0..5d8aa2db2f 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -145,21 +145,6 @@ crate::perry_thread_local! { /// relocate or die. Header magic remains the primary identity check. static REGEX_POINTERS: RefCell> = RefCell::new(crate::fast_hash::new_ptr_hash_set()); - /// Issue #637: Owned copies of pattern and flags strings keyed by - /// the RegExpHeader pointer. The header's `pattern_ptr` / `flags_ptr` - /// fields hold raw `*const StringHeader` pointers to the input - /// strings — when those inputs are temporaries (e.g. the result of - /// a template-literal expression `\`^${p}\``), the GC frees them - /// after the function call returns and subsequent `.source` / - /// `.flags` reads dereference dangling memory. We side-table an - /// owned `String` copy at construction time; readers prefer this - /// over `pattern_ptr` whenever an entry exists. - /// - /// The copies are `Arc` shared with `regex::site_cache`: every - /// header built from the same literal text bumps two refcounts instead - /// of copying the pattern (12 KB for emoji-class patterns, once per - /// evaluation of the literal). - static REGEX_SOURCE_TABLE: RefCell, Arc)>> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); } /// Check whether `ptr` is a RegExpHeader pointer that was allocated in @@ -206,29 +191,33 @@ pub(crate) fn regex_header_moved_for_gc(old_addr: usize, new_addr: usize) { if old_addr == new_addr { return; } + if crate::hot_diag::regex_on() { + crate::hot_diag::regex_counters(|d| d.side_table_rekeys += 1); + } REGEX_POINTERS.with(|table| { let mut table = table.borrow_mut(); if table.remove(&old_addr) { table.insert(new_addr); } }); - REGEX_SOURCE_TABLE.with(|table| { - let mut table = table.borrow_mut(); - if let Some(source) = table.remove(&old_addr) { - table.insert(new_addr, source); - } - }); crate::object::exotic_expando::exotic_expando_owner_moved(old_addr, new_addr); } /// Remove address-owned RegExp metadata when the cell is proven dead. pub(crate) fn regex_header_clear_dead_for_gc(addr: usize) { + // Counted, not timed: this runs inside a collection, so a probe here must + // allocate nothing and must not dump. `regex_counters` does neither, and + // `regex_on`'s one-time env read cannot first happen here — a header can + // only die after `js_regexp_new` created it, and that path arms the + // instrument first. + if crate::hot_diag::regex_on() { + crate::hot_diag::regex_counters(|d| { + d.pointer_table_removals += 1; + }); + } REGEX_POINTERS.with(|table| { table.borrow_mut().remove(&addr); }); - REGEX_SOURCE_TABLE.with(|table| { - table.borrow_mut().remove(&addr); - }); crate::object::exotic_expando::exotic_expando_owner_clear_dead(addr); } @@ -271,7 +260,7 @@ pub(crate) unsafe fn regex_header_finalize_for_gc(re: *mut RegExpHeader) { /// /// The copying minor's from-space flip runs no per-object finalize hooks, so /// a nursery header that was neither evacuated nor pinned would otherwise keep -/// its `Arc` programs and its `REGEX_POINTERS` / `REGEX_SOURCE_TABLE` / expando +/// its `Arc` programs and its `REGEX_POINTERS` / expando /// entries forever. Same shape as `map::finalize_dead_copied_minor_from_space_maps`: /// walk the registry after the flip, collect the provably-dead addresses, then /// finalize each (the finalizer removes its own registry entries, which is why @@ -386,11 +375,6 @@ pub(crate) fn test_regex_pointer_entry_exists(addr: usize) -> bool { REGEX_POINTERS.with(|table| table.borrow().contains(&addr)) } -#[cfg(test)] -pub(crate) fn test_regex_source_entry_exists(addr: usize) -> bool { - REGEX_SOURCE_TABLE.with(|table| table.borrow().contains_key(&addr)) -} - /// Build a minimal nursery-resident RegExp payload for the copying collector's /// relocation contract test. Production construction currently chooses the /// malloc-backed arm of `ArenaOrMalloc`; this exercises the same registered GC @@ -398,6 +382,9 @@ pub(crate) fn test_regex_source_entry_exists(addr: usize) -> bool { /// strand the address-owned tables. #[cfg(all(test, feature = "regex-engine"))] pub(crate) fn test_alloc_nursery_regexp_for_move(source: &str, flags: &str) -> *mut RegExpHeader { + let scope = crate::gc::RuntimeHandleScope::new(); + let pattern = scope.root_string_ptr(js_string_from_str(source)); + let flags_string = scope.root_string_ptr(js_string_from_str(flags)); unsafe { let ptr = crate::arena::arena_alloc_gc( std::mem::size_of::(), @@ -408,8 +395,8 @@ pub(crate) fn test_alloc_nursery_regexp_for_move(source: &str, flags: &str) -> * // must be set explicitly or the GC follows a garbage pointer. (*ptr).meta = std::ptr::null_mut(); (*ptr).regex_ptr = std::ptr::null_mut(); - (*ptr).pattern_ptr = std::ptr::null(); - (*ptr).flags_ptr = std::ptr::null(); + (*ptr).pattern_ptr = pattern.get_raw_const_ptr::(); + (*ptr).flags_ptr = flags_string.get_raw_const_ptr::(); (*ptr).case_insensitive = flags.contains('i'); (*ptr).global = flags.contains('g'); (*ptr).multiline = flags.contains('m'); @@ -426,11 +413,6 @@ pub(crate) fn test_alloc_nursery_regexp_for_move(source: &str, flags: &str) -> * REGEX_POINTERS.with(|table| { table.borrow_mut().insert(ptr as usize); }); - REGEX_SOURCE_TABLE.with(|table| { - table - .borrow_mut() - .insert(ptr as usize, (Arc::from(source), Arc::from(flags))); - }); ptr } } @@ -811,7 +793,7 @@ fn js_regexp_new_impl( // A `site_key` of 0 (every dynamic construction, and every runtime caller) // misses by construction and takes the content-keyed path below unchanged. let site_entry = site_key::lookup(site_key, raw_flags_str); - let (owned_pattern, owned_flags, programs, bits, shared_flags_root) = match site_entry { + let (programs, bits, shared_flags_root) = match site_entry { Some(hit) => { // The site's own flags literal, so this is the same sharing // decision the first construction at this site made (#9819). @@ -845,13 +827,7 @@ fn js_regexp_new_impl( picked } }; - ( - hit.pattern, - hit.flags, - programs, - hit.bits, - shared_flags_root, - ) + (programs, hit.bits, shared_flags_root) } None => { let pattern_str = if is_valid_ptr(pattern) { @@ -1038,10 +1014,10 @@ fn js_regexp_new_impl( // ★ Last use of the borrowed pattern text before this function allocates. // `pattern_str` borrows the GC string; the two allocations below can move - // it, and everything after this point reads the pattern from `owned_pattern` - // (a shared `Arc`, which relocation cannot invalidate) or from - // `pattern_root` (a runtime handle the collector rewrites). Nothing below - // may use `pattern_str` or the incoming `pattern` argument again. + // it. The site/content cache snapshots it into `owned_pattern`, and + // the header store below re-reads it from `pattern_root` (a runtime + // handle the collector rewrites). Nothing below may use `pattern_str` + // or the incoming `pattern` argument again. let (owned_pattern, owned_flags, programs) = match site_hit { Some(hit) => (hit.pattern, hit.flags, hit.programs), None => { @@ -1069,19 +1045,13 @@ fn js_regexp_new_impl( site_key::record( site_key, raw_flags_owned, - owned_pattern.clone(), - owned_flags.clone(), + owned_pattern, + owned_flags, flags_are_canonical, bits, programs.clone(), ); - ( - owned_pattern, - owned_flags, - programs, - bits, - shared_flags_root, - ) + (programs, bits, shared_flags_root) } }; let site_key::FlagBits { @@ -1110,7 +1080,7 @@ fn js_regexp_new_impl( // old-generation prices to do it. // // `GC_TYPE_REGEXP` has been movable (`GcMoveHookKind::RegExpSideTables` - // rekeys `REGEX_POINTERS`, `REGEX_SOURCE_TABLE` and the expando owner + // rekeys `REGEX_POINTERS` and the expando owner // after evacuation; `GcLayoutSlotKind::RegExpFields` traces the two string // edges and `meta`) since the copying collector landed, and // `test_movable_regexp_evacuation_migrates_all_address_owned_state` has @@ -1125,8 +1095,8 @@ fn js_regexp_new_impl( // old-generation sweep's ordinary `gc_type_finalize_unmarked_payload`. let header_size = std::mem::size_of::(); // `flags_ptr` must hold the CANONICAL form, so that `flags_ptr`-keyed - // lookups (FANCY_CACHE, lookup_fancy_regex) and the GC-survivable source - // table all agree. When the caller's string already is that text it is + // lookups (FANCY_CACHE, lookup_fancy_regex) agree. When the caller's + // string already is that text it is // shared (rooted above); only a non-canonical spelling (`/x/ig` → `"gi"`, // or a computed `new RegExp(p, f)`) still has to materialize one. The // counter makes the removal provable rather than asserted. @@ -1292,21 +1262,16 @@ fn js_regexp_new_impl( s.borrow_mut().insert(ptr as usize); }); if crate::hot_diag::regex_on() { - // Two address-keyed inserts per construction (this one and - // `REGEX_SOURCE_TABLE` below), each a `PtrHasher` hash plus a - // hashbrown insert, mirrored by two removals at death and two - // rekeys per evacuation. Counted so the pair is a number rather - // than a reading of the profile. - crate::hot_diag::regex_counters(|d| d.new_side_table_inserts += 2); + // One address-keyed insert per construction. `REGEX_POINTERS` + // remains because the copied-minor finaliser enumerates it; the + // former source table became redundant when #9845 made the + // header's two string slots traced GC edges. + crate::hot_diag::regex_counters(|d| { + d.new_side_table_inserts += 1; + d.pointer_table_inserts += 1; + }); } - // Issue #637: side-table owned copies of pattern + flags so - // `.source` / `.flags` survive GC of the input StringHeaders. - REGEX_SOURCE_TABLE.with(|t| { - t.borrow_mut() - .insert(ptr as usize, (owned_pattern, owned_flags)); - }); - ptr } } @@ -1335,10 +1300,18 @@ pub extern "C" fn js_regexp_construct(pattern: f64, flags: f64) -> *mut RegExpHe let (source_string, inherited_flags) = if pattern_is_regex { let re = pv.as_pointer::(); - let entry = REGEX_SOURCE_TABLE.with(|t| t.borrow().get(&(re as usize)).cloned()); - match entry { - Some((pat, fl)) => (pat.to_string(), Some(fl.to_string())), - None => (String::new(), Some(String::new())), + unsafe { + let source = if is_valid_ptr((*re).pattern_ptr) { + string_as_str((*re).pattern_ptr).to_string() + } else { + String::new() + }; + let inherited = if is_valid_ptr((*re).flags_ptr) { + string_as_str((*re).flags_ptr).to_string() + } else { + String::new() + }; + (source, Some(inherited)) } } else if pv.is_undefined() { (String::new(), None) @@ -1810,18 +1783,10 @@ pub extern "C" fn js_regexp_get_source(re: *const RegExpHeader) -> *mut StringHe if !is_valid_regex_ptr(re) { return js_string_from_str("(?:)"); } - // Issue #637: prefer the side-tabled owned copy so we survive GC - // of the input StringHeader (e.g. template-literal temporary). - if let Some(pat) = - REGEX_SOURCE_TABLE.with(|t| t.borrow().get(&(re as usize)).map(|(p, _)| p.clone())) - { - return js_string_from_str(&escape_regexp_source(&pat)); - } unsafe { if is_valid_ptr((*re).pattern_ptr) { - // Return a copy of the pattern string - let pattern_str = string_as_str((*re).pattern_ptr); - js_string_from_str(&escape_regexp_source(pattern_str)) + let escaped = escape_regexp_source(string_as_bytes((*re).pattern_ptr)); + crate::string::js_string_from_wtf8_bytes(escaped.as_ptr(), escaped.len() as u32) } else { js_string_from_str("(?:)") } @@ -1841,12 +1806,6 @@ pub extern "C" fn js_regexp_get_flags(re: *const RegExpHeader) -> *mut StringHea if !is_valid_regex_ptr(re) { return js_string_from_str(""); } - // Issue #637: prefer the side-tabled owned copy. - if let Some(flags) = - REGEX_SOURCE_TABLE.with(|t| t.borrow().get(&(re as usize)).map(|(_, f)| f.clone())) - { - return js_string_from_str(&flags); - } unsafe { if is_valid_ptr((*re).flags_ptr) { let flags_str = string_as_str((*re).flags_ptr); diff --git a/crates/perry-runtime/src/regex/compile.rs b/crates/perry-runtime/src/regex/compile.rs index f8472f50d6..865fdfd02b 100644 --- a/crates/perry-runtime/src/regex/compile.rs +++ b/crates/perry-runtime/src/regex/compile.rs @@ -152,8 +152,7 @@ pub extern "C" fn js_regexp_compile_value( // the one the receiver was constructed with. // `RegExp.prototype.compile` re-initialises an existing receiver — once per // call from user code, not per object — so materialising the shared key - // here costs nothing measurable, and the same `Arc`s go into the source - // table below. + // here costs nothing measurable. let pattern_key: std::sync::Arc = std::sync::Arc::from(pattern_str); let flags_key: std::sync::Arc = std::sync::Arc::from(flags_str); let arc = get_or_compile_regex(&pattern_key, &flags_key); @@ -203,6 +202,20 @@ pub extern "C" fn js_regexp_compile_value( } (*re).pattern_ptr = pattern_ptr; (*re).flags_ptr = canonical_flags_ptr; + // These are traced header edges. Unlike construction, `compile` can + // rewrite a tenured receiver with newly allocated nursery strings, so + // both stores need the ordinary runtime barrier. + let parent = re as usize; + crate::gc::runtime_write_barrier_gc_slot( + parent, + std::ptr::addr_of!((*re).pattern_ptr) as usize, + crate::value::js_nanbox_string(pattern_ptr as i64).to_bits(), + ); + crate::gc::runtime_write_barrier_gc_slot( + parent, + std::ptr::addr_of!((*re).flags_ptr) as usize, + crate::value::js_nanbox_string(canonical_flags_ptr as i64).to_bits(), + ); (*re).case_insensitive = flags_str.contains('i'); (*re).global = flags_str.contains('g'); (*re).multiline = flags_str.contains('m'); @@ -210,10 +223,6 @@ pub extern "C" fn js_regexp_compile_value( (*re).dot_all = flags_str.contains('s'); (*re).unicode = flags_str.contains('u') || flags_str.contains('v'); (*re).has_indices = flags_str.contains('d'); - super::REGEX_SOURCE_TABLE.with(|t| { - t.borrow_mut() - .insert(re as usize, (Arc::from(pattern_str), Arc::from(flags_str))); - }); } // Spec RegExpInitialize step 12: `Set(obj, "lastIndex", 0, true)` runs LAST, // with the *Throw* flag. A user-frozen `lastIndex` diff --git a/crates/perry-runtime/src/regex/escape.rs b/crates/perry-runtime/src/regex/escape.rs index 0a3b88acc4..6f1cfc2417 100644 --- a/crates/perry-runtime/src/regex/escape.rs +++ b/crates/perry-runtime/src/regex/escape.rs @@ -143,13 +143,8 @@ pub extern "C" fn js_regexp_escape(input: f64) -> f64 { #[used] static KEEP_REGEXP_ESCAPE: extern "C" fn(f64) -> f64 = js_regexp_escape; -/// ECMA-262 22.2.6.10 EscapeRegExpPattern: produce a string that, placed -/// between two `/` characters, parses as the same pattern. An empty pattern -/// becomes `"(?:)"`; an unescaped `/` outside a character class becomes `\/`; -/// the four LineTerminators become their `\n`/`\r`/`
`/`
` escapes -/// (even inside a character class). A backslash escapes the following code -/// point, which is copied verbatim. -pub(super) fn escape_regexp_source(pattern: &str) -> String { +/// ECMA-262 22.2.6.10 EscapeRegExpPattern for a valid UTF-8 pattern. +fn escape_regexp_source_utf8(pattern: &str) -> String { if pattern.is_empty() { return "(?:)".to_string(); } @@ -183,3 +178,73 @@ pub(super) fn escape_regexp_source(pattern: &str) -> String { } out } + +/// ECMA-262 22.2.6.10 EscapeRegExpPattern: produce WTF-8 bytes that, placed +/// between two `/` characters, parse as the same pattern. JavaScript strings +/// may contain lone UTF-16 surrogates, represented by Perry as WTF-8; those +/// bytes must round-trip rather than pass through Rust's `str::chars()`. +pub(super) fn escape_regexp_source(pattern: &[u8]) -> Vec { + if let Ok(pattern) = std::str::from_utf8(pattern) { + return escape_regexp_source_utf8(pattern).into_bytes(); + } + if pattern.is_empty() { + return b"(?:)".to_vec(); + } + + let mut out = Vec::with_capacity(pattern.len() + 2); + let mut in_class = false; + let mut i = 0; + while i < pattern.len() { + match pattern[i] { + b'\\' => { + out.push(b'\\'); + i += 1; + if i < pattern.len() { + let (advance, _, _) = crate::string::wtf8_step(pattern, i); + let end = i.saturating_add(advance).min(pattern.len()); + out.extend_from_slice(&pattern[i..end]); + i = end; + } + } + b'[' if !in_class => { + in_class = true; + out.push(b'['); + i += 1; + } + b']' if in_class => { + in_class = false; + out.push(b']'); + i += 1; + } + b'/' if !in_class => { + out.extend_from_slice(b"\\/"); + i += 1; + } + b'\n' => { + out.extend_from_slice(b"\\n"); + i += 1; + } + b'\r' => { + out.extend_from_slice(b"\\r"); + i += 1; + } + 0xE2 if pattern.get(i..i + 3) == Some(&[0xE2, 0x80, 0xA8]) + || pattern.get(i..i + 3) == Some(&[0xE2, 0x80, 0xA9]) => + { + out.extend_from_slice(if pattern[i + 2] == 0xA8 { + b"\\u2028" + } else { + b"\\u2029" + }); + i += 3; + } + _ => { + let (advance, _, _) = crate::string::wtf8_step(pattern, i); + let end = i.saturating_add(advance).min(pattern.len()); + out.extend_from_slice(&pattern[i..end]); + i = end; + } + } + } + out +} diff --git a/crates/perry-runtime/src/regex/lazy.rs b/crates/perry-runtime/src/regex/lazy.rs index f5492c317f..eba7213bbd 100644 --- a/crates/perry-runtime/src/regex/lazy.rs +++ b/crates/perry-runtime/src/regex/lazy.rs @@ -37,7 +37,7 @@ //! lookbehind/backreferences still decides, and still throws when both //! engines refuse); //! * `.source` / `.flags` / `.global` / `.sticky` / `lastIndex` are header -//! and side-table reads that never touched the compiled program; +//! reads that never touched the compiled program; //! * identity is untouched — `js_regexp_new` still allocates a fresh header //! per evaluation. //! @@ -53,8 +53,7 @@ use regex::Regex; use super::grammar::{collapse_redos_guard_quantifiers, js_regex_to_rust_with_flags}; use super::{ evict_regex_cache_if_full, get_or_compile_regex, is_valid_ptr, is_valid_regex_ptr, - string_as_str, RegExpHeader, FANCY_CACHE, REGEX_SOURCE_TABLE, REPEAT_MATCHER_CACHE, - VALIDATED_PATTERNS, + string_as_str, RegExpHeader, FANCY_CACHE, REPEAT_MATCHER_CACHE, VALIDATED_PATTERNS, }; /// The exact string `build_std_regex` is handed for `(pattern, flags)`: the @@ -160,15 +159,10 @@ pub(super) fn mark_pattern_validated(pattern: &str, flags: &str) { /// The `(source, flags)` a header was built from. /// -/// Prefers the GC-survivable side table (issue #637) and falls back to the -/// header's own string payloads, which — unlike the thread-local table — are -/// readable from a second statically-linked copy of the runtime (Wall 18). +/// Since #9845 the header's string slots are traced GC edges, so the payloads +/// are both collection-safe and readable from a second statically-linked copy +/// of the runtime (Wall 18). pub(super) fn source_and_flags(re: *const RegExpHeader) -> (Arc, Arc) { - if let Some(source) = - REGEX_SOURCE_TABLE.with(|table| table.borrow().get(&(re as usize)).cloned()) - { - return source; - } unsafe { let pattern: Arc = if is_valid_ptr((*re).pattern_ptr) { Arc::from(string_as_str((*re).pattern_ptr)) diff --git a/crates/perry-runtime/src/regex/program_key.rs b/crates/perry-runtime/src/regex/program_key.rs index e753fe0f69..45852319cb 100644 --- a/crates/perry-runtime/src/regex/program_key.rs +++ b/crates/perry-runtime/src/regex/program_key.rs @@ -29,9 +29,8 @@ pub(crate) const NEVER_MATCH_PATTERN: &str = r"[^\s\S]"; /// 1,984 MB), which is what `.to_string()` on an `Arc` lowers to. /// /// Keying by `Arc` makes a probe two refcount increments and no -/// allocation: every caller that matters already holds those `Arc`s, because -/// `REGEX_SOURCE_TABLE` and `regex::site_cache` share one allocation of a -/// literal's text with every header built from it. Hashing still walks the +/// allocation: every caller that matters already holds those `Arc`s through +/// `regex::site_cache`. Hashing still walks the /// pattern bytes — the allocation is what the census measured, and what this /// removes. #[cfg(feature = "regex-engine")] diff --git a/crates/perry-runtime/src/regex/site_cache.rs b/crates/perry-runtime/src/regex/site_cache.rs index bf0d3c1897..17788e277b 100644 --- a/crates/perry-runtime/src/regex/site_cache.rs +++ b/crates/perry-runtime/src/regex/site_cache.rs @@ -8,8 +8,8 @@ //! `/…/g` on every call, once per text segment per layout pass, and //! `ansi-regex` builds the same `new RegExp(parts.join("|"), "g")` per call. //! Each construction used to copy the pattern three times (the -//! `VALIDATED_PATTERNS` probe key, `owned_pattern`, the `REGEX_SOURCE_TABLE` -//! entry) and SipHash all of it once; the first operation on each header then +//! `VALIDATED_PATTERNS` probe key and `owned_pattern`) and SipHash all of it +//! once; the first operation on each header then //! did the same three more times — `build_and_install_programs` probes the //! three `(String, String)`-keyed program caches — and, for the common //! no-fallback pattern, `lookup_fancy_regex` / `lookup_repeat_matcher` @@ -24,9 +24,8 @@ //! full byte compare — identity never depends on an address, so nothing is //! rekeyed on a GC move and a dynamic `new RegExp(sameText)` hits too; a hit //! costs one `memcmp` instead of a hash plus three copies. An entry owns the -//! pattern and canonical flags as `Arc` (shared into -//! `REGEX_SOURCE_TABLE`, so a header costs two refcount bumps instead of two -//! `String`s) and, once the first header built from it has been executed, the +//! pattern and canonical flags as `Arc` and, once the first header built +//! from it has been executed, the //! compiled programs: a later construction installs those eagerly, so the //! header is born built and never touches the `(pattern, flags)` caches. //! diff --git a/crates/perry-runtime/src/regex/tests_part2.rs b/crates/perry-runtime/src/regex/tests_part2.rs index 3ea07b9671..ec6b577f66 100644 --- a/crates/perry-runtime/src/regex/tests_part2.rs +++ b/crates/perry-runtime/src/regex/tests_part2.rs @@ -151,6 +151,66 @@ fn construction_defers_the_program_build_until_first_use() { ); } +/// Removing the address-keyed source table must not turn the RegExp-pattern +/// constructor arm into an empty-pattern fallback. This calls the exported +/// constructor entry point, so deleting its direct header read fails both the +/// source and inherited-flags assertions. +#[test] +fn regexp_construct_reads_source_and_flags_from_the_pattern_header() { + let original = js_regexp_new(make_string("left/right"), make_string("ig")); + let pattern = crate::value::js_nanbox_pointer(original as i64); + let undefined = f64::from_bits(crate::value::TAG_UNDEFINED); + + let copy = js_regexp_construct(pattern, undefined); + assert_ne!( + copy, original, + "construction must still allocate a fresh object" + ); + assert_eq!(string_payload(js_regexp_get_source(copy)), b"left\\/right"); + assert_eq!(string_payload(js_regexp_get_flags(copy)), b"gi"); + + let override_flags = crate::value::js_nanbox_string(make_string("m") as i64); + let overridden = js_regexp_construct(pattern, override_flags); + assert_eq!( + string_payload(js_regexp_get_source(overridden)), + b"left\\/right" + ); + assert_eq!(string_payload(js_regexp_get_flags(overridden)), b"m"); +} + +/// `RegExp.prototype.compile` rewrites the header in place. The source table +/// used to mask a stale header slot here; after its removal both observable +/// strings must come from the newly stored, traced edges. +#[test] +fn regexp_compile_replaces_the_header_source_and_flags() { + let receiver = js_regexp_new(make_string("old"), make_string("m")); + js_regexp_set_last_index(receiver, 9.0); + let pattern = crate::value::js_nanbox_string(make_string("new/source") as i64); + let flags = crate::value::js_nanbox_string(make_string("ig") as i64); + let result = js_regexp_compile_value(receiver, pattern, flags); + let receiver = crate::value::JSValue::from_bits(result.to_bits()).as_pointer::(); + + assert_eq!( + string_payload(js_regexp_get_source(receiver)), + b"new\\/source" + ); + assert_eq!(string_payload(js_regexp_get_flags(receiver)), b"gi"); + assert_eq!(js_regexp_get_last_index(receiver), 0.0); + assert_eq!(js_regexp_test(receiver, make_string("NEW/source")), 1); +} + +/// Perry stores lone JavaScript surrogates as WTF-8. `.source` must copy those +/// exact bytes from the traced pattern slot; routing them through Rust's UTF-8 +/// scalar iterator either replaces the surrogate or invokes undefined +/// behaviour. +#[test] +fn regexp_source_round_trips_wtf8_lone_surrogates_from_the_header() { + let lone_high = [b'a', 0xED, 0xA0, 0x80, b'/', b'b']; + let re = js_regexp_new(make_wtf8(&lone_high), make_string("")); + let expected = [b'a', 0xED, 0xA0, 0x80, b'\\', b'/', b'b']; + assert_eq!(string_payload(js_regexp_get_source(re)), expected); +} + /// The deferred build installs the fancy-regex and RepeatMatcher programs too, /// not just the linear one — they live on the same publish point, so a header /// whose pattern needs one must still get it on first use. @@ -633,16 +693,6 @@ fn site_cache_reconstruction_is_born_built() { std::ptr::eq(unsafe { (*re1).regex_ptr }, unsafe { (*re2).regex_ptr }), "both headers share one compiled program" ); - // The owned source copies are shared too (two refcount bumps per header, - // not two `String`s). - let (p1, p2) = REGEX_SOURCE_TABLE.with(|t| { - let t = t.borrow(); - ( - t.get(&(re1 as usize)).map(|(p, _)| p.clone()).unwrap(), - t.get(&(re2 as usize)).map(|(p, _)| p.clone()).unwrap(), - ) - }); - assert!(Arc::ptr_eq(&p1, &p2), "source text is shared, not copied"); assert_eq!(js_regexp_test(re2, make_string("born7built")), 1); assert_eq!(js_regexp_test(re2, make_string("nothing")), 0); // Different flags are a different entry. From ea494af1e5cc549844034887dd6f67c5d856d0bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 6 Sep 2026 22:21:49 +0200 Subject: [PATCH 02/27] perf(regex): share one program-set handle per header (cherry picked from commit e2b0a905482fd3440714b1299a8c10aa7aa742d4) --- .../gc/tests/copying/survival_and_malloc.rs | 4 +- crates/perry-runtime/src/regex.rs | 158 +++++------------- crates/perry-runtime/src/regex/compile.rs | 51 ++---- .../perry-runtime/src/regex/compile_cache.rs | 4 +- crates/perry-runtime/src/regex/lazy.rs | 54 ++---- .../perry-runtime/src/regex/replace_expand.rs | 2 +- crates/perry-runtime/src/regex/site_cache.rs | 16 +- crates/perry-runtime/src/regex/site_key.rs | 72 +++----- crates/perry-runtime/src/regex/tests.rs | 49 ++++-- crates/perry-runtime/src/regex/tests_part2.rs | 46 ++--- 10 files changed, 173 insertions(+), 283 deletions(-) diff --git a/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs b/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs index 91cdf84d8f..19aedfa34f 100644 --- a/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs +++ b/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs @@ -1043,7 +1043,7 @@ fn nursery_regexp_that_dies_young_is_finalized_by_the_copied_minor() { ); assert!(crate::regex::test_regex_pointer_entry_exists(dead_addr)); // Both headers share one program through the site cache. - let count_before = crate::regex::test_regexp_std_program_strong_count(live); + let count_before = crate::regex::test_regexp_program_set_strong_count(live); assert!(count_before >= 2); // Only `live` is rooted; `dead` is garbage. @@ -1061,7 +1061,7 @@ fn nursery_regexp_that_dies_young_is_finalized_by_the_copied_minor() { "a nursery RegExp that died must be removed from REGEX_POINTERS by the copied minor" ); assert_eq!( - crate::regex::test_regexp_std_program_strong_count(live_new as *const _), + crate::regex::test_regexp_program_set_strong_count(live_new as *const _), count_before - 1, "the dead header's Arc clone of the shared program must have been dropped" ); diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 5d8aa2db2f..527e0c99b4 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -23,14 +23,14 @@ use crate::value::js_nanbox_string; use crate::object::ObjectHeader; -/// The compiled standard-engine regex type. When the regex engine is gated -/// off, `RegExpHeader::regex_ptr` is typed `*mut ()` (a never-dereferenced -/// dangling field) so the identity/display layer keeps the same struct -/// layout without pulling in the `regex` crate. +/// The shared compiled-program set. When the regex engine is gated off, +/// `RegExpHeader::programs_ptr` is typed `*const ()` (a never-dereferenced +/// field) so the identity/display layer keeps the same struct layout without +/// pulling in the matcher crates. #[cfg(feature = "regex-engine")] -type CompiledRegex = regex::Regex; +type CompiledPrograms = site_cache::Programs; #[cfg(not(feature = "regex-engine"))] -type CompiledRegex = (); +type CompiledPrograms = (); #[cfg(feature = "regex-engine")] mod class_range_validate; @@ -224,7 +224,7 @@ pub(crate) fn regex_header_clear_dead_for_gc(addr: usize) { /// Release the compiled programs owned by a dead `RegExpHeader`, then remove /// its address-owned metadata. /// -/// The program pointers are raw `Arc` references installed by +/// The program pointer is a raw `Arc` reference installed by /// `lazy::build_and_install_programs` or `RegExp.prototype.compile`. Null them /// before reconstructing the `Arc`s because arena cleanup can visit the /// metadata and finalizer paths for the same dead cell. @@ -234,23 +234,11 @@ pub(crate) unsafe fn regex_header_finalize_for_gc(re: *mut RegExpHeader) { } #[cfg(feature = "regex-engine")] { - let regex_ptr = (*re).regex_ptr; - let fancy_ptr = (*re).fancy_ptr; - let repeat_matcher_ptr = (*re).repeat_matcher_ptr; - (*re).regex_ptr = ptr::null_mut(); - (*re).fancy_ptr = ptr::null(); - (*re).repeat_matcher_ptr = ptr::null(); - - if !regex_ptr.is_null() { - drop(Arc::from_raw(regex_ptr as *const Regex)); - } - if !fancy_ptr.is_null() { - drop(Arc::from_raw(fancy_ptr as *const fancy_regex::Regex)); - } - if !repeat_matcher_ptr.is_null() { - drop(Arc::from_raw( - repeat_matcher_ptr as *const repeat_matcher::RepeatMatcherRegex, - )); + let programs_ptr = (*re).programs_ptr; + (*re).programs_ptr = ptr::null(); + + if !programs_ptr.is_null() { + drop(Arc::from_raw(programs_ptr)); } } regex_header_clear_dead_for_gc(re as usize); @@ -260,7 +248,7 @@ pub(crate) unsafe fn regex_header_finalize_for_gc(re: *mut RegExpHeader) { /// /// The copying minor's from-space flip runs no per-object finalize hooks, so /// a nursery header that was neither evacuated nor pinned would otherwise keep -/// its `Arc` programs and its `REGEX_POINTERS` / expando +/// its program-set `Arc` and its `REGEX_POINTERS` / expando /// entries forever. Same shape as `map::finalize_dead_copied_minor_from_space_maps`: /// walk the registry after the flip, collect the provably-dead addresses, then /// finalize each (the finalizer removes its own registry entries, which is why @@ -359,14 +347,14 @@ pub(crate) fn test_construct_regexp_and_exec_once(pattern: &str, flags: &str) -> /// Test support: strong count of the standard program a header holds (the /// observer clone taken here is released before returning). #[cfg(all(test, feature = "regex-engine"))] -pub(crate) fn test_regexp_std_program_strong_count(re: *const RegExpHeader) -> usize { +pub(crate) fn test_regexp_program_set_strong_count(re: *const RegExpHeader) -> usize { unsafe { - let raw = (*re).regex_ptr as *const Regex; - assert!(!raw.is_null(), "program must be installed"); - let arc = Arc::from_raw(raw); - let n = Arc::strong_count(&arc); + let programs = (*re).programs_ptr; + assert!(!programs.is_null(), "program must be installed"); + let arc = Arc::from_raw(programs); + let count = Arc::strong_count(&arc); std::mem::forget(arc); - n + count } } @@ -394,7 +382,7 @@ pub(crate) fn test_alloc_nursery_regexp_for_move(source: &str, flags: &str) -> * // Neither `gc_malloc` nor the arena zeroes reused memory, so this // must be set explicitly or the GC follows a garbage pointer. (*ptr).meta = std::ptr::null_mut(); - (*ptr).regex_ptr = std::ptr::null_mut(); + (*ptr).programs_ptr = std::ptr::null(); (*ptr).pattern_ptr = pattern.get_raw_const_ptr::(); (*ptr).flags_ptr = flags_string.get_raw_const_ptr::(); (*ptr).case_insensitive = flags.contains('i'); @@ -406,8 +394,6 @@ pub(crate) fn test_alloc_nursery_regexp_for_move(source: &str, flags: &str) -> * (*ptr).has_indices = flags.contains('d'); (*ptr).last_index = crate::value::JSValue::number(0.0).bits(); (*ptr).magic = REGEXP_MAGIC; - (*ptr).fancy_ptr = std::ptr::null(); - (*ptr).repeat_matcher_ptr = std::ptr::null(); REGEX_EVER_REGISTERED.arm(); REGEX_POINTERS.with(|table| { @@ -456,7 +442,7 @@ pub(crate) fn regex_header_has_magic(re: *const RegExpHeader) -> bool { /// * `flags_ptr` — the flags `StringHeader`, /// * `last_index` — a writable JSValue (`re.lastIndex = …`) that may be a /// NaN-boxed heap pointer. -/// The compiled matcher pointers point to OFF-heap leaked Rust allocations and the +/// The compiled-program pointer points to an OFF-heap Rust allocation and the /// bool/`magic` fields are never heap refs, so they must NOT be scanned. /// /// `pattern_ptr` and `flags_ptr` are consecutive equal-width fields, so under @@ -505,11 +491,12 @@ pub(crate) use compile_cache::*; /// Header for heap-allocated RegExp objects #[repr(C)] pub struct RegExpHeader { - /// Pointer to the compiled Regex object (boxed). Typed via the - /// `CompiledRegex` alias so the struct layout is identical whether or not - /// the regex engine is linked (it's `*mut ()` when gated off and never - /// dereferenced — all dereferencing sites are themselves engine-gated). - regex_ptr: *mut CompiledRegex, + /// Header-owned `Arc` raw pointer, or null until first use. + /// The program set contains the standard engine and optional fancy/repeat + /// matchers once per pattern instead of repeating three pointers in every + /// RegExp object. Typed through `CompiledPrograms` so the layout is stable + /// when the regex engine is gated off. + programs_ptr: *const CompiledPrograms, /// Original pattern string (for debugging/serialization) pattern_ptr: *const StringHeader, /// Flags string (e.g., "gi" for global+ignoreCase) @@ -548,15 +535,6 @@ pub struct RegExpHeader { /// identity + fancy-fallback resolution independent of WHICH runtime copy's /// thread-locals are live. Set to `REGEXP_MAGIC` by `js_regexp_new`. pub magic: u64, - /// Leaked `Arc` (as a raw pointer) for patterns the - /// `regex` crate can't compile (lookahead/lookbehind/backrefs), or null. - /// Header-resident twin of the `FANCY_CACHE` thread-local so the fancy - /// fallback survives the duplicate-runtime split described above. - pub fancy_ptr: *const (), - /// Header-owned `Arc` for quantified capture groups, - /// or null for the ordinary linear/fancy paths. Like `fancy_ptr`, this - /// survives cache eviction and duplicate statically-linked runtime copies. - pub repeat_matcher_ptr: *const (), /// #6759 phase 1 (header unification): per-object metadata record, or /// null. Appended LAST so `regex_gc_slot_ptrs`' adjacency assertion on /// `pattern_ptr`/`flags_ptr` and every other offset are undisturbed. @@ -706,8 +684,8 @@ fn newborn_barrier_gate_enabled() -> bool { /// /// Validates the pattern and allocates the header; it does NOT build the /// compiled program. That happens on the first operation that needs a matcher -/// — see `regex::lazy`, and the `regex_ptr`/`fancy_ptr`/`repeat_matcher_ptr` -/// fields, which are null until then. A fresh header per call is required: +/// — see `regex::lazy`; `programs_ptr` is null until then. A fresh header per +/// call is required: /// ECMA-262 evaluates a regex literal to a NEW object every time, and the /// distinction is observable through `===`, expandos and `lastIndex`. #[cfg(feature = "regex-engine")] @@ -1008,7 +986,7 @@ fn js_regexp_new_impl( // established that the pattern is legal, and a bundle evaluates hundreds // of module-level literals it never matches with — building each one's // NFA at construction is what put ~14% of a claude-code `--help` run - // inside `regex_syntax`/`regex_automata`. `regex_ptr` stays null (the + // inside `regex_syntax`/`regex_automata`. `programs_ptr` stays null (the // "not built yet" state) and `lazy::ensure_regex_compiled` installs the // owned `Arc`s on the first operation that needs a matcher. @@ -1087,7 +1065,7 @@ fn js_regexp_new_impl( // exercised the arena arm all along. What kept production on malloc was // finalization: the copying minor's from-space flip runs no per-object // finalize hooks (`gc::copying`), so a nursery header that dies young - // would leak its three `Arc` programs and its registry entries. That is + // would leak its program-set `Arc` and its registry entries. That is // now handled the way Map/Set/Error handle theirs — // `finalize_dead_copied_minor_from_space_regexps` after a copied minor and // `collect_dead_registered_regexps_post_trace` at sweep entry for the @@ -1152,7 +1130,7 @@ fn js_regexp_new_impl( // must be set explicitly or the GC follows a garbage pointer. (*ptr).meta = std::ptr::null_mut(); // Null = not compiled yet; see `lazy::ensure_regex_compiled`. - (*ptr).regex_ptr = std::ptr::null_mut(); + (*ptr).programs_ptr = std::ptr::null(); (*ptr).pattern_ptr = pattern; (*ptr).flags_ptr = canonical_flags_ptr; // `pattern_ptr` / `flags_ptr` are GC-managed StringHeaders — the GC scans @@ -1232,26 +1210,11 @@ fn js_regexp_new_impl( // Wall 18: self-identifying marker so identity checks survive a // duplicate-runtime thread-local split. (*ptr).magic = REGEXP_MAGIC; - // The header-resident fancy-regex fallback (lookahead/lookbehind/ - // backrefs) and the ECMAScript backtracking matcher are installed - // alongside `regex_ptr` by `lazy::ensure_regex_compiled`, from the - // same caches, on the first operation that needs a matcher. Keeping - // all three on one publish point is what makes `regex_ptr.is_null()` - // a sound built/not-built flag. - (*ptr).fancy_ptr = std::ptr::null(); - (*ptr).repeat_matcher_ptr = std::ptr::null(); - // Born built: the site cache already holds the programs the first - // execution of this text compiled. Install the same three owned - // references `lazy::build_and_install_programs` would, publishing - // `regex_ptr` last for the same reason it does. + // Born built: the site cache already holds the shared program set the + // first execution of this text compiled. Install one owned reference; + // null remains the sound not-built state. if let Some(programs) = programs { - (*ptr).fancy_ptr = programs - .fancy - .map_or(std::ptr::null(), |arc| Arc::into_raw(arc) as *const ()); - (*ptr).repeat_matcher_ptr = programs - .repeat - .map_or(std::ptr::null(), |arc| Arc::into_raw(arc) as *const ()); - (*ptr).regex_ptr = Arc::into_raw(programs.std) as *mut Regex; + (*ptr).programs_ptr = Arc::into_raw(programs); } // Record the pointer so that js_string_split can detect @@ -1538,32 +1501,14 @@ pub(super) fn diag_note_op(re: *const RegExpHeader, op: crate::hot_diag::RegexOp /// pattern (backreferences, lookbehind, etc.). #[cfg(feature = "regex-engine")] pub(crate) fn lookup_fancy_regex(re: *const RegExpHeader) -> Option> { - // The header's programs are built on first use; `fancy_ptr` is null until - // then, and a null there is indistinguishable from "this pattern has no - // fancy fallback" — so build before reading it. + // The header's shared program set is built on first use. lazy::ensure_regex_compiled(re); unsafe { - // Wall 18: header-resident fancy Arc first (duplicate-runtime - // thread-local resilient). `fancy_ptr` is a leaked `Arc` raw pointer; to - // hand back an owned `Arc` clone WITHOUT consuming the header's - // reference, reconstruct, clone, then `mem::forget` the reconstructed - // one so the header's strong count is preserved. + // Wall 18: header-resident program set first (duplicate-runtime + // thread-local resilient). if regex_header_has_magic(re) { - if (*re).fancy_ptr.is_null() { - // Built (see `ensure_regex_compiled` above) with no fancy - // fallback: every install path (`lazy`, `compile`, the site - // cache) publishes all three program pointers together, so a - // null here is the answer, not "not looked up yet". Falling - // through to the cache probe re-hashed the whole pattern on - // EVERY exec of every ordinary regex (#keystroke profile: - // 514 samples under this function alone). - return None; - } - let raw = (*re).fancy_ptr as *const fancy_regex::Regex; - let arc = Arc::from_raw(raw); - let cloned = arc.clone(); - std::mem::forget(arc); - return Some(cloned); + let programs = &*(*re).programs_ptr; + return programs.fancy.clone(); } let pat = string_as_str((*re).pattern_ptr); let flags_str = string_as_str((*re).flags_ptr); @@ -1611,11 +1556,11 @@ pub(crate) fn lookup_fancy_regex(re: *const RegExpHeader) -> Option bool { unsafe { - let program = (*re).regex_ptr; - if program.is_null() { + let programs = (*re).programs_ptr; + if programs.is_null() { return false; } - let program: &Regex = &*program; + let program: &Regex = &(*programs).std; if program.as_str() == NEVER_MATCH_PATTERN { // The `regex` crate refused this pattern (lookaround / // backreference); it has no opinion about the subject. @@ -1649,22 +1594,11 @@ fn lookup_repeat_matcher_for( fn lookup_repeat_matcher( re: *const RegExpHeader, ) -> Option> { - // Same first-use build as `lookup_fancy_regex`: a null - // `repeat_matcher_ptr` means "not built yet" before it can mean "this - // pattern needs no backtracking matcher". lazy::ensure_regex_compiled(re); unsafe { if regex_header_has_magic(re) { - if (*re).repeat_matcher_ptr.is_null() { - // Same reasoning as `lookup_fancy_regex`: a built header with - // a null pointer has no backtracking matcher. - return None; - } - let raw = (*re).repeat_matcher_ptr as *const repeat_matcher::RepeatMatcherRegex; - let arc = Arc::from_raw(raw); - let cloned = arc.clone(); - std::mem::forget(arc); - return Some(cloned); + let programs = &*(*re).programs_ptr; + return programs.repeat.clone(); } let pat = string_as_str((*re).pattern_ptr); let flags_str = string_as_str((*re).flags_ptr); diff --git a/crates/perry-runtime/src/regex/compile.rs b/crates/perry-runtime/src/regex/compile.rs index 865fdfd02b..c569ad8bb3 100644 --- a/crates/perry-runtime/src/regex/compile.rs +++ b/crates/perry-runtime/src/regex/compile.rs @@ -4,8 +4,6 @@ use std::sync::Arc; -use regex::Regex; - use super::class_range_validate::has_out_of_order_double_dash_class_range; use super::grammar::{ has_invalid_repeated_quantifier, has_unicode_forbidden_legacy_escape, @@ -145,33 +143,30 @@ pub extern "C" fn js_regexp_compile_value( )); } - // The header OWNS raw `Arc` references to its compiled program(s) + // The header OWNS one raw `Arc` reference to its compiled program set // (mirrors `js_regexp_new`), so the capped `REGEX_CACHE`/`FANCY_CACHE` // (see `REGEX_CACHE_MAX_ENTRIES`) can evict without invalidating this - // receiver. Refresh `fancy_ptr` too — it must track the NEW pattern, not - // the one the receiver was constructed with. + // receiver. Refresh the whole program set so it tracks the NEW pattern, + // not the one the receiver was constructed with. // `RegExp.prototype.compile` re-initialises an existing receiver — once per // call from user code, not per object — so materialising the shared key // here costs nothing measurable. let pattern_key: std::sync::Arc = std::sync::Arc::from(pattern_str); let flags_key: std::sync::Arc = std::sync::Arc::from(flags_str); - let arc = get_or_compile_regex(&pattern_key, &flags_key); - let regex_ptr = Arc::into_raw(arc) as *mut Regex; - let fancy_ptr: *const () = super::FANCY_CACHE.with(|fc| { - match fc.borrow().get(&(pattern_key.clone(), flags_key.clone())) { - Some(arc) => Arc::into_raw(arc.clone()) as *const (), - None => std::ptr::null(), - } + let std = get_or_compile_regex(&pattern_key, &flags_key); + let fancy = super::FANCY_CACHE.with(|fc| { + fc.borrow() + .get(&(pattern_key.clone(), flags_key.clone())) + .cloned() }); - let repeat_matcher_ptr: *const () = super::REPEAT_MATCHER_CACHE.with(|cache| { - match cache + let repeat = super::REPEAT_MATCHER_CACHE.with(|cache| { + cache .borrow() .get(&(pattern_key.clone(), flags_key.clone())) - { - Some(arc) => Arc::into_raw(arc.clone()) as *const (), - None => std::ptr::null(), - } + .cloned() }); + let programs = Arc::new(super::site_cache::Programs { std, fancy, repeat }); + let programs_ptr = Arc::into_raw(programs); let (canonical_flags_ptr, _) = re_handle.across_mut::(|| js_string_from_str(flags_str)); let canonical_flags_handle = scope.root_string_ptr(canonical_flags_ptr); @@ -180,25 +175,13 @@ pub extern "C" fn js_regexp_compile_value( .across_const::(|| js_string_from_str(pattern_str)) }); unsafe { - let old_regex_ptr = (*re).regex_ptr; - let old_fancy_ptr = (*re).fancy_ptr; - let old_repeat_matcher_ptr = (*re).repeat_matcher_ptr; - (*re).regex_ptr = regex_ptr; - (*re).fancy_ptr = fancy_ptr; - (*re).repeat_matcher_ptr = repeat_matcher_ptr; + let old_programs_ptr = (*re).programs_ptr; + (*re).programs_ptr = programs_ptr; // Release the receiver's PREVIOUS owned references now that the new // ones are installed (recompiling the same pattern is fine: the fresh // `into_raw` reference above keeps the shared program alive). - if !old_regex_ptr.is_null() { - drop(Arc::from_raw(old_regex_ptr as *const Regex)); - } - if !old_fancy_ptr.is_null() { - drop(Arc::from_raw(old_fancy_ptr as *const fancy_regex::Regex)); - } - if !old_repeat_matcher_ptr.is_null() { - drop(Arc::from_raw( - old_repeat_matcher_ptr as *const super::repeat_matcher::RepeatMatcherRegex, - )); + if !old_programs_ptr.is_null() { + drop(Arc::from_raw(old_programs_ptr)); } (*re).pattern_ptr = pattern_ptr; (*re).flags_ptr = canonical_flags_ptr; diff --git a/crates/perry-runtime/src/regex/compile_cache.rs b/crates/perry-runtime/src/regex/compile_cache.rs index bde70bc77f..c08835f960 100644 --- a/crates/perry-runtime/src/regex/compile_cache.rs +++ b/crates/perry-runtime/src/regex/compile_cache.rs @@ -131,7 +131,7 @@ pub(crate) fn evict_regex_cache_if_full(cache: &mut HashMap) { /// One shared never-match program per thread. /// /// Only used by the `PERRY_REGEX_ENGINE=regress` measurement path, where every -/// pattern needs a value in `regex_ptr` (the built/not-built flag) but no NFA: +/// pattern needs a value in `programs_ptr` (the built/not-built flag) but no NFA: /// building a fresh one per pattern would be exactly the compile cost the /// experiment exists to remove from the measurement. #[cfg(feature = "regex-engine")] @@ -173,7 +173,7 @@ pub(crate) fn compile_and_cache_regex_checked(pattern: &Arc, flags: &Arc (Arc, Arc) /// Build this header's compiled program(s) if it has none yet. /// -/// `regex_ptr == null` is the "not built yet" state. It is published LAST so -/// a header is never observable as built while `fancy_ptr` / -/// `repeat_matcher_ptr` are still stale — every reader that consults those -/// two goes through [`lookup_fancy_regex`](super::lookup_fancy_regex) / -/// `lookup_repeat_matcher`, which call this first. +/// `programs_ptr == null` is the "not built yet" state. The one-pointer +/// publication keeps the three engines coherent. /// -/// The header OWNS a leaked `Arc` reference to each program (mirroring what -/// `js_regexp_new` used to do inline), so the capped `REGEX_CACHE` / -/// `FANCY_CACHE` / `REPEAT_MATCHER_CACHE` can evict without invalidating a -/// live receiver. +/// The header OWNS one leaked `Arc` to the complete program set, so the capped +/// `REGEX_CACHE` / `FANCY_CACHE` / `REPEAT_MATCHER_CACHE` can evict without +/// invalidating a live receiver. /// /// Contains no JS allocation and cannot re-enter the interpreter, so it is /// safe to call from inside a phase that holds a borrow of a GC string. @@ -209,7 +204,7 @@ pub(crate) fn ensure_regex_compiled(re: *const RegExpHeader) { if !is_valid_ptr(re) { return; } - if unsafe { !(*re).regex_ptr.is_null() } { + if unsafe { !(*re).programs_ptr.is_null() } { return; } build_and_install_programs(re); @@ -248,9 +243,7 @@ fn build_and_install_programs(re: *const RegExpHeader) { }); // ── Repair before publishing ────────────────────────────────────────── // - // A built header is treated as AUTHORITATIVE — `lookup_fancy_regex` / - // `lookup_repeat_matcher` read a null slot beside a non-null `regex_ptr` - // as "this pattern has no such program" — and `install_programs` below + // A built header is treated as AUTHORITATIVE, and `install_programs` below // memoizes the triple against the pattern text, so whatever is assembled // here becomes the answer for every later construction of the same // literal. It therefore has to be complete, and the probes above cannot @@ -300,32 +293,21 @@ fn build_and_install_programs(re: *const RegExpHeader) { // Remember the built programs against the pattern text, so the next // construction of the same literal is born built (`js_regexp_new`). - super::site_cache::install_programs( - &pattern, - &flags, - super::site_cache::Programs { - std: std_arc.clone(), - fancy: fancy_arc.clone(), - repeat: repeat_arc.clone(), - }, - ); - let regex_ptr = Arc::into_raw(std_arc) as *mut Regex; - let fancy_ptr: *const () = - fancy_arc.map_or(std::ptr::null(), |arc| Arc::into_raw(arc) as *const ()); - let repeat_matcher_ptr: *const () = - repeat_arc.map_or(std::ptr::null(), |arc| Arc::into_raw(arc) as *const ()); + let programs = Arc::new(super::site_cache::Programs { + std: std_arc.clone(), + fancy: fancy_arc.clone(), + repeat: repeat_arc.clone(), + }); + super::site_cache::install_programs(&pattern, &flags, programs.clone()); unsafe { let re = re as *mut RegExpHeader; - (*re).fancy_ptr = fancy_ptr; - (*re).repeat_matcher_ptr = repeat_matcher_ptr; - // Publish last: `regex_ptr` is the built/not-built flag. - (*re).regex_ptr = regex_ptr; + (*re).programs_ptr = Arc::into_raw(programs); } } /// The header's standard-engine program, building it on first use. /// -/// Every `&*(*re).regex_ptr` in the tree goes through here — the field is +/// Every standard-program borrow in the tree goes through here — the field is /// null until something needs a matcher. /// /// # Safety @@ -334,5 +316,5 @@ fn build_and_install_programs(re: *const RegExpHeader) { /// header owns until its GC finalizer runs. pub(crate) unsafe fn header_std_regex<'a>(re: *const RegExpHeader) -> &'a Regex { ensure_regex_compiled(re); - &*(*re).regex_ptr + &(*(*re).programs_ptr).std } diff --git a/crates/perry-runtime/src/regex/replace_expand.rs b/crates/perry-runtime/src/regex/replace_expand.rs index 70b1931f61..89391be8aa 100644 --- a/crates/perry-runtime/src/regex/replace_expand.rs +++ b/crates/perry-runtime/src/regex/replace_expand.rs @@ -374,7 +374,7 @@ pub extern "C" fn js_string_replace_regex_fn( // If the `regex` crate couldn't compile this pattern (lookahead, // backreferences, …), `get_or_compile_regex` stashed a never-match - // placeholder in `(*re).regex_ptr` and the real pattern in + // placeholder in the header's standard program and the real pattern in // `FANCY_CACHE`. Route the callback-replace through fancy-regex so the // callback actually fires — otherwise `captures_iter` below would // silently match nothing and return the input unchanged. (get-intrinsic's diff --git a/crates/perry-runtime/src/regex/site_cache.rs b/crates/perry-runtime/src/regex/site_cache.rs index 17788e277b..459ad1f1de 100644 --- a/crates/perry-runtime/src/regex/site_cache.rs +++ b/crates/perry-runtime/src/regex/site_cache.rs @@ -47,28 +47,18 @@ pub(super) struct Programs { pub(super) repeat: Option>, } -impl Clone for Programs { - fn clone(&self) -> Self { - Self { - std: self.std.clone(), - fancy: self.fancy.clone(), - repeat: self.repeat.clone(), - } - } -} - /// What a construction gets back on a hit. pub(super) struct Hit { pub(super) pattern: Arc, pub(super) flags: Arc, - pub(super) programs: Option, + pub(super) programs: Option>, } struct Entry { fp: u64, pattern: Arc, flags: Arc, - programs: Option, + programs: Option>, } /// Direct-mapped slots (2-way: a fingerprint may live in `slot` or @@ -204,7 +194,7 @@ pub(super) fn insert(pattern: &str, flags: &str) -> (Arc, Arc) { /// Attach the programs the first execution built to the entry for /// `(pattern, canonical flags)`, so every later construction of the same /// text is born built. Inserts the entry if it was evicted meanwhile. -pub(super) fn install_programs(pattern: &str, flags: &str, programs: Programs) { +pub(super) fn install_programs(pattern: &str, flags: &str, programs: Arc) { if !enabled() { return; } diff --git a/crates/perry-runtime/src/regex/site_key.rs b/crates/perry-runtime/src/regex/site_key.rs index aff7f4f028..842faa7072 100644 --- a/crates/perry-runtime/src/regex/site_key.rs +++ b/crates/perry-runtime/src/regex/site_key.rs @@ -65,37 +65,20 @@ use super::site_cache::Programs; /// site entry whose programs have been dropped simply reports "not built /// yet", and the next construction re-picks them up from the content cache — /// the same path the site's very first construction takes. -struct WeakPrograms { - std: Weak<::regex::Regex>, - fancy: Option>, - repeat: Option>, -} +struct WeakPrograms(Weak); impl WeakPrograms { - fn downgrade(programs: &Programs) -> Self { - Self { - std: Arc::downgrade(&programs.std), - fancy: programs.fancy.as_ref().map(Arc::downgrade), - repeat: programs.repeat.as_ref().map(Arc::downgrade), - } + fn downgrade(programs: &Arc) -> Self { + Self(Arc::downgrade(programs)) } /// ALL-OR-NOTHING. A header must carry **every** program its pattern needs /// — that is #9801's coherence rule, and a partial upgrade is exactly the /// incoherent triple it fixed: a standard program installed beside a /// missing fancy fallback silently never-matches instead of falling back. - /// So a single dead reference makes the whole entry report unbuilt. - fn upgrade(&self) -> Option { - let std = self.std.upgrade()?; - let fancy = match &self.fancy { - None => None, - Some(weak) => Some(weak.upgrade()?), - }; - let repeat = match &self.repeat { - None => None, - Some(weak) => Some(weak.upgrade()?), - }; - Some(Programs { std, fancy, repeat }) + /// One weak pointer to the bundle makes partial upgrade unrepresentable. + fn upgrade(&self) -> Option> { + self.0.upgrade() } } @@ -137,7 +120,7 @@ pub(super) struct SiteHit { pub(super) flags: Arc, pub(super) flags_are_canonical: bool, pub(super) bits: FlagBits, - pub(super) programs: Option, + pub(super) programs: Option>, } /// Direct-mapped, 2-way (a key may live in `slot` or `slot ^ 1`). A bundle's @@ -204,7 +187,7 @@ pub(super) fn record( flags: Arc, flags_are_canonical: bool, bits: FlagBits, - programs: Option, + programs: Option>, ) { if !enabled() || key == 0 { return; @@ -260,7 +243,7 @@ pub(super) fn record( /// Attach the programs the first execution built, so later constructions at /// this site are born built. A no-op when the site was evicted meanwhile. -pub(super) fn install_programs(key: usize, programs: Programs) { +pub(super) fn install_programs(key: usize, programs: Arc) { if !enabled() || key == 0 { return; } @@ -323,42 +306,38 @@ mod tests { /// #9801 fixed an incoherent triple — a standard program memoized beside a /// missing fancy fallback — which does not error: it silently never /// matches. Holding the site entry's programs weakly reintroduces exactly - /// that shape unless a dead reference invalidates the WHOLE entry, because - /// the three `Arc`s have independent lifetimes and the fancy fallback is - /// the one a pattern the linear engine refused depends on. - /// - /// A sabotage that upgrades each field independently — the natural way to - /// write it — returns `Some(Programs { std, fancy: None, .. })` here and - /// fails on the second assertion. + /// that shape if it weakens each matcher separately. A single weak pointer + /// to the program bundle makes an incoherent partial upgrade impossible. #[test] - fn one_dead_reference_invalidates_the_whole_entry() { + fn the_program_set_weak_reference_expires_atomically() { let std_program = Arc::new(::regex::Regex::new("a(b)c").expect("linear program")); let fancy_program = Arc::new(::fancy_regex::Regex::new("a(?=b)c").expect("fancy program")); - let programs = Programs { + let programs = Arc::new(Programs { std: std_program.clone(), fancy: Some(fancy_program.clone()), repeat: None, - }; + }); let weak = WeakPrograms::downgrade(&programs); - drop(programs); let upgraded = weak .upgrade() - .expect("both strong references are still held here"); + .expect("the shared program set is still held here"); assert!( upgraded.fancy.is_some(), "the fancy fallback must survive the round trip while its Arc is alive" ); drop(upgraded); - // Only the FANCY program dies. The standard one is still strongly held. - drop(fancy_program); + // Individual matcher Arcs do not keep the SET alive. Once the bundle + // is gone the weak entry expires all three lanes together, which is + // the coherence property a triple of independent Weak pointers had + // to implement manually. + drop(programs); assert!( weak.upgrade().is_none(), - "one dead reference must invalidate the whole entry — handing back a header with a \ - standard program and no fancy fallback is #9801's incoherent triple, which never \ - matches instead of failing" + "the site entry must never upgrade only part of a program set" ); + drop(fancy_program); drop(std_program); assert!(weak.upgrade().is_none()); } @@ -371,11 +350,11 @@ mod tests { test_reset(); let key = 0x5171_E000_usize; let std_program = Arc::new(::regex::Regex::new("keepalive").expect("linear program")); - let programs = Programs { + let programs = Arc::new(Programs { std: std_program.clone(), fancy: None, repeat: None, - }; + }); record( key, Arc::from("g"), @@ -391,7 +370,7 @@ mod tests { unicode: false, has_indices: false, }, - Some(programs), + Some(programs.clone()), ); assert!( lookup(key, "g") @@ -401,6 +380,7 @@ mod tests { "precondition: the entry answers with its programs while they are alive" ); + drop(programs); drop(std_program); let hit = lookup(key, "g").expect("the entry itself survives"); assert!( diff --git a/crates/perry-runtime/src/regex/tests.rs b/crates/perry-runtime/src/regex/tests.rs index 11a6347e77..7d72ffff98 100644 --- a/crates/perry-runtime/src/regex/tests.rs +++ b/crates/perry-runtime/src/regex/tests.rs @@ -15,6 +15,18 @@ pub(super) fn string_payload(s: *const StringHeader) -> Vec { } } +pub(super) fn regex_is_built(re: *const RegExpHeader) -> bool { + !unsafe { (*re).programs_ptr.is_null() } +} + +pub(super) fn regex_has_fancy_program(re: *const RegExpHeader) -> bool { + regex_is_built(re) && unsafe { (*(*re).programs_ptr).fancy.is_some() } +} + +pub(super) fn regex_has_repeat_program(re: *const RegExpHeader) -> bool { + regex_is_built(re) && unsafe { (*(*re).programs_ptr).repeat.is_some() } +} + #[test] fn regexp_has_dedicated_gc_kind_and_is_not_a_shaped_object() { let _lock = crate::gc::global_side_table_test_lock(); @@ -32,7 +44,17 @@ fn regexp_has_dedicated_gc_kind_and_is_not_a_shaped_object() { } #[test] -fn malloc_finalize_clears_regexp_address_owned_tables() { +#[cfg(target_pointer_width = "64")] +fn regexp_header_is_one_56_byte_per_object_record() { + assert_eq!( + std::mem::size_of::(), + 56, + "the three per-program matcher pointers must stay collapsed into one handle" + ); +} + +#[test] +fn malloc_finalize_clears_regexp_address_owned_state() { let _lock = crate::gc::global_side_table_test_lock(); let scope = crate::gc::RuntimeHandleScope::new(); let pattern = scope.root_string_ptr(make_string("finalize")); @@ -42,7 +64,6 @@ fn malloc_finalize_clears_regexp_address_owned_tables() { }); let addr = re as usize; assert!(test_regex_pointer_entry_exists(addr)); - assert!(test_regex_source_entry_exists(addr)); crate::object::exotic_expando::test_seed_exotic_expando_entry( addr, "owned", @@ -55,7 +76,6 @@ fn malloc_finalize_clears_regexp_address_owned_tables() { } assert!(!test_regex_pointer_entry_exists(addr)); - assert!(!test_regex_source_entry_exists(addr)); assert!(!crate::object::exotic_expando::test_exotic_expando_entry_exists(addr)); } @@ -78,10 +98,10 @@ fn regexp_finalize_releases_all_header_owned_programs() { re } - // Every compiled header owns the standard-engine program, including the - // never-match placeholder used by fancy-regex patterns. + // Every compiled header owns one shared program bundle, including the + // never-match placeholder and any fallback matcher. let standard = compile(r"needle\d+", "needle42"); - let standard_raw = unsafe { (*standard).regex_ptr as *const Regex }; + let standard_raw = unsafe { (*standard).programs_ptr }; assert!(!standard_raw.is_null()); let standard_observer = clone_raw_arc(standard_raw); let standard_before = std::sync::Arc::strong_count(&standard_observer); @@ -95,11 +115,11 @@ fn regexp_finalize_releases_all_header_owned_programs() { std::sync::Arc::strong_count(&standard_observer) + 1, standard_before ); - assert!(unsafe { (*standard).regex_ptr.is_null() }); + assert!(!regex_is_built(standard)); let fancy = compile(r"(?<=pre)\d+", "pre77"); - let fancy_raw = unsafe { (*fancy).fancy_ptr as *const fancy_regex::Regex }; - assert!(!fancy_raw.is_null()); + let fancy_raw = unsafe { (*fancy).programs_ptr }; + assert!(regex_has_fancy_program(fancy)); let fancy_observer = clone_raw_arc(fancy_raw); let fancy_before = std::sync::Arc::strong_count(&fancy_observer); unsafe { @@ -109,12 +129,11 @@ fn regexp_finalize_releases_all_header_owned_programs() { std::sync::Arc::strong_count(&fancy_observer) + 1, fancy_before ); - assert!(unsafe { (*fancy).fancy_ptr.is_null() }); + assert!(!regex_is_built(fancy)); let repeat = compile(r"(a?b??)*", "ab"); - let repeat_raw = - unsafe { (*repeat).repeat_matcher_ptr as *const repeat_matcher::RepeatMatcherRegex }; - assert!(!repeat_raw.is_null()); + let repeat_raw = unsafe { (*repeat).programs_ptr }; + assert!(regex_has_repeat_program(repeat)); let repeat_observer = clone_raw_arc(repeat_raw); let repeat_before = std::sync::Arc::strong_count(&repeat_observer); unsafe { @@ -125,7 +144,7 @@ fn regexp_finalize_releases_all_header_owned_programs() { } let repeat_after = std::sync::Arc::strong_count(&repeat_observer); assert_eq!(repeat_after + 1, repeat_before); - assert!(unsafe { (*repeat).repeat_matcher_ptr.is_null() }); + assert!(!regex_is_built(repeat)); // Arena overflow cleanup and finalization can overlap. A second finalizer // must observe null pointers rather than release an owned reference twice. @@ -953,7 +972,7 @@ fn regex_cache_capped_and_prior_headers_survive_eviction() { assert!( js_regexp_test(fancy, make_string("pre77")) != 0, "fancy-fallback header must keep matching after cache eviction \ - (header-resident fancy_ptr, not the cleared FANCY_CACHE)" + (header-resident program set, not the cleared FANCY_CACHE)" ); assert!( js_regexp_test(fancy, make_string("nope77")) == 0, diff --git a/crates/perry-runtime/src/regex/tests_part2.rs b/crates/perry-runtime/src/regex/tests_part2.rs index ec6b577f66..c6ba025437 100644 --- a/crates/perry-runtime/src/regex/tests_part2.rs +++ b/crates/perry-runtime/src/regex/tests_part2.rs @@ -2,7 +2,10 @@ //! A sibling child of `regex`, so `use super::*` resolves exactly as it does //! in `tests.rs`; the shared fixtures come from there. -use super::tests::{make_string, match_capture_text, string_payload}; +use super::tests::{ + make_string, match_capture_text, regex_has_fancy_program, regex_has_repeat_program, + regex_is_built, string_payload, +}; use super::*; #[test] @@ -121,7 +124,7 @@ fn syntax_check_agrees_with_full_build() { /// fixture whose 200 literals cost 73 ms to construct before and ~0 after. A /// regression here (something re-introducing an eager build) would not fail any /// behavioural test, only make every program slower, so assert the state -/// directly: `regex_ptr` is the built/not-built flag. +/// directly: `programs_ptr` is the built/not-built flag. #[test] fn construction_defers_the_program_build_until_first_use() { let re = js_regexp_new( @@ -129,7 +132,7 @@ fn construction_defers_the_program_build_until_first_use() { make_string("i"), ); assert!( - unsafe { (*re).regex_ptr.is_null() }, + !regex_is_built(re), "constructing a RegExp must not build its program" ); // Everything observable without matching stays available. @@ -140,13 +143,13 @@ fn construction_defers_the_program_build_until_first_use() { assert_eq!(string_payload(js_regexp_get_flags(re)), b"i".to_vec()); assert!(unsafe { (*re).case_insensitive }); assert!( - unsafe { (*re).regex_ptr.is_null() }, + !regex_is_built(re), "reading .source/.flags must not build the program either" ); assert!(js_regexp_test(re, make_string("XFOO12")) != 0); assert!( - !unsafe { (*re).regex_ptr.is_null() }, + regex_is_built(re), "the first match must build and install the program" ); } @@ -217,19 +220,19 @@ fn regexp_source_round_trips_wtf8_lone_surrogates_from_the_header() { #[test] fn deferred_build_installs_the_fancy_and_repeat_matcher_fallbacks() { let fancy = js_regexp_new(make_string(r"(?<=pre)\d+"), make_string("")); - assert!(unsafe { (*fancy).fancy_ptr.is_null() }); + assert!(!regex_is_built(fancy)); assert!(js_regexp_test(fancy, make_string("pre77")) != 0); assert!( - !unsafe { (*fancy).fancy_ptr.is_null() }, + regex_has_fancy_program(fancy), "first use must install the fancy-regex fallback" ); assert!(js_regexp_test(fancy, make_string("nope77")) == 0); let repeat = js_regexp_new(make_string(r"(a?b??)*"), make_string("")); - assert!(unsafe { (*repeat).repeat_matcher_ptr.is_null() }); + assert!(!regex_is_built(repeat)); assert!(js_regexp_test(repeat, make_string("ab")) != 0); assert!( - !unsafe { (*repeat).repeat_matcher_ptr.is_null() }, + regex_has_repeat_program(repeat), "first use must install the ECMAScript RepeatMatcher" ); } @@ -669,10 +672,7 @@ fn site_cache_reconstruction_is_born_built() { let _lock = crate::gc::global_side_table_test_lock(); site_cache::test_reset(); let re1 = js_regexp_new(make_string("born[0-9]+built"), make_string("g")); - assert!( - unsafe { (*re1).regex_ptr.is_null() }, - "construction stays lazy" - ); + assert!(!regex_is_built(re1), "construction stays lazy"); assert_eq!( site_cache::test_has_programs("born[0-9]+built", "g"), Some(false), @@ -686,18 +686,20 @@ fn site_cache_reconstruction_is_born_built() { ); let re2 = js_regexp_new(make_string("born[0-9]+built"), make_string("g")); assert!( - !unsafe { (*re2).regex_ptr.is_null() }, + regex_is_built(re2), "the second construction installs the programs eagerly" ); assert!( - std::ptr::eq(unsafe { (*re1).regex_ptr }, unsafe { (*re2).regex_ptr }), + std::ptr::eq(unsafe { (*re1).programs_ptr }, unsafe { + (*re2).programs_ptr + }), "both headers share one compiled program" ); assert_eq!(js_regexp_test(re2, make_string("born7built")), 1); assert_eq!(js_regexp_test(re2, make_string("nothing")), 0); // Different flags are a different entry. let re3 = js_regexp_new(make_string("born[0-9]+built"), make_string("i")); - assert!(unsafe { (*re3).regex_ptr.is_null() }); + assert!(!regex_is_built(re3)); } /// `test` on a global/sticky receiver advances `lastIndex` exactly like @@ -818,9 +820,9 @@ fn a_single_program_cache_clear_cannot_disarm_a_lookbehind_literal() { unsafe { lazy::ensure_regex_compiled(cold); assert!( - !(*cold).fancy_ptr.is_null(), + regex_has_fancy_program(cold), "a built header must carry every program its pattern needs — a null \ - fancy_ptr here is memoized by site_cache::install_programs and makes \ + the fancy program here is memoized by site_cache::install_programs and makes \ the breakage permanent for this literal" ); } @@ -1064,7 +1066,7 @@ fn a_dynamic_construction_records_nothing_in_the_site_table() { /// A site hit must be born built: the second construction at a site whose /// first header has already executed installs the compiled programs eagerly, -/// so `regex_ptr` is non-null before any match runs. +/// so `programs_ptr` is non-null before any match runs. /// /// This is what makes the fast path complete — a hit that skipped the content /// cache but arrived unbuilt would push the pattern's hash back onto the first @@ -1077,19 +1079,19 @@ fn a_site_hit_after_the_first_execution_is_born_built() { let first = js_regexp_new_site(make_string("bo+rn"), make_string(""), key); assert!( - unsafe { (*first).regex_ptr }.is_null(), + !regex_is_built(first), "construction must not build the program (that is #5777's deferred build)" ); assert!(js_regexp_test(first, make_string("boorn")) != 0); assert!( - !unsafe { (*first).regex_ptr }.is_null(), + regex_is_built(first), "the first execution installs the programs" ); // Second construction at the SAME site. let second = js_regexp_new_site(make_string("bo+rn"), make_string(""), key); assert!( - !unsafe { (*second).regex_ptr }.is_null(), + regex_is_built(second), "a site hit must install the programs the site already compiled, so the header is born \ built and the first match pays no lookup" ); From eefd58b3c324b77b039406ec8343a16a70b8ed61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 6 Sep 2026 22:23:56 +0200 Subject: [PATCH 03/27] perf(regex): tag the selected matcher on each header (cherry picked from commit 7a44e5948a2e51dca5f6de9eb76d639bf34de811) --- crates/perry-runtime/src/regex.rs | 53 +++++++++++++++---- crates/perry-runtime/src/regex/compile.rs | 2 + crates/perry-runtime/src/regex/lazy.rs | 1 + crates/perry-runtime/src/regex/site_cache.rs | 12 +++++ crates/perry-runtime/src/regex/tests_part2.rs | 32 +++++++++++ 5 files changed, 89 insertions(+), 11 deletions(-) diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 527e0c99b4..1350926d40 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -392,6 +392,7 @@ pub(crate) fn test_alloc_nursery_regexp_for_move(source: &str, flags: &str) -> * (*ptr).dot_all = flags.contains('s'); (*ptr).unicode = flags.contains('u') || flags.contains('v'); (*ptr).has_indices = flags.contains('d'); + (*ptr).matcher_kind = MatcherKind::Unbuilt; (*ptr).last_index = crate::value::JSValue::number(0.0).bits(); (*ptr).magic = REGEXP_MAGIC; @@ -488,6 +489,15 @@ mod compile_cache; #[cfg(feature = "regex-engine")] pub(crate) use compile_cache::*; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u8)] +pub(super) enum MatcherKind { + Unbuilt, + Standard, + Fancy, + Repeat, +} + /// Header for heap-allocated RegExp objects #[repr(C)] pub struct RegExpHeader { @@ -512,6 +522,9 @@ pub struct RegExpHeader { pub dot_all: bool, pub unicode: bool, pub has_indices: bool, + /// Selected engine after the first build. This occupies the byte that was + /// padding before `last_index`, so it does not grow the 56-byte header. + matcher_kind: MatcherKind, /// `lastIndex` is a writable data property holding an *arbitrary* JSValue /// (spec: `Set(R, "lastIndex", v)` with no coercion on write). Stored as the /// raw NaN-boxed bits; `exec`/`test` apply `ToLength` on read to derive the @@ -531,8 +544,8 @@ pub struct RegExpHeader { /// string pattern → never matches → get-intrinsic's `stringToPath` returns /// `[]` → `intrinsic %% does not exist!` → express adapter load `exit(1)`. /// - /// Storing the marker (and the fancy-regex Arc) ON the heap header makes - /// identity + fancy-fallback resolution independent of WHICH runtime copy's + /// Storing the marker and program-set handle ON the heap header makes + /// identity + fallback resolution independent of WHICH runtime copy's /// thread-locals are live. Set to `REGEXP_MAGIC` by `js_regexp_new`. pub magic: u64, /// #6759 phase 1 (header unification): per-object metadata record, or @@ -1206,6 +1219,7 @@ fn js_regexp_new_impl( (*ptr).dot_all = dot_all; (*ptr).unicode = unicode; (*ptr).has_indices = has_indices; + (*ptr).matcher_kind = MatcherKind::Unbuilt; (*ptr).last_index = crate::value::JSValue::number(0.0).bits(); // Wall 18: self-identifying marker so identity checks survive a // duplicate-runtime thread-local split. @@ -1214,6 +1228,7 @@ fn js_regexp_new_impl( // first execution of this text compiled. Install one owned reference; // null remains the sound not-built state. if let Some(programs) = programs { + (*ptr).matcher_kind = programs.matcher_kind(); (*ptr).programs_ptr = Arc::into_raw(programs); } @@ -1405,16 +1420,32 @@ pub(crate) fn regexp_test_str_bounded(re: *const RegExpHeader, hay: &str) -> Opt if crate::hot_diag::regex_on() { diag_note_op(re, crate::hot_diag::RegexOp::Test); } - if let Some(repeat_matcher) = lookup_repeat_matcher(re) { - return Some(repeat_matcher.regex.find(hay).is_some()); - } - if let Some(fre) = lookup_fancy_regex(re) { - return match fre.is_match(hay) { - Ok(v) => Some(v), - Err(_) => None, - }; + lazy::ensure_regex_compiled(re); + let programs = &*(*re).programs_ptr; + match (*re).matcher_kind { + MatcherKind::Repeat => { + let repeat = programs + .repeat + .as_ref() + .expect("repeat matcher tag must name a repeat program"); + Some(repeat.regex.find(hay).is_some()) + } + MatcherKind::Fancy => { + let fancy = programs + .fancy + .as_ref() + .expect("fancy matcher tag must name a fancy program"); + match fancy.is_match(hay) { + Ok(v) => Some(v), + Err(_) => None, + } + } + MatcherKind::Standard => Some(programs.std.is_match(hay)), + MatcherKind::Unbuilt => { + debug_assert!(false, "compiled header kept the unbuilt matcher tag"); + Some(programs.std.is_match(hay)) + } } - Some(lazy::header_std_regex(re).is_match(hay)) } } diff --git a/crates/perry-runtime/src/regex/compile.rs b/crates/perry-runtime/src/regex/compile.rs index c569ad8bb3..cb0aae6e43 100644 --- a/crates/perry-runtime/src/regex/compile.rs +++ b/crates/perry-runtime/src/regex/compile.rs @@ -166,6 +166,7 @@ pub extern "C" fn js_regexp_compile_value( .cloned() }); let programs = Arc::new(super::site_cache::Programs { std, fancy, repeat }); + let matcher_kind = programs.matcher_kind(); let programs_ptr = Arc::into_raw(programs); let (canonical_flags_ptr, _) = re_handle.across_mut::(|| js_string_from_str(flags_str)); @@ -176,6 +177,7 @@ pub extern "C" fn js_regexp_compile_value( }); unsafe { let old_programs_ptr = (*re).programs_ptr; + (*re).matcher_kind = matcher_kind; (*re).programs_ptr = programs_ptr; // Release the receiver's PREVIOUS owned references now that the new // ones are installed (recompiling the same pattern is fine: the fresh diff --git a/crates/perry-runtime/src/regex/lazy.rs b/crates/perry-runtime/src/regex/lazy.rs index e912e944cd..845f1fb20b 100644 --- a/crates/perry-runtime/src/regex/lazy.rs +++ b/crates/perry-runtime/src/regex/lazy.rs @@ -301,6 +301,7 @@ fn build_and_install_programs(re: *const RegExpHeader) { super::site_cache::install_programs(&pattern, &flags, programs.clone()); unsafe { let re = re as *mut RegExpHeader; + (*re).matcher_kind = programs.matcher_kind(); (*re).programs_ptr = Arc::into_raw(programs); } } diff --git a/crates/perry-runtime/src/regex/site_cache.rs b/crates/perry-runtime/src/regex/site_cache.rs index 459ad1f1de..675fe657df 100644 --- a/crates/perry-runtime/src/regex/site_cache.rs +++ b/crates/perry-runtime/src/regex/site_cache.rs @@ -47,6 +47,18 @@ pub(super) struct Programs { pub(super) repeat: Option>, } +impl Programs { + pub(super) fn matcher_kind(&self) -> super::MatcherKind { + if self.repeat.is_some() { + super::MatcherKind::Repeat + } else if self.fancy.is_some() { + super::MatcherKind::Fancy + } else { + super::MatcherKind::Standard + } + } +} + /// What a construction gets back on a hit. pub(super) struct Hit { pub(super) pattern: Arc, diff --git a/crates/perry-runtime/src/regex/tests_part2.rs b/crates/perry-runtime/src/regex/tests_part2.rs index c6ba025437..bf801c218f 100644 --- a/crates/perry-runtime/src/regex/tests_part2.rs +++ b/crates/perry-runtime/src/regex/tests_part2.rs @@ -237,6 +237,38 @@ fn deferred_build_installs_the_fancy_and_repeat_matcher_fallbacks() { ); } +/// Sabotage for the bounded Segmenter lane: this pattern's standard program +/// is the never-match placeholder, so a wrong `Standard` tag returns false. +/// Exercise all three installation routes that must publish the tag beside the +/// program handle: lazy build, born-built cache hit, and `compile`. +#[test] +fn bounded_test_matcher_tag_routes_fancy_patterns_to_fancy_regex() { + let _lock = crate::gc::global_side_table_test_lock(); + site_cache::test_reset(); + let pattern = r"(?<=left)right"; + + let cold = js_regexp_new(make_string(pattern), make_string("")); + assert_eq!(unsafe { (*cold).matcher_kind }, MatcherKind::Unbuilt); + assert_eq!(regexp_test_str_bounded(cold, "leftright"), Some(true)); + assert_eq!(unsafe { (*cold).matcher_kind }, MatcherKind::Fancy); + + let born_built = js_regexp_new(make_string(pattern), make_string("")); + assert!(regex_is_built(born_built)); + assert_eq!(unsafe { (*born_built).matcher_kind }, MatcherKind::Fancy); + assert_eq!( + regexp_test_str_bounded(born_built, "leftwrong"), + Some(false) + ); + + let compiled = js_regexp_new(make_string("plain"), make_string("")); + let pattern_value = crate::value::js_nanbox_string(make_string(pattern) as i64); + let flags_value = crate::value::js_nanbox_string(make_string("") as i64); + let result = js_regexp_compile_value(compiled, pattern_value, flags_value); + let compiled = crate::value::JSValue::from_bits(result.to_bits()).as_pointer::(); + assert_eq!(unsafe { (*compiled).matcher_kind }, MatcherKind::Fancy); + assert_eq!(regexp_test_str_bounded(compiled, "leftright"), Some(true)); +} + /// Two evaluations of the same pattern are still distinct objects with /// independent `lastIndex`, and deferring the build does not let them share a /// header (ECMA-262 requires a fresh object per evaluation — the same From 474236aa048b1e387fadcffa39d3a7a47bbb1a10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 6 Sep 2026 22:27:18 +0200 Subject: [PATCH 04/27] test(regex): isolate WTF-8 source from matcher parsing (cherry picked from commit 6ea7ad9ebe3a01caa56e8dc33e10afd3d574f585) --- crates/perry-runtime/src/regex/tests_part2.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/regex/tests_part2.rs b/crates/perry-runtime/src/regex/tests_part2.rs index bf801c218f..65086f719f 100644 --- a/crates/perry-runtime/src/regex/tests_part2.rs +++ b/crates/perry-runtime/src/regex/tests_part2.rs @@ -209,7 +209,11 @@ fn regexp_compile_replaces_the_header_source_and_flags() { #[test] fn regexp_source_round_trips_wtf8_lone_surrogates_from_the_header() { let lone_high = [b'a', 0xED, 0xA0, 0x80, b'/', b'b']; - let re = js_regexp_new(make_wtf8(&lone_high), make_string("")); + let re = js_regexp_new(make_string("placeholder"), make_string("")); + let pattern = make_wtf8(&lone_high); + unsafe { + (*re).pattern_ptr = pattern; + } let expected = [b'a', 0xED, 0xA0, 0x80, b'\\', b'/', b'b']; assert_eq!(string_payload(js_regexp_get_source(re)), expected); } From 39e520cd834b7fa4460ce843761785ef6759e1af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 6 Sep 2026 22:36:31 +0200 Subject: [PATCH 05/27] refactor(regex): split header properties and tests (cherry picked from commit c217a231c0e0133885ce105aa66de313f6c70be6) --- crates/perry-runtime/src/regex.rs | 80 +---- crates/perry-runtime/src/regex/properties.rs | 78 +++++ .../perry-runtime/src/regex/tests_header.rs | 281 ++++++++++++++++++ crates/perry-runtime/src/regex/tests_part2.rs | 254 ---------------- scripts/gc_runtime_root_holders.json | 7 - 5 files changed, 366 insertions(+), 334 deletions(-) create mode 100644 crates/perry-runtime/src/regex/properties.rs create mode 100644 crates/perry-runtime/src/regex/tests_header.rs diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 1350926d40..2c2614473b 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -62,6 +62,7 @@ mod grammar; mod lazy; #[cfg(feature = "regex-engine")] mod match_all; +mod properties; #[cfg(feature = "regex-engine")] mod repeat_matcher; #[cfg(feature = "regex-engine")] @@ -83,7 +84,6 @@ mod utf16; use class_range_validate::has_out_of_order_double_dash_class_range; #[cfg(feature = "regex-engine")] pub use compile::js_regexp_compile_value; -use escape::escape_regexp_source; pub use escape::js_regexp_escape; #[cfg(feature = "regex-engine")] use exec_array::{ @@ -105,6 +105,10 @@ pub(crate) use match_all::dispatch_regexp_string_iterator_method_builtin; pub use match_all::{ dispatch_regexp_string_iterator_method, js_string_match_all, js_string_match_all_value, }; +pub use properties::{ + js_regexp_empty_source, js_regexp_get_flags, js_regexp_get_last_index, js_regexp_get_source, + js_regexp_set_last_index, js_regexp_to_string, +}; /// Class id for `RegExp String Iterator` exotic objects. Referenced by the /// always-linked iterator-prototype dispatch, so it stays ungated even when @@ -1742,79 +1746,9 @@ pub(crate) fn test_last_exec_groups() -> usize { LAST_EXEC_GROUPS.with(|g| *g.borrow() as usize) } -/// Get regex.source — returns the pattern string -#[no_mangle] -pub extern "C" fn js_regexp_get_source(re: *const RegExpHeader) -> *mut StringHeader { - if !is_valid_regex_ptr(re) { - return js_string_from_str("(?:)"); - } - unsafe { - if is_valid_ptr((*re).pattern_ptr) { - let escaped = escape_regexp_source(string_as_bytes((*re).pattern_ptr)); - crate::string::js_string_from_wtf8_bytes(escaped.as_ptr(), escaped.len() as u32) - } else { - js_string_from_str("(?:)") - } - } -} - -/// `RegExp.prototype.source` for the prototype object itself (no -/// `[[OriginalSource]]`) returns the canonical empty source `"(?:)"`. -#[no_mangle] -pub extern "C" fn js_regexp_empty_source() -> *mut StringHeader { - js_string_from_str("(?:)") -} - -/// Get regex.flags — returns the flags string -#[no_mangle] -pub extern "C" fn js_regexp_get_flags(re: *const RegExpHeader) -> *mut StringHeader { - if !is_valid_regex_ptr(re) { - return js_string_from_str(""); - } - unsafe { - if is_valid_ptr((*re).flags_ptr) { - let flags_str = string_as_str((*re).flags_ptr); - js_string_from_str(flags_str) - } else { - js_string_from_str("") - } - } -} - -/// `RegExp.prototype.toString()` — `/source/flags`. Used by both the -/// `regex.toString()` method dispatch and ToString coercion (`String(re)`, -/// template literals). Node never produces `"[object Object]"` for a RegExp. -#[no_mangle] -pub extern "C" fn js_regexp_to_string(re: *const RegExpHeader) -> *mut StringHeader { - let src = js_regexp_get_source(re); - let flg = js_regexp_get_flags(re); - let out = format!("/{}/{}", string_as_str(src), string_as_str(flg)); - js_string_from_str(&out) -} - -/// Get regex.lastIndex — returns the stored value (NaN-boxed JSValue bits as -/// f64). Usually a number, but `re.lastIndex = obj` round-trips the object. -#[no_mangle] -pub extern "C" fn js_regexp_get_last_index(re: *const RegExpHeader) -> f64 { - if !is_valid_regex_ptr(re) { - return 0.0; - } - unsafe { f64::from_bits((*re).last_index) } -} - -/// Set regex.lastIndex — stores the value verbatim (no coercion on write, per -/// spec `Set(R, "lastIndex", v)`). -#[no_mangle] -pub extern "C" fn js_regexp_set_last_index(re: *mut RegExpHeader, value: f64) { - if !is_valid_regex_ptr(re) { - return; - } - unsafe { - (*re).last_index = value.to_bits(); - } -} - #[cfg(all(test, feature = "regex-engine"))] mod tests; #[cfg(all(test, feature = "regex-engine"))] mod tests_part2; +#[cfg(all(test, feature = "regex-engine"))] +mod tests_header; diff --git a/crates/perry-runtime/src/regex/properties.rs b/crates/perry-runtime/src/regex/properties.rs new file mode 100644 index 0000000000..e4f0a53f3e --- /dev/null +++ b/crates/perry-runtime/src/regex/properties.rs @@ -0,0 +1,78 @@ +//! Observable RegExp data properties and stringification. + +use super::escape::escape_regexp_source; +use super::RegExpHeader; +use super::{is_valid_ptr, is_valid_regex_ptr, js_string_from_str, string_as_bytes, string_as_str}; +use crate::string::StringHeader; + +/// Get regex.source — returns the pattern string. +#[no_mangle] +pub extern "C" fn js_regexp_get_source(re: *const RegExpHeader) -> *mut StringHeader { + if !is_valid_regex_ptr(re) { + return js_string_from_str("(?:)"); + } + unsafe { + if is_valid_ptr((*re).pattern_ptr) { + let escaped = escape_regexp_source(string_as_bytes((*re).pattern_ptr)); + crate::string::js_string_from_wtf8_bytes(escaped.as_ptr(), escaped.len() as u32) + } else { + js_string_from_str("(?:)") + } + } +} + +/// `RegExp.prototype.source` for the prototype object itself (no +/// `[[OriginalSource]]`) returns the canonical empty source `"(?:)"`. +#[no_mangle] +pub extern "C" fn js_regexp_empty_source() -> *mut StringHeader { + js_string_from_str("(?:)") +} + +/// Get regex.flags — returns the flags string. +#[no_mangle] +pub extern "C" fn js_regexp_get_flags(re: *const RegExpHeader) -> *mut StringHeader { + if !is_valid_regex_ptr(re) { + return js_string_from_str(""); + } + unsafe { + if is_valid_ptr((*re).flags_ptr) { + let flags_str = string_as_str((*re).flags_ptr); + js_string_from_str(flags_str) + } else { + js_string_from_str("") + } + } +} + +/// `RegExp.prototype.toString()` — `/source/flags`. Used by both the +/// `regex.toString()` method dispatch and ToString coercion (`String(re)`, +/// template literals). Node never produces `"[object Object]"` for a RegExp. +#[no_mangle] +pub extern "C" fn js_regexp_to_string(re: *const RegExpHeader) -> *mut StringHeader { + let src = js_regexp_get_source(re); + let flg = js_regexp_get_flags(re); + let out = format!("/{}/{}", string_as_str(src), string_as_str(flg)); + js_string_from_str(&out) +} + +/// Get regex.lastIndex — returns the stored value (NaN-boxed JSValue bits as +/// f64). Usually a number, but `re.lastIndex = obj` round-trips the object. +#[no_mangle] +pub extern "C" fn js_regexp_get_last_index(re: *const RegExpHeader) -> f64 { + if !is_valid_regex_ptr(re) { + return 0.0; + } + unsafe { f64::from_bits((*re).last_index) } +} + +/// Set regex.lastIndex — stores the value verbatim (no coercion on write, per +/// spec `Set(R, "lastIndex", v)`). +#[no_mangle] +pub extern "C" fn js_regexp_set_last_index(re: *mut RegExpHeader, value: f64) { + if !is_valid_regex_ptr(re) { + return; + } + unsafe { + (*re).last_index = value.to_bits(); + } +} diff --git a/crates/perry-runtime/src/regex/tests_header.rs b/crates/perry-runtime/src/regex/tests_header.rs new file mode 100644 index 0000000000..dcf8eacc17 --- /dev/null +++ b/crates/perry-runtime/src/regex/tests_header.rs @@ -0,0 +1,281 @@ +use super::*; + +fn make_string(s: &str) -> *mut StringHeader { + crate::string::js_string_from_bytes(s.as_ptr(), s.len() as u32) +} + +fn make_wtf8(bytes: &[u8]) -> *mut StringHeader { + crate::string::js_string_from_wtf8_bytes(bytes.as_ptr(), bytes.len() as u32) +} + +fn string_payload(s: *const StringHeader) -> Vec { + unsafe { + std::slice::from_raw_parts(crate::string::string_data(s), (*s).byte_len as usize).to_vec() + } +} + +fn regex_is_built(re: *const RegExpHeader) -> bool { + !unsafe { (*re).programs_ptr.is_null() } +} + +fn regex_has_fancy_program(re: *const RegExpHeader) -> bool { + regex_is_built(re) && unsafe { (*(*re).programs_ptr).fancy.is_some() } +} + +fn regex_has_repeat_program(re: *const RegExpHeader) -> bool { + regex_is_built(re) && unsafe { (*(*re).programs_ptr).repeat.is_some() } +} + +/// Construction must NOT build the automaton; the first operation that needs a +/// matcher must. +/// +/// This is the structural half of the perf fix — the wall-clock half is a +/// fixture whose 200 literals cost 73 ms to construct before and ~0 after. A +/// regression here (something re-introducing an eager build) would not fail any +/// behavioural test, only make every program slower, so assert the state +/// directly: `programs_ptr` is the built/not-built flag. +#[test] +fn construction_defers_the_program_build_until_first_use() { + let re = js_regexp_new( + make_string("[A-Za-z]+(?:foo|bar)[0-9]{1,4}"), + make_string("i"), + ); + assert!( + !regex_is_built(re), + "constructing a RegExp must not build its program" + ); + // Everything observable without matching stays available. + assert_eq!( + string_payload(js_regexp_get_source(re)), + b"[A-Za-z]+(?:foo|bar)[0-9]{1,4}".to_vec() + ); + assert_eq!(string_payload(js_regexp_get_flags(re)), b"i".to_vec()); + assert!(unsafe { (*re).case_insensitive }); + assert!( + !regex_is_built(re), + "reading .source/.flags must not build the program either" + ); + + assert!(js_regexp_test(re, make_string("XFOO12")) != 0); + assert!( + regex_is_built(re), + "the first match must build and install the program" + ); +} + +/// Removing the address-keyed source table must not turn the RegExp-pattern +/// constructor arm into an empty-pattern fallback. This calls the exported +/// constructor entry point, so deleting its direct header read fails both the +/// source and inherited-flags assertions. +#[test] +fn regexp_construct_reads_source_and_flags_from_the_pattern_header() { + let original = js_regexp_new(make_string("left/right"), make_string("ig")); + let pattern = crate::value::js_nanbox_pointer(original as i64); + let undefined = f64::from_bits(crate::value::TAG_UNDEFINED); + + let copy = js_regexp_construct(pattern, undefined); + assert_ne!( + copy, original, + "construction must still allocate a fresh object" + ); + assert_eq!(string_payload(js_regexp_get_source(copy)), b"left\\/right"); + assert_eq!(string_payload(js_regexp_get_flags(copy)), b"gi"); + + let override_flags = crate::value::js_nanbox_string(make_string("m") as i64); + let overridden = js_regexp_construct(pattern, override_flags); + assert_eq!( + string_payload(js_regexp_get_source(overridden)), + b"left\\/right" + ); + assert_eq!(string_payload(js_regexp_get_flags(overridden)), b"m"); +} + +/// `RegExp.prototype.compile` rewrites the header in place. The source table +/// used to mask a stale header slot here; after its removal both observable +/// strings must come from the newly stored, traced edges. +#[test] +fn regexp_compile_replaces_the_header_source_and_flags() { + let receiver = js_regexp_new(make_string("old"), make_string("m")); + js_regexp_set_last_index(receiver, 9.0); + let pattern = crate::value::js_nanbox_string(make_string("new/source") as i64); + let flags = crate::value::js_nanbox_string(make_string("ig") as i64); + let result = js_regexp_compile_value(receiver, pattern, flags); + let receiver = crate::value::JSValue::from_bits(result.to_bits()).as_pointer::(); + + assert_eq!( + string_payload(js_regexp_get_source(receiver)), + b"new\\/source" + ); + assert_eq!(string_payload(js_regexp_get_flags(receiver)), b"gi"); + assert_eq!(js_regexp_get_last_index(receiver), 0.0); + assert_eq!(js_regexp_test(receiver, make_string("NEW/source")), 1); +} + +/// Perry stores lone JavaScript surrogates as WTF-8. `.source` must copy those +/// exact bytes from the traced pattern slot; routing them through Rust's UTF-8 +/// scalar iterator either replaces the surrogate or invokes undefined +/// behaviour. +#[test] +fn regexp_source_round_trips_wtf8_lone_surrogates_from_the_header() { + let lone_high = [b'a', 0xED, 0xA0, 0x80, b'/', b'b']; + let re = js_regexp_new(make_string("placeholder"), make_string("")); + let pattern = make_wtf8(&lone_high); + unsafe { + (*re).pattern_ptr = pattern; + } + let expected = [b'a', 0xED, 0xA0, 0x80, b'\\', b'/', b'b']; + assert_eq!(string_payload(js_regexp_get_source(re)), expected); +} + +/// The deferred build installs the fancy-regex and RepeatMatcher programs too, +/// not just the linear one — they live on the same publish point, so a header +/// whose pattern needs one must still get it on first use. +#[test] +fn deferred_build_installs_the_fancy_and_repeat_matcher_fallbacks() { + let fancy = js_regexp_new(make_string(r"(?<=pre)\d+"), make_string("")); + assert!(!regex_is_built(fancy)); + assert!(js_regexp_test(fancy, make_string("pre77")) != 0); + assert!( + regex_has_fancy_program(fancy), + "first use must install the fancy-regex fallback" + ); + assert!(js_regexp_test(fancy, make_string("nope77")) == 0); + + let repeat = js_regexp_new(make_string(r"(a?b??)*"), make_string("")); + assert!(!regex_is_built(repeat)); + assert!(js_regexp_test(repeat, make_string("ab")) != 0); + assert!( + regex_has_repeat_program(repeat), + "first use must install the ECMAScript RepeatMatcher" + ); +} + +/// Sabotage for the bounded Segmenter lane: this pattern's standard program +/// is the never-match placeholder, so a wrong `Standard` tag returns false. +/// Exercise all three installation routes that must publish the tag beside the +/// program handle: lazy build, born-built cache hit, and `compile`. +#[test] +fn bounded_test_matcher_tag_routes_fancy_patterns_to_fancy_regex() { + let _lock = crate::gc::global_side_table_test_lock(); + site_cache::test_reset(); + let pattern = r"(?<=left)right"; + + let cold = js_regexp_new(make_string(pattern), make_string("")); + assert_eq!(unsafe { (*cold).matcher_kind }, MatcherKind::Unbuilt); + assert_eq!(regexp_test_str_bounded(cold, "leftright"), Some(true)); + assert_eq!(unsafe { (*cold).matcher_kind }, MatcherKind::Fancy); + + let born_built = js_regexp_new(make_string(pattern), make_string("")); + assert!(regex_is_built(born_built)); + assert_eq!(unsafe { (*born_built).matcher_kind }, MatcherKind::Fancy); + assert_eq!( + regexp_test_str_bounded(born_built, "leftwrong"), + Some(false) + ); + + let compiled = js_regexp_new(make_string("plain"), make_string("")); + let pattern_value = crate::value::js_nanbox_string(make_string(pattern) as i64); + let flags_value = crate::value::js_nanbox_string(make_string("") as i64); + let result = js_regexp_compile_value(compiled, pattern_value, flags_value); + let compiled = crate::value::JSValue::from_bits(result.to_bits()).as_pointer::(); + assert_eq!(unsafe { (*compiled).matcher_kind }, MatcherKind::Fancy); + assert_eq!(regexp_test_str_bounded(compiled, "leftright"), Some(true)); +} + +/// Two evaluations of the same pattern are still distinct objects with +/// independent `lastIndex`, and deferring the build does not let them share a +/// header (ECMA-262 requires a fresh object per evaluation — the same +/// invariant the closure-literal singleton fix restored for functions). +#[test] +fn deferred_build_keeps_per_object_identity_and_last_index() { + let a = js_regexp_new(make_string("x"), make_string("g")); + let b = js_regexp_new(make_string("x"), make_string("g")); + assert_ne!( + a as usize, b as usize, + "each evaluation is a distinct object" + ); + assert!(!js_regexp_exec(a, make_string("xx")).is_null()); + assert_eq!(regex_last_index_offset(a), 1); + assert_eq!( + regex_last_index_offset(b), + 0, + "a sibling regex must not inherit lastIndex through the shared program" + ); +} + +/// The validated-pattern set is capped like the program caches: it holds owned +/// pattern text (`emoji-regex` is ~12,807 chars) and is fed by `new +/// RegExp(userInput)`, so an uncapped one would be the same attacker-driven +/// growth the compiled-program caches were capped for. +#[test] +fn validated_pattern_set_is_capped() { + for i in 0..(REGEX_CACHE_MAX_ENTRIES * 2 + 10) { + lazy::mark_pattern_validated(&format!("validfill{i}[a-z]+"), ""); + } + let len = VALIDATED_PATTERNS.with(|c| c.borrow().len()); + assert!( + len <= REGEX_CACHE_MAX_ENTRIES, + "VALIDATED_PATTERNS must stay capped at {REGEX_CACHE_MAX_ENTRIES} entries, got {len}" + ); +} + +/// The `[\s\S]` → `(?s:.)` rewrite must not move a single match result. +/// +/// The rewrite exists purely to dodge a 1.1-million-iteration case fold in +/// `regex_syntax` (see `grammar::push_any_char`), so the only thing that may +/// change is how long construction takes. Everything a program can observe — +/// what matches, what a capture group holds, which group number it is, and +/// that the NEGATED forms still match nothing — is pinned here, because a +/// silently widened character class produces no error anywhere: only a wrong +/// answer, on inputs a syntax test never looks at. +#[test] +fn any_char_rewrite_preserves_match_behaviour() { + // Matches every code point, newlines included, with and without `i`. + for pattern in ["[\\s\\S]", "[^]", "[\\d\\D]", "[\\w\\W]", "[\\S\\s]"] { + for flags in ["", "i", "u", "iu", "m"] { + let re = js_regexp_new(make_string(pattern), make_string(flags)); + for subject in ["a", "\n", " ", "\u{1F600}", "Ω", "\r"] { + assert!( + js_regexp_test(re, make_string(subject)) != 0, + "/{pattern}/{flags} must match {subject:?}" + ); + } + } + } + + // The negated forms are the exact opposite and must still match NOTHING. + for pattern in ["[^\\s\\S]", "[^\\w\\W]", "[]"] { + let re = js_regexp_new(make_string(pattern), make_string("i")); + for subject in ["a", "\n", "Ω"] { + assert!( + js_regexp_test(re, make_string(subject)) == 0, + "/{pattern}/i must not match {subject:?}" + ); + } + } + + // A class that is NOT a complementary pair keeps its narrow meaning. + let narrow = js_regexp_new(make_string("[\\d\\s]"), make_string("i")); + assert!(js_regexp_test(narrow, make_string("7")) != 0); + assert!(js_regexp_test(narrow, make_string("a")) == 0); + + // The rewrite emits a NON-capturing group, so group numbering is + // unchanged: `$1` is still `b`, not the any-char. + let re = js_regexp_new(make_string("a[\\s\\S](b)"), make_string("")); + let m = js_regexp_exec(re, make_string("a\nb")); + assert!(!m.is_null(), "a[\\s\\S](b) must match \"a\\nb\""); + + // Quantifiers still bind to the any-char, lazily and greedily. + let lazy = js_regexp_new(make_string("([\\s\\S]*?)"), make_string("i")); + assert!(js_regexp_test(lazy, make_string("one\ntwo")) != 0); + let greedy = js_regexp_new(make_string("^[\\s\\S]{3}$"), make_string("")); + assert!(js_regexp_test(greedy, make_string("a\nb")) != 0); + assert!(js_regexp_test(greedy, make_string("a\nbc")) == 0); + + // `.source` still reports what the author wrote, not the translation. + let re = js_regexp_new(make_string("[\\s\\S]+"), make_string("gi")); + assert_eq!( + string_payload(js_regexp_get_source(re)), + b"[\\s\\S]+".to_vec() + ); +} diff --git a/crates/perry-runtime/src/regex/tests_part2.rs b/crates/perry-runtime/src/regex/tests_part2.rs index 65086f719f..e8cda486be 100644 --- a/crates/perry-runtime/src/regex/tests_part2.rs +++ b/crates/perry-runtime/src/regex/tests_part2.rs @@ -117,260 +117,6 @@ fn syntax_check_agrees_with_full_build() { } } -/// Construction must NOT build the automaton; the first operation that needs a -/// matcher must. -/// -/// This is the structural half of the perf fix — the wall-clock half is a -/// fixture whose 200 literals cost 73 ms to construct before and ~0 after. A -/// regression here (something re-introducing an eager build) would not fail any -/// behavioural test, only make every program slower, so assert the state -/// directly: `programs_ptr` is the built/not-built flag. -#[test] -fn construction_defers_the_program_build_until_first_use() { - let re = js_regexp_new( - make_string("[A-Za-z]+(?:foo|bar)[0-9]{1,4}"), - make_string("i"), - ); - assert!( - !regex_is_built(re), - "constructing a RegExp must not build its program" - ); - // Everything observable without matching stays available. - assert_eq!( - string_payload(js_regexp_get_source(re)), - b"[A-Za-z]+(?:foo|bar)[0-9]{1,4}".to_vec() - ); - assert_eq!(string_payload(js_regexp_get_flags(re)), b"i".to_vec()); - assert!(unsafe { (*re).case_insensitive }); - assert!( - !regex_is_built(re), - "reading .source/.flags must not build the program either" - ); - - assert!(js_regexp_test(re, make_string("XFOO12")) != 0); - assert!( - regex_is_built(re), - "the first match must build and install the program" - ); -} - -/// Removing the address-keyed source table must not turn the RegExp-pattern -/// constructor arm into an empty-pattern fallback. This calls the exported -/// constructor entry point, so deleting its direct header read fails both the -/// source and inherited-flags assertions. -#[test] -fn regexp_construct_reads_source_and_flags_from_the_pattern_header() { - let original = js_regexp_new(make_string("left/right"), make_string("ig")); - let pattern = crate::value::js_nanbox_pointer(original as i64); - let undefined = f64::from_bits(crate::value::TAG_UNDEFINED); - - let copy = js_regexp_construct(pattern, undefined); - assert_ne!( - copy, original, - "construction must still allocate a fresh object" - ); - assert_eq!(string_payload(js_regexp_get_source(copy)), b"left\\/right"); - assert_eq!(string_payload(js_regexp_get_flags(copy)), b"gi"); - - let override_flags = crate::value::js_nanbox_string(make_string("m") as i64); - let overridden = js_regexp_construct(pattern, override_flags); - assert_eq!( - string_payload(js_regexp_get_source(overridden)), - b"left\\/right" - ); - assert_eq!(string_payload(js_regexp_get_flags(overridden)), b"m"); -} - -/// `RegExp.prototype.compile` rewrites the header in place. The source table -/// used to mask a stale header slot here; after its removal both observable -/// strings must come from the newly stored, traced edges. -#[test] -fn regexp_compile_replaces_the_header_source_and_flags() { - let receiver = js_regexp_new(make_string("old"), make_string("m")); - js_regexp_set_last_index(receiver, 9.0); - let pattern = crate::value::js_nanbox_string(make_string("new/source") as i64); - let flags = crate::value::js_nanbox_string(make_string("ig") as i64); - let result = js_regexp_compile_value(receiver, pattern, flags); - let receiver = crate::value::JSValue::from_bits(result.to_bits()).as_pointer::(); - - assert_eq!( - string_payload(js_regexp_get_source(receiver)), - b"new\\/source" - ); - assert_eq!(string_payload(js_regexp_get_flags(receiver)), b"gi"); - assert_eq!(js_regexp_get_last_index(receiver), 0.0); - assert_eq!(js_regexp_test(receiver, make_string("NEW/source")), 1); -} - -/// Perry stores lone JavaScript surrogates as WTF-8. `.source` must copy those -/// exact bytes from the traced pattern slot; routing them through Rust's UTF-8 -/// scalar iterator either replaces the surrogate or invokes undefined -/// behaviour. -#[test] -fn regexp_source_round_trips_wtf8_lone_surrogates_from_the_header() { - let lone_high = [b'a', 0xED, 0xA0, 0x80, b'/', b'b']; - let re = js_regexp_new(make_string("placeholder"), make_string("")); - let pattern = make_wtf8(&lone_high); - unsafe { - (*re).pattern_ptr = pattern; - } - let expected = [b'a', 0xED, 0xA0, 0x80, b'\\', b'/', b'b']; - assert_eq!(string_payload(js_regexp_get_source(re)), expected); -} - -/// The deferred build installs the fancy-regex and RepeatMatcher programs too, -/// not just the linear one — they live on the same publish point, so a header -/// whose pattern needs one must still get it on first use. -#[test] -fn deferred_build_installs_the_fancy_and_repeat_matcher_fallbacks() { - let fancy = js_regexp_new(make_string(r"(?<=pre)\d+"), make_string("")); - assert!(!regex_is_built(fancy)); - assert!(js_regexp_test(fancy, make_string("pre77")) != 0); - assert!( - regex_has_fancy_program(fancy), - "first use must install the fancy-regex fallback" - ); - assert!(js_regexp_test(fancy, make_string("nope77")) == 0); - - let repeat = js_regexp_new(make_string(r"(a?b??)*"), make_string("")); - assert!(!regex_is_built(repeat)); - assert!(js_regexp_test(repeat, make_string("ab")) != 0); - assert!( - regex_has_repeat_program(repeat), - "first use must install the ECMAScript RepeatMatcher" - ); -} - -/// Sabotage for the bounded Segmenter lane: this pattern's standard program -/// is the never-match placeholder, so a wrong `Standard` tag returns false. -/// Exercise all three installation routes that must publish the tag beside the -/// program handle: lazy build, born-built cache hit, and `compile`. -#[test] -fn bounded_test_matcher_tag_routes_fancy_patterns_to_fancy_regex() { - let _lock = crate::gc::global_side_table_test_lock(); - site_cache::test_reset(); - let pattern = r"(?<=left)right"; - - let cold = js_regexp_new(make_string(pattern), make_string("")); - assert_eq!(unsafe { (*cold).matcher_kind }, MatcherKind::Unbuilt); - assert_eq!(regexp_test_str_bounded(cold, "leftright"), Some(true)); - assert_eq!(unsafe { (*cold).matcher_kind }, MatcherKind::Fancy); - - let born_built = js_regexp_new(make_string(pattern), make_string("")); - assert!(regex_is_built(born_built)); - assert_eq!(unsafe { (*born_built).matcher_kind }, MatcherKind::Fancy); - assert_eq!( - regexp_test_str_bounded(born_built, "leftwrong"), - Some(false) - ); - - let compiled = js_regexp_new(make_string("plain"), make_string("")); - let pattern_value = crate::value::js_nanbox_string(make_string(pattern) as i64); - let flags_value = crate::value::js_nanbox_string(make_string("") as i64); - let result = js_regexp_compile_value(compiled, pattern_value, flags_value); - let compiled = crate::value::JSValue::from_bits(result.to_bits()).as_pointer::(); - assert_eq!(unsafe { (*compiled).matcher_kind }, MatcherKind::Fancy); - assert_eq!(regexp_test_str_bounded(compiled, "leftright"), Some(true)); -} - -/// Two evaluations of the same pattern are still distinct objects with -/// independent `lastIndex`, and deferring the build does not let them share a -/// header (ECMA-262 requires a fresh object per evaluation — the same -/// invariant the closure-literal singleton fix restored for functions). -#[test] -fn deferred_build_keeps_per_object_identity_and_last_index() { - let a = js_regexp_new(make_string("x"), make_string("g")); - let b = js_regexp_new(make_string("x"), make_string("g")); - assert_ne!( - a as usize, b as usize, - "each evaluation is a distinct object" - ); - assert!(!js_regexp_exec(a, make_string("xx")).is_null()); - assert_eq!(regex_last_index_offset(a), 1); - assert_eq!( - regex_last_index_offset(b), - 0, - "a sibling regex must not inherit lastIndex through the shared program" - ); -} - -/// The validated-pattern set is capped like the program caches: it holds owned -/// pattern text (`emoji-regex` is ~12,807 chars) and is fed by `new -/// RegExp(userInput)`, so an uncapped one would be the same attacker-driven -/// growth the compiled-program caches were capped for. -#[test] -fn validated_pattern_set_is_capped() { - for i in 0..(REGEX_CACHE_MAX_ENTRIES * 2 + 10) { - lazy::mark_pattern_validated(&format!("validfill{i}[a-z]+"), ""); - } - let len = VALIDATED_PATTERNS.with(|c| c.borrow().len()); - assert!( - len <= REGEX_CACHE_MAX_ENTRIES, - "VALIDATED_PATTERNS must stay capped at {REGEX_CACHE_MAX_ENTRIES} entries, got {len}" - ); -} - -/// The `[\s\S]` → `(?s:.)` rewrite must not move a single match result. -/// -/// The rewrite exists purely to dodge a 1.1-million-iteration case fold in -/// `regex_syntax` (see `grammar::push_any_char`), so the only thing that may -/// change is how long construction takes. Everything a program can observe — -/// what matches, what a capture group holds, which group number it is, and -/// that the NEGATED forms still match nothing — is pinned here, because a -/// silently widened character class produces no error anywhere: only a wrong -/// answer, on inputs a syntax test never looks at. -#[test] -fn any_char_rewrite_preserves_match_behaviour() { - // Matches every code point, newlines included, with and without `i`. - for pattern in ["[\\s\\S]", "[^]", "[\\d\\D]", "[\\w\\W]", "[\\S\\s]"] { - for flags in ["", "i", "u", "iu", "m"] { - let re = js_regexp_new(make_string(pattern), make_string(flags)); - for subject in ["a", "\n", " ", "\u{1F600}", "Ω", "\r"] { - assert!( - js_regexp_test(re, make_string(subject)) != 0, - "/{pattern}/{flags} must match {subject:?}" - ); - } - } - } - - // The negated forms are the exact opposite and must still match NOTHING. - for pattern in ["[^\\s\\S]", "[^\\w\\W]", "[]"] { - let re = js_regexp_new(make_string(pattern), make_string("i")); - for subject in ["a", "\n", "Ω"] { - assert!( - js_regexp_test(re, make_string(subject)) == 0, - "/{pattern}/i must not match {subject:?}" - ); - } - } - - // A class that is NOT a complementary pair keeps its narrow meaning. - let narrow = js_regexp_new(make_string("[\\d\\s]"), make_string("i")); - assert!(js_regexp_test(narrow, make_string("7")) != 0); - assert!(js_regexp_test(narrow, make_string("a")) == 0); - - // The rewrite emits a NON-capturing group, so group numbering is - // unchanged: `$1` is still `b`, not the any-char. - let re = js_regexp_new(make_string("a[\\s\\S](b)"), make_string("")); - let m = js_regexp_exec(re, make_string("a\nb")); - assert!(!m.is_null(), "a[\\s\\S](b) must match \"a\\nb\""); - - // Quantifiers still bind to the any-char, lazily and greedily. - let lazy = js_regexp_new(make_string("([\\s\\S]*?)"), make_string("i")); - assert!(js_regexp_test(lazy, make_string("one\ntwo")) != 0); - let greedy = js_regexp_new(make_string("^[\\s\\S]{3}$"), make_string("")); - assert!(js_regexp_test(greedy, make_string("a\nb")) != 0); - assert!(js_regexp_test(greedy, make_string("a\nbc")) == 0); - - // `.source` still reports what the author wrote, not the translation. - let re = js_regexp_new(make_string("[\\s\\S]+"), make_string("gi")); - assert_eq!( - string_payload(js_regexp_get_source(re)), - b"[\\s\\S]+".to_vec() - ); -} - /// #9305 fallout: the translator spells ECMAScript's ASCII `\b`/`\B` as /// `(?-iu:\b)`, which fancy-regex's parser rejects (`NonUnicodeUnsupported`). /// Any lookaround/backreference pattern containing a word boundary therefore diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 377b5a96b7..abab4f7215 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -734,13 +734,6 @@ "scanner": "regex::regex_header_moved_for_gc / regex_header_clear_dead_for_gc (regex.rs), the RegExp move/death hooks the copying minor and sweep invoke", "why": "Address-KEYED owner set, not a root: the key is rekeyed when the RegExpHeader moves and removed when it dies; it never keeps the header alive. Reached from GC hooks, not from a registered scanner, so the walk misses it." }, - { - "file": "crates/perry-runtime/src/regex.rs", - "name": "REGEX_SOURCE_TABLE", - "verdict": "covered_elsewhere", - "scanner": "regex::regex_header_moved_for_gc / regex_header_clear_dead_for_gc (regex.rs)", - "why": "Address-KEYED owner table (owned String copies of pattern/flags); rekeyed on move, cleared on death, same hooks as REGEX_POINTERS." - }, { "file": "crates/perry-runtime/src/regex.rs", "name": "VALIDATED_PATTERNS", From f25a712768da4eaf9c2151f16a41c66f23f42567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 6 Sep 2026 22:39:57 +0200 Subject: [PATCH 06/27] fix(regex): retain canonical flags through allocation (cherry picked from commit 883d334a68f1159903922476090549e1097bd346) --- crates/perry-runtime/src/gc/types.rs | 6 +++--- crates/perry-runtime/src/regex.rs | 8 ++++---- crates/perry-runtime/src/regex/match_all.rs | 2 +- crates/perry-runtime/src/regex/replace_expand.rs | 2 +- crates/perry-runtime/src/regex/replace_expand_fancy.rs | 4 ++-- crates/perry-runtime/src/string/split.rs | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/perry-runtime/src/gc/types.rs b/crates/perry-runtime/src/gc/types.rs index 08f9832cd5..97129c6088 100644 --- a/crates/perry-runtime/src/gc/types.rs +++ b/crates/perry-runtime/src/gc/types.rs @@ -224,9 +224,9 @@ pub(crate) enum GcMoveHookKind { /// live on the Error's traced `ObjectMeta` edge and need no side-table /// rekeying. ErrorSideTables, - /// Rekey RegExp identity/source registries plus its exotic expando owner - /// entry. `GC_TYPE_REGEXP` is movable, and all three tables use the - /// payload address as their key. + /// Rekey the RegExp identity registry plus its exotic expando owner entry. + /// `GC_TYPE_REGEXP` is movable, and both tables use the payload address as + /// their key. RegExpSideTables, } diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 2c2614473b..3170cdd277 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -788,7 +788,7 @@ fn js_regexp_new_impl( // A `site_key` of 0 (every dynamic construction, and every runtime caller) // misses by construction and takes the content-keyed path below unchanged. let site_entry = site_key::lookup(site_key, raw_flags_str); - let (programs, bits, shared_flags_root) = match site_entry { + let (programs, bits, shared_flags_root, owned_flags) = match site_entry { Some(hit) => { // The site's own flags literal, so this is the same sharing // decision the first construction at this site made (#9819). @@ -822,7 +822,7 @@ fn js_regexp_new_impl( picked } }; - (programs, hit.bits, shared_flags_root) + (programs, hit.bits, shared_flags_root, hit.flags) } None => { let pattern_str = if is_valid_ptr(pattern) { @@ -1041,12 +1041,12 @@ fn js_regexp_new_impl( site_key, raw_flags_owned, owned_pattern, - owned_flags, + owned_flags.clone(), flags_are_canonical, bits, programs.clone(), ); - (programs, bits, shared_flags_root) + (programs, bits, shared_flags_root, owned_flags) } }; let site_key::FlagBits { diff --git a/crates/perry-runtime/src/regex/match_all.rs b/crates/perry-runtime/src/regex/match_all.rs index 998e2ad83f..f533dd4955 100644 --- a/crates/perry-runtime/src/regex/match_all.rs +++ b/crates/perry-runtime/src/regex/match_all.rs @@ -87,7 +87,7 @@ unsafe fn materialize_match_all_results( // Phase 1 (borrowing, no JS allocation): snapshot every match into owned // Rust data. The fancy-regex fallback (lookbehind/backreferences) is - // needed because the never-match placeholder in `regex_ptr` would yield + // needed because the never-match standard program would yield // an empty iterator otherwise. // The scan starts AT `search_start` inside the whole subject — never on a // `&str_data[search_start..]` slice, which would strip the context every diff --git a/crates/perry-runtime/src/regex/replace_expand.rs b/crates/perry-runtime/src/regex/replace_expand.rs index 89391be8aa..19214cce1a 100644 --- a/crates/perry-runtime/src/regex/replace_expand.rs +++ b/crates/perry-runtime/src/regex/replace_expand.rs @@ -485,7 +485,7 @@ pub extern "C" fn js_string_replace_regex_named( // Fancy-regex fallback (lookbehind/backreferences): expand `$` // and friends against the fancy captures instead of the never-match - // placeholder stored in `regex_ptr`. + // placeholder stored as the standard program. if let Some(fre) = lookup_fancy_regex(re) { return replace_regex_str_fancy(str_data, &fre, (*re).global, repl_str); } diff --git a/crates/perry-runtime/src/regex/replace_expand_fancy.rs b/crates/perry-runtime/src/regex/replace_expand_fancy.rs index 22e78cee6a..9106ade261 100644 --- a/crates/perry-runtime/src/regex/replace_expand_fancy.rs +++ b/crates/perry-runtime/src/regex/replace_expand_fancy.rs @@ -197,7 +197,7 @@ pub extern "C" fn js_string_replace_regex( // Pattern the `regex` crate couldn't compile (lookbehind/backreferences) // → drive the replacement through fancy-regex. Otherwise the never-match - // placeholder in `regex_ptr` would leave the input unchanged. + // placeholder standard program would leave the input unchanged. if let Some(fre) = lookup_fancy_regex(re) { return replace_regex_str_fancy(str_data, &fre, (*re).global, repl_str); } @@ -369,7 +369,7 @@ pub extern "C" fn js_string_search_regex(s: *const StringHeader, re: *const RegE } // Fancy-regex fallback (lookbehind/backreferences): the never-match - // placeholder in `regex_ptr` would always report -1 otherwise. + // placeholder standard program would always report -1 otherwise. if let Some(fre) = lookup_fancy_regex(re) { return match fre.find(str_data) { Ok(Some(m)) => byte_index_to_utf16_index(str_data, m.start()) as i32, diff --git a/crates/perry-runtime/src/string/split.rs b/crates/perry-runtime/src/string/split.rs index addeb94e55..286fccf888 100644 --- a/crates/perry-runtime/src/string/split.rs +++ b/crates/perry-runtime/src/string/split.rs @@ -460,7 +460,7 @@ pub extern "C" fn js_string_split_n( // both. Detect regex delimiters by checking whether the pointer was // recorded by `js_regexp_new` and delegate to `js_string_split_regex` // on a match. Otherwise the regex header would be read as a - // StringHeader and segfault on the first byte of its `regex_ptr`. + // StringHeader and segfault on the first byte of its program-set pointer. #[cfg(feature = "regex-engine")] if crate::regex::is_regex_pointer(delimiter as *const u8) { return crate::regex::js_string_split_regex_n( From 13e0cf6b53036a24622bd3813e646b48e822e9a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 02:29:14 +0200 Subject: [PATCH 07/27] perf(regex): preserve live literal programs on eviction Replace whole-map overflow clears with one-entry eviction, and keep content-cache entries pinned while a recorded literal site refers to them. Only dynamic or displaced-site entries can leave the bounded table. Add a sabotage test that crosses both cache bounds, collects dead nursery headers, and proves the recorded literal does not rebuild. (cherry picked from commit ce9e12801e8d83fae471e06cc85429257ac10854) --- changelog.d/9918-regex-cache-eviction.md | 1 + crates/perry-runtime/src/hot_diag.rs | 37 ++- crates/perry-runtime/src/regex.rs | 8 +- .../perry-runtime/src/regex/compile_cache.rs | 29 +-- crates/perry-runtime/src/regex/lazy.rs | 14 +- crates/perry-runtime/src/regex/site_cache.rs | 214 +++++++++--------- crates/perry-runtime/src/regex/site_key.rs | 37 ++- crates/perry-runtime/src/regex/tests.rs | 2 +- crates/perry-runtime/src/regex/tests_cache.rs | 139 ++++++++++++ 9 files changed, 351 insertions(+), 130 deletions(-) create mode 100644 changelog.d/9918-regex-cache-eviction.md create mode 100644 crates/perry-runtime/src/regex/tests_cache.rs diff --git a/changelog.d/9918-regex-cache-eviction.md b/changelog.d/9918-regex-cache-eviction.md new file mode 100644 index 0000000000..29fd1168f7 --- /dev/null +++ b/changelog.d/9918-regex-cache-eviction.md @@ -0,0 +1 @@ +Keep compiled programs for recorded regular-expression literal sites across bounded cache eviction, and replace whole-cache overflow clears with one-entry eviction. diff --git a/crates/perry-runtime/src/hot_diag.rs b/crates/perry-runtime/src/hot_diag.rs index dcfd44a6dc..10f063eebe 100644 --- a/crates/perry-runtime/src/hot_diag.rs +++ b/crates/perry-runtime/src/hot_diag.rs @@ -136,6 +136,9 @@ pub struct RegexDiag { pub compiles_std: u64, pub compiles_fancy: u64, pub compiles_repeat: u64, + /// One-entry evictions after a regex cache reaches its bound. The former + /// wholesale-clear counter remains as a zeroed regression control. + pub cache_evictions: u64, pub cache_clears: u64, /// `lazy::build_and_install_programs` runs (one per header that is /// executed at least once). @@ -200,6 +203,10 @@ pub struct RegexDiag { /// CONTENT-keyed cache; a site hit never reaches it, so the two are /// disjoint and `site_key_hit + site_hit <= new`. pub new_site_key_hit: u64, + #[cfg(test)] + test_program_builds: u64, + #[cfg(test)] + test_cache_evictions: u64, per_pattern: HashMap, } @@ -262,6 +269,33 @@ pub fn regex_with(f: impl FnOnce(&mut RegexDiag)) { }); } +#[cfg(test)] +pub(crate) fn test_reset_regex_builds_and_evictions() { + REGEX_DIAG.with(|diag| { + let mut diag = diag.borrow_mut(); + diag.test_program_builds = 0; + diag.test_cache_evictions = 0; + }); +} + +#[cfg(test)] +pub(crate) fn test_note_regex_program_build() { + REGEX_DIAG.with(|diag| diag.borrow_mut().test_program_builds += 1); +} + +#[cfg(test)] +pub(crate) fn test_note_regex_cache_eviction() { + REGEX_DIAG.with(|diag| diag.borrow_mut().test_cache_evictions += 1); +} + +#[cfg(test)] +pub(crate) fn test_regex_builds_and_evictions() -> (u64, u64) { + REGEX_DIAG.with(|diag| { + let diag = diag.borrow(); + (diag.test_program_builds, diag.test_cache_evictions) + }) +} + impl RegexDiag { fn pat(&mut self, pattern_addr: usize, pattern: &[u8], flags: &str) -> &mut PatStat { let entry = self.per_pattern.entry(pattern_addr).or_default(); @@ -341,7 +375,7 @@ impl RegexDiag { let _ = writeln!( out, "[regex-diag] t={secs:.1}s new={} validated_hit={} site_hit={} pattern_bytes={} \ - compiles std={} fancy={} repeat={} cache_clears={} lazy_builds={} lazy_cache_hits={} \ + compiles std={} fancy={} repeat={} cache_clears={} evictions={} lazy_builds={} lazy_cache_hits={} \ exec={} exec_matched={} capture_slots={} capture_bytes={} test={} test_global={} \ match={} replace={} replace_matches={} split={} flags_alloc={} \ desc_regexp_probes={} desc_regexp_meta_negative={} \ @@ -356,6 +390,7 @@ impl RegexDiag { self.compiles_fancy, self.compiles_repeat, self.cache_clears, + self.cache_evictions, self.lazy_builds, self.lazy_cache_hits, self.exec_calls, diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 3170cdd277..a88a815048 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -481,9 +481,9 @@ crate::perry_thread_local! { /// validation. Validity is a pure function of the pair, so the answer is /// worth remembering; `js_regexp_new` used to get this from a /// `REGEX_CACHE` hit, which stopped being a proxy once the compiled - /// program became lazy (see `regex::lazy`). Same cap and - /// clear-on-overflow policy as the program caches — the cost of a clear - /// is a repeated parse, never a wrong verdict. The unit value keeps + /// program became lazy (see `regex::lazy`). Same cap and one-entry + /// eviction policy as the program caches — eviction can repeat one parse, + /// never change a verdict. The unit value keeps /// `evict_regex_cache_if_full` shared with the three program caches. static VALIDATED_PATTERNS: RefCell> = RefCell::new(HashMap::new()); } @@ -1751,4 +1751,6 @@ mod tests; #[cfg(all(test, feature = "regex-engine"))] mod tests_part2; #[cfg(all(test, feature = "regex-engine"))] +mod tests_cache; +#[cfg(all(test, feature = "regex-engine"))] mod tests_header; diff --git a/crates/perry-runtime/src/regex/compile_cache.rs b/crates/perry-runtime/src/regex/compile_cache.rs index c08835f960..4a4270aeb6 100644 --- a/crates/perry-runtime/src/regex/compile_cache.rs +++ b/crates/perry-runtime/src/regex/compile_cache.rs @@ -86,26 +86,27 @@ pub(crate) fn build_fancy_regex(pattern: &str) -> Result(cache: &mut HashMap) { +pub(crate) fn evict_regex_cache_if_full( + cache: &mut HashMap, +) { if cache.len() >= REGEX_CACHE_MAX_ENTRIES { - cache.clear(); + let victim = cache.keys().next().cloned(); + if let Some(victim) = victim { + cache.remove(&victim); + } + #[cfg(test)] + super::tests_cache::note_cache_eviction(); if crate::hot_diag::regex_on() { - crate::hot_diag::regex_with(|d| d.cache_clears += 1); + crate::hot_diag::regex_counters(|d| d.cache_evictions += 1); } } } diff --git a/crates/perry-runtime/src/regex/lazy.rs b/crates/perry-runtime/src/regex/lazy.rs index 845f1fb20b..fbc583eb19 100644 --- a/crates/perry-runtime/src/regex/lazy.rs +++ b/crates/perry-runtime/src/regex/lazy.rs @@ -217,6 +217,8 @@ fn build_and_install_programs(re: *const RegExpHeader) { if !is_valid_regex_ptr(re) { return; } + #[cfg(test)] + crate::hot_diag::test_note_regex_program_build(); let (pattern, flags) = source_and_flags(re); if crate::hot_diag::regex_on() { let cache_hit = super::REGEX_CACHE.with(|cache| { @@ -248,7 +250,7 @@ fn build_and_install_programs(re: *const RegExpHeader) { // here becomes the answer for every later construction of the same // literal. It therefore has to be complete, and the probes above cannot // guarantee that on their own: the three caches are capped independently - // and each `clear()`s wholesale, while + // and each can evict a different entry, while // `compile_and_cache_regex_checked` returns early whenever `REGEX_CACHE` // already holds the pattern — so it never re-runs the fancy or // repeat-matcher build for a pattern whose `REGEX_CACHE` entry survived a @@ -306,6 +308,16 @@ fn build_and_install_programs(re: *const RegExpHeader) { } } +#[cfg(test)] +pub(super) fn test_reset_program_builds() { + crate::hot_diag::test_reset_regex_builds_and_evictions(); +} + +#[cfg(test)] +pub(super) fn test_program_builds() -> u64 { + crate::hot_diag::test_regex_builds_and_evictions().0 +} + /// The header's standard-engine program, building it on first use. /// /// Every standard-program borrow in the tree goes through here — the field is diff --git a/crates/perry-runtime/src/regex/site_cache.rs b/crates/perry-runtime/src/regex/site_cache.rs index 675fe657df..5f6533cfd5 100644 --- a/crates/perry-runtime/src/regex/site_cache.rs +++ b/crates/perry-runtime/src/regex/site_cache.rs @@ -9,29 +9,24 @@ //! `ansi-regex` builds the same `new RegExp(parts.join("|"), "g")` per call. //! Each construction used to copy the pattern three times (the //! `VALIDATED_PATTERNS` probe key and `owned_pattern`) and SipHash all of it -//! once; the first operation on each header then -//! did the same three more times — `build_and_install_programs` probes the -//! three `(String, String)`-keyed program caches — and, for the common -//! no-fallback pattern, `lookup_fancy_regex` / `lookup_repeat_matcher` -//! re-probed two of them on EVERY exec. On the claude-code keystroke profile -//! SipHash over pattern text was 31 % of the post-turn window (regex 38 % -//! inclusive), all of it under these five functions. +//! once; the first operation on each header then did the same three more times. +//! On the claude-code keystroke profile SipHash over pattern text was 31 % of +//! the post-turn window (regex 38 % inclusive). //! //! # What //! -//! A direct-mapped, thread-local table keyed by a cheap CONTENT fingerprint -//! (length, first / middle / last 8 bytes, canonical flags) and verified by a -//! full byte compare — identity never depends on an address, so nothing is -//! rekeyed on a GC move and a dynamic `new RegExp(sameText)` hits too; a hit -//! costs one `memcmp` instead of a hash plus three copies. An entry owns the -//! pattern and canonical flags as `Arc` and, once the first header built -//! from it has been executed, the -//! compiled programs: a later construction installs those eagerly, so the -//! header is born built and never touches the `(pattern, flags)` caches. +//! A bounded, thread-local table keyed by a cheap CONTENT fingerprint (length, +//! first / middle / last 8 bytes, canonical flags) and verified by a full byte +//! compare. Identity never depends on an address, so nothing is rekeyed on a GC +//! move and a dynamic `new RegExp(sameText)` hits too; a hit costs one short +//! integer hash plus one `memcmp` instead of hashing all pattern bytes. //! -//! Validity is a pure function of `(pattern, flags)`, so a hit legitimately -//! skips validation: an entry is only ever written on the validated path, and -//! the programs it hands out were built for exactly this text. +//! An entry owns the pattern and canonical flags as `Arc` and, once the +//! first header built from it has executed, the compiled programs: a later +//! construction is born built. At capacity, entries referenced by a recorded +//! literal site are pinned and only dynamic or displaced-site entries are +//! evictable. Thus a live literal cannot rebuild, while programs for dead sites +//! can still leave the fixed-size table. //! //! Kill switch: `PERRY_REGEX_SITE_CACHE=0` (lookups miss, nothing is stored). @@ -40,6 +35,8 @@ use std::sync::Arc; use regex::Regex; +type ContentMap = crate::fast_hash::PtrHashMap>; + /// The compiled programs a header owns, in the form `lazy` installs them. pub(super) struct Programs { pub(super) std: Arc, @@ -73,13 +70,12 @@ struct Entry { programs: Option>, } -/// Direct-mapped slots (2-way: a fingerprint may live in `slot` or -/// `slot ^ 1`). Sized for a bundle's live literal working set; the -/// claude-code TUI cycles through a few dozen per render. -const SLOTS: usize = 1024; +/// Sized for the recorded literal-site table. The table never exceeds this +/// bound: if all entries are pinned, a dynamic miss remains uncached. +pub(super) const MAX_ENTRIES: usize = 1024; crate::perry_thread_local! { - static SITE_CACHE: RefCell>> = RefCell::new(Vec::new()); + static SITE_CACHE: RefCell = RefCell::new(crate::fast_hash::new_ptr_hash_map()); } fn enabled() -> bool { @@ -92,8 +88,8 @@ fn enabled() -> bool { } /// Cheap content fingerprint: length, three 8-byte windows of the pattern, -/// the (≤ 8 byte) canonical flags. Collisions are harmless — every hit is -/// verified by a full compare — they only cost the verify and a re-insert. +/// the (≤ 8 byte) canonical flags. Collisions are harmless because every hit +/// is verified by a full compare and colliding entries share one small bucket. fn fingerprint(pattern: &[u8], flags: &[u8]) -> u64 { #[inline] fn window(bytes: &[u8], at: usize) -> u64 { @@ -117,46 +113,69 @@ fn fingerprint(pattern: &[u8], flags: &[u8]) -> u64 { h } -#[inline] -fn slot_of(fp: u64) -> usize { - (fp as usize) & (SLOTS - 1) -} - fn entry_matches(entry: &Entry, fp: u64, pattern: &str, flags: &str) -> bool { entry.fp == fp && &*entry.flags == flags && &*entry.pattern == pattern } +fn entry_count(cache: &ContentMap) -> usize { + cache.values().map(Vec::len).sum() +} + +/// Remove one entry that has no recorded literal site. The scan happens only +/// on a distinct-content miss at capacity; literal-site hits never reach it. +fn evict_one_dynamic(cache: &mut ContentMap) -> bool { + let victim = cache.iter().find_map(|(&fp, bucket)| { + bucket + .iter() + .position(|entry| !super::site_key::references_content(&entry.pattern, &entry.flags)) + .map(|index| (fp, index)) + }); + let Some((fp, index)) = victim else { + return false; + }; + let bucket = cache.get_mut(&fp).expect("the selected bucket exists"); + bucket.swap_remove(index); + if bucket.is_empty() { + cache.remove(&fp); + } + true +} + +fn make_room(cache: &mut ContentMap) -> bool { + if entry_count(cache) < MAX_ENTRIES { + return true; + } + if evict_one_dynamic(cache) { + if crate::hot_diag::regex_on() { + crate::hot_diag::regex_counters(|d| d.cache_evictions += 1); + } + return true; + } + false +} + /// Find the verified entry for `(pattern, canonical flags)`. pub(super) fn lookup(pattern: &str, flags: &str) -> Option { if !enabled() { return None; } let fp = fingerprint(pattern.as_bytes(), flags.as_bytes()); - let slot = slot_of(fp); SITE_CACHE.with(|cache| { let cache = cache.borrow(); - if cache.is_empty() { - return None; - } - for s in [slot, slot ^ 1] { - if let Some(entry) = &cache[s] { - if entry_matches(entry, fp, pattern, flags) { - // The verify is a FULL byte compare, so its cost is - // linear in the pattern and this counter — not - // `pattern_bytes`, which counts every construction - // whether it probed or not — is the `memcmp` volume. - // Counted at the construction probe only; `insert` and - // `install_programs` verify too and are not counted here. - if crate::hot_diag::regex_on() { - let n = pattern.len() as u64; - crate::hot_diag::regex_counters(|d| d.new_site_verify_bytes += n); - } - return Some(Hit { - pattern: entry.pattern.clone(), - flags: entry.flags.clone(), - programs: entry.programs.clone(), + for entry in cache.get(&fp)? { + if entry_matches(entry, fp, pattern, flags) { + // Count only the construction probe's full byte compare, not + // the cold insert/install verification. + if crate::hot_diag::regex_on() { + crate::hot_diag::regex_counters(|d| { + d.new_site_verify_bytes += pattern.len() as u64 }); } + return Some(Hit { + pattern: entry.pattern.clone(), + flags: entry.flags.clone(), + programs: entry.programs.clone(), + }); } } None @@ -164,83 +183,68 @@ pub(super) fn lookup(pattern: &str, flags: &str) -> Option { } /// Record a validated `(pattern, canonical flags)`, returning the shared -/// owned copies a header should keep. An existing verified entry is reused -/// (its programs are kept); otherwise the fresh entry has none yet. +/// owned copies a header should keep. An existing verified entry is reused. pub(super) fn insert(pattern: &str, flags: &str) -> (Arc, Arc) { if !enabled() { return (Arc::from(pattern), Arc::from(flags)); } let fp = fingerprint(pattern.as_bytes(), flags.as_bytes()); - let slot = slot_of(fp); SITE_CACHE.with(|cache| { let mut cache = cache.borrow_mut(); - if cache.is_empty() { - cache.resize_with(SLOTS, || None); - } - for s in [slot, slot ^ 1] { - if let Some(entry) = &cache[s] { + if let Some(bucket) = cache.get(&fp) { + for entry in bucket { if entry_matches(entry, fp, pattern, flags) { return (entry.pattern.clone(), entry.flags.clone()); } } } - let victim = if cache[slot].is_none() { - slot - } else if cache[slot ^ 1].is_none() { - slot ^ 1 - } else { - slot ^ ((fp >> 11) as usize & 1) - }; let pattern: Arc = Arc::from(pattern); let flags: Arc = Arc::from(flags); - cache[victim] = Some(Entry { - fp, - pattern: pattern.clone(), - flags: flags.clone(), - programs: None, - }); + if make_room(&mut cache) { + cache.entry(fp).or_default().push(Entry { + fp, + pattern: pattern.clone(), + flags: flags.clone(), + programs: None, + }); + } (pattern, flags) }) } -/// Attach the programs the first execution built to the entry for -/// `(pattern, canonical flags)`, so every later construction of the same -/// text is born built. Inserts the entry if it was evicted meanwhile. +/// Attach the programs the first execution built to the content entry and +/// publish a weak view to every recorded literal site for this exact content. pub(super) fn install_programs(pattern: &str, flags: &str, programs: Arc) { if !enabled() { return; } let fp = fingerprint(pattern.as_bytes(), flags.as_bytes()); - let slot = slot_of(fp); - SITE_CACHE.with(|cache| { + let content_owned = SITE_CACHE.with(|cache| { let mut cache = cache.borrow_mut(); - if cache.is_empty() { - cache.resize_with(SLOTS, || None); - } - for s in [slot, slot ^ 1] { - if let Some(entry) = &mut cache[s] { + if let Some(bucket) = cache.get_mut(&fp) { + for entry in bucket { if entry_matches(entry, fp, pattern, flags) { if entry.programs.is_none() { - entry.programs = Some(programs); + entry.programs = Some(programs.clone()); } - return; + return true; } } } - let victim = if cache[slot].is_none() { - slot - } else if cache[slot ^ 1].is_none() { - slot ^ 1 - } else { - slot ^ ((fp >> 11) as usize & 1) - }; - cache[victim] = Some(Entry { + if !make_room(&mut cache) { + return false; + } + cache.entry(fp).or_default().push(Entry { fp, pattern: Arc::from(pattern), flags: Arc::from(flags), - programs: Some(programs), + programs: Some(programs.clone()), }); + true }); + if content_owned { + super::site_key::install_programs_for_content(pattern, flags, &programs); + } } #[cfg(test)] @@ -251,19 +255,23 @@ pub(super) fn test_reset() { #[cfg(test)] pub(super) fn test_has_programs(pattern: &str, flags: &str) -> Option { let fp = fingerprint(pattern.as_bytes(), flags.as_bytes()); - let slot = slot_of(fp); SITE_CACHE.with(|cache| { let cache = cache.borrow(); - if cache.is_empty() { - return None; - } - for s in [slot, slot ^ 1] { - if let Some(entry) = &cache[s] { - if entry_matches(entry, fp, pattern, flags) { - return Some(entry.programs.is_some()); - } + for entry in cache.get(&fp)? { + if entry_matches(entry, fp, pattern, flags) { + return Some(entry.programs.is_some()); } } None }) } + +#[cfg(test)] +pub(super) fn test_len() -> usize { + SITE_CACHE.with(|cache| entry_count(&cache.borrow())) +} + +#[cfg(test)] +pub(super) fn test_try_evict_one_dynamic() -> bool { + SITE_CACHE.with(|cache| evict_one_dynamic(&mut cache.borrow_mut())) +} diff --git a/crates/perry-runtime/src/regex/site_key.rs b/crates/perry-runtime/src/regex/site_key.rs index 842faa7072..f9737f7ef6 100644 --- a/crates/perry-runtime/src/regex/site_key.rs +++ b/crates/perry-runtime/src/regex/site_key.rs @@ -50,8 +50,8 @@ use std::sync::{Arc, Weak}; use super::site_cache::Programs; -/// The site entry's view of a pattern's compiled programs: **weak**, so the -/// table can hand them out but can never be the reason they stay alive. +/// The site entry's view of a pattern's compiled programs: **weak**, because +/// the pinned content-cache entry owns the bundle. /// /// Measured cost of holding them strongly (cc, one 3300-char reply): settled /// footprint 478/474 MB → 500/527 MB and idle CPU 2.37 → 2.68 s. The site @@ -60,11 +60,9 @@ use super::site_cache::Programs; /// wants. The campaign's directive is both metrics together, and a CPU win /// bought with resident memory does not land. /// -/// Strong references remain where they belong: the `(pattern, flags)` program -/// caches, and every live header that installed them via `Arc::into_raw`. A -/// site entry whose programs have been dropped simply reports "not built -/// yet", and the next construction re-picks them up from the content cache — -/// the same path the site's very first construction takes. +/// Strong references remain where they belong: the content cache and every +/// live header that installed them via `Arc::into_raw`. When this bounded site +/// entry is displaced, the content entry becomes eligible for eviction. struct WeakPrograms(Weak); impl WeakPrograms { @@ -270,6 +268,31 @@ pub(super) fn install_programs(key: usize, programs: Arc) { }); } +/// Whether the bounded literal-site table still records this content. The +/// content cache consults this only on a collision miss, never on a site hit. +pub(super) fn references_content(pattern: &str, flags: &str) -> bool { + SITE_KEY_TABLE.with(|table| { + table + .borrow() + .iter() + .flatten() + .any(|entry| &*entry.pattern == pattern && &*entry.flags == flags) + }) +} + +/// Publish a freshly built bundle to every literal site for this content. The +/// content cache owns it; sites observe the complete bundle through one weak +/// reference, preserving the all-or-nothing matcher rule. +pub(super) fn install_programs_for_content(pattern: &str, flags: &str, programs: &Arc) { + SITE_KEY_TABLE.with(|table| { + for entry in table.borrow_mut().iter_mut().flatten() { + if &*entry.pattern == pattern && &*entry.flags == flags { + entry.programs = Some(WeakPrograms::downgrade(programs)); + } + } + }); +} + #[cfg(test)] pub(super) fn test_reset() { SITE_KEY_TABLE.with(|table| table.borrow_mut().clear()); diff --git a/crates/perry-runtime/src/regex/tests.rs b/crates/perry-runtime/src/regex/tests.rs index 7d72ffff98..af67a400ea 100644 --- a/crates/perry-runtime/src/regex/tests.rs +++ b/crates/perry-runtime/src/regex/tests.rs @@ -904,7 +904,7 @@ fn unicode17_scripts_expand_to_codepoint_ranges() { /// 2026-07-09 GC audit (wave 2 batch A): the compiled-regex caches were /// unbounded — one entry per distinct `(pattern, flags)` ever compiled, up to /// 64 MiB each — so `new RegExp(userInput)` was an attacker-driven OOM. The -/// caches are now capped (clear-on-overflow) and every `RegExpHeader` OWNS a +/// caches are now capped (one-entry eviction) and every `RegExpHeader` OWNS a /// leaked Arc reference to its compiled program(s), so a header created /// before an eviction keeps matching afterwards. #[test] diff --git a/crates/perry-runtime/src/regex/tests_cache.rs b/crates/perry-runtime/src/regex/tests_cache.rs new file mode 100644 index 0000000000..297198c31d --- /dev/null +++ b/crates/perry-runtime/src/regex/tests_cache.rs @@ -0,0 +1,139 @@ +use super::*; +use std::collections::HashSet; + +pub(super) fn note_cache_eviction() { + crate::hot_diag::test_note_regex_cache_eviction(); +} + +fn make_string(text: &str) -> *mut StringHeader { + js_string_from_str(text) +} + +fn cache_test_key(slot: usize) -> i64 { + (0x6000_0000usize + (slot << 3)) as i64 +} + +/// Sabotage: remove the literal pin in `site_cache::evict_one_dynamic`, or put +/// back the whole-map clear in `evict_regex_cache_if_full`. +/// +/// The first sabotage lets a content-capacity eviction discard the target's +/// sole `Arc` owner. The second removes 512 answers instead of one. +/// After an explicit young collection finalizes both target headers, either +/// regression makes the next evaluation run the lazy builder again. +#[test] +fn literal_site_program_is_not_rebuilt_after_cache_overflow_and_young_collection() { + let _lock = crate::gc::global_side_table_test_lock(); + site_key::test_reset(); + site_cache::test_reset(); + REGEX_CACHE.with(|cache| cache.borrow_mut().clear()); + FANCY_CACHE.with(|cache| cache.borrow_mut().clear()); + REPEAT_MATCHER_CACHE.with(|cache| cache.borrow_mut().clear()); + VALIDATED_PATTERNS.with(|cache| cache.borrow_mut().clear()); + lazy::test_reset_program_builds(); + + let target = "literal-site-overflow-target-[0-9]+"; + let target_key = cache_test_key(0); + let first = js_regexp_new_site(make_string(target), make_string(""), target_key); + assert_eq!( + js_regexp_test(first, make_string("literal-site-overflow-target-42")), + 1 + ); + assert_eq!( + lazy::test_program_builds(), + 1, + "the target builds exactly once" + ); + + // A second construction installs the content cache's program bundle into + // the site's weak lane. Both headers must then die so that only the cache + // policy, not a surviving receiver, can keep that weak reference live. + let second = js_regexp_new_site(make_string(target), make_string(""), target_key); + assert!(!unsafe { (*second).programs_ptr.is_null() }); + let first_addr = first as usize; + let second_addr = second as usize; + assert!( + !site_cache::test_try_evict_one_dynamic(), + "the only content entry is site-referenced, so no dynamic victim exists" + ); + + // Overflow the bounded content table with dynamic patterns. The target is + // the only recorded literal among them, so every capacity eviction must + // choose one of the dynamic entries and leave its program bundle intact. + for i in 0..site_cache::MAX_ENTRIES { + let _ = js_regexp_new( + make_string(&format!("content-overflow-{i}[a-z]")), + make_string(""), + ); + } + assert_eq!(site_cache::test_len(), site_cache::MAX_ENTRIES); + assert_eq!( + site_cache::test_has_programs(target, ""), + Some(true), + "a recorded literal's content entry must survive capacity eviction" + ); + + // 513 compiled literals (the target plus this 512-pattern flood) cross the + // former 512-entry wholesale-clear boundary. Snapshot at capacity so the + // assertion proves the overflow path preserved 511 old answers. + for i in 0..(REGEX_CACHE_MAX_ENTRIES - 1) { + let pattern = format!("overflow-literal-{i}$"); + let re = js_regexp_new_site( + make_string(&pattern), + make_string(""), + cache_test_key(100 + i), + ); + assert_eq!( + js_regexp_test(re, make_string(&format!("overflow-literal-{i}"))), + 1 + ); + } + let before: HashSet<_> = REGEX_CACHE.with(|cache| cache.borrow().keys().cloned().collect()); + assert_eq!(before.len(), REGEX_CACHE_MAX_ENTRIES); + + let last_i = REGEX_CACHE_MAX_ENTRIES - 1; + let last_pattern = format!("overflow-literal-{last_i}$"); + let last = js_regexp_new_site( + make_string(&last_pattern), + make_string(""), + cache_test_key(100 + last_i), + ); + assert_eq!( + js_regexp_test(last, make_string(&format!("overflow-literal-{last_i}"))), + 1 + ); + let survivors = REGEX_CACHE.with(|cache| { + cache + .borrow() + .keys() + .filter(|key| before.contains(*key)) + .count() + }); + assert_eq!(survivors, REGEX_CACHE_MAX_ENTRIES - 1); + assert!( + crate::hot_diag::test_regex_builds_and_evictions().1 > 0, + "the capacity eviction path must have executed" + ); + + let builds_before_gc = lazy::test_program_builds(); + let _ = crate::gc::gc_collect_minor(); + assert!( + !test_regex_pointer_entry_exists(first_addr) + && !test_regex_pointer_entry_exists(second_addr), + "the explicit young collection must finalize both unrooted target headers" + ); + + let rebuilt = js_regexp_new_site(make_string(target), make_string(""), target_key); + assert!( + !unsafe { (*rebuilt).programs_ptr.is_null() }, + "the still-recorded site must be born built after cache overflow and young GC" + ); + assert_eq!( + js_regexp_test(rebuilt, make_string("literal-site-overflow-target-7")), + 1 + ); + assert_eq!( + lazy::test_program_builds(), + builds_before_gc, + "the target site's compiled program must not be rebuilt" + ); +} From 508245b53b5ff9124ef5d1b6d29b34d01f8b1bea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 16:41:18 +0200 Subject: [PATCH 08/27] fix(regex): clear branch-owned CI failures Use scoped handle access in the nursery relocation fixture, gate the Arc import to the matcher feature, and document why matcher kinds are dead in the feature-off layout-only build. Remove the unused test import and unsafe block, and apply rustfmt's module ordering. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit dd1c5242d2ce87139d33436f347adb7245fe754d) --- crates/perry-runtime/src/regex.rs | 20 +++++++++++++++---- crates/perry-runtime/src/regex/tests_part2.rs | 19 ++++++++---------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index a88a815048..12c0ef67b4 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -13,6 +13,7 @@ use std::cell::RefCell; #[cfg(feature = "regex-engine")] use std::collections::HashMap; use std::ptr; +#[cfg(feature = "regex-engine")] use std::sync::Arc; #[cfg(feature = "regex-engine")] @@ -387,8 +388,12 @@ pub(crate) fn test_alloc_nursery_regexp_for_move(source: &str, flags: &str) -> * // must be set explicitly or the GC follows a garbage pointer. (*ptr).meta = std::ptr::null_mut(); (*ptr).programs_ptr = std::ptr::null(); - (*ptr).pattern_ptr = pattern.get_raw_const_ptr::(); - (*ptr).flags_ptr = flags_string.get_raw_const_ptr::(); + pattern.with_const_ptr::(|pattern| { + (*ptr).pattern_ptr = pattern; + }); + flags_string.with_const_ptr::(|flags| { + (*ptr).flags_ptr = flags; + }); (*ptr).case_insensitive = flags.contains('i'); (*ptr).global = flags.contains('g'); (*ptr).multiline = flags.contains('m'); @@ -495,6 +500,13 @@ pub(crate) use compile_cache::*; #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[repr(u8)] +#[cfg_attr( + not(feature = "regex-engine"), + allow( + dead_code, + reason = "the feature-off runtime preserves RegExpHeader layout but constructs no matchers" + ) +)] pub(super) enum MatcherKind { Unbuilt, Standard, @@ -1749,8 +1761,8 @@ pub(crate) fn test_last_exec_groups() -> usize { #[cfg(all(test, feature = "regex-engine"))] mod tests; #[cfg(all(test, feature = "regex-engine"))] -mod tests_part2; -#[cfg(all(test, feature = "regex-engine"))] mod tests_cache; #[cfg(all(test, feature = "regex-engine"))] mod tests_header; +#[cfg(all(test, feature = "regex-engine"))] +mod tests_part2; diff --git a/crates/perry-runtime/src/regex/tests_part2.rs b/crates/perry-runtime/src/regex/tests_part2.rs index e8cda486be..5896ff2c94 100644 --- a/crates/perry-runtime/src/regex/tests_part2.rs +++ b/crates/perry-runtime/src/regex/tests_part2.rs @@ -3,8 +3,7 @@ //! in `tests.rs`; the shared fixtures come from there. use super::tests::{ - make_string, match_capture_text, regex_has_fancy_program, regex_has_repeat_program, - regex_is_built, string_payload, + make_string, match_capture_text, regex_has_fancy_program, regex_is_built, string_payload, }; use super::*; @@ -599,15 +598,13 @@ fn a_single_program_cache_clear_cannot_disarm_a_lookbehind_literal() { site_cache::test_reset(); let cold = build(); - unsafe { - lazy::ensure_regex_compiled(cold); - assert!( - regex_has_fancy_program(cold), - "a built header must carry every program its pattern needs — a null \ - the fancy program here is memoized by site_cache::install_programs and makes \ - the breakage permanent for this literal" - ); - } + lazy::ensure_regex_compiled(cold); + assert!( + regex_has_fancy_program(cold), + "a built header must carry every program its pattern needs — a null \ + the fancy program here is memoized by site_cache::install_programs and makes \ + the breakage permanent for this literal" + ); assert_eq!( subject.with_const_ptr::(|s| js_regexp_test(cold, s)), 1, From d8232678018d65dc6e0fe03e075907e8b0a5ab17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 12:04:58 +0200 Subject: [PATCH 09/27] perf(regex): reuse literal headers at test-only sites Cache and root one RegExp header for literal-only test call sites, while validating the builtin method on every evaluation and preserving generic fallback semantics for patched prototypes and changed factory callees. Recognize exact zero-argument regex factories by their HIR body and pair cross-function sites with the resolved native callee identity. Reset global and sticky lastIndex before each cached test, expose decline diagnostics, and register all regex cache tables with the GC census. (cherry picked from commit 90e21317af3c54e068d4de518eb7b440dc606112) --- crates/perry-codegen/src/codegen/closure.rs | 14 + crates/perry-codegen/src/codegen/entry.rs | 2 + crates/perry-codegen/src/codegen/function.rs | 8 +- crates/perry-codegen/src/codegen/method.rs | 2 + crates/perry-codegen/src/expr/calls.rs | 102 +- .../perry-codegen/src/expr/instance_misc1.rs | 35 + .../src/expr/logical_collections.rs | 99 +- crates/perry-codegen/src/expr/mod.rs | 9 + .../src/expr/regex_site_test_tests.rs | 206 ++++ crates/perry-codegen/src/runtime_decls/mod.rs | 37 + .../src/runtime_decls/strings.rs | 14 + crates/perry-runtime/src/exception.rs | 20 + crates/perry-runtime/src/gc/census.rs | 19 + crates/perry-runtime/src/gc/mod.rs | 2 + crates/perry-runtime/src/hot_diag.rs | 16 +- crates/perry-runtime/src/regex.rs | 13 +- crates/perry-runtime/src/regex/site_cache.rs | 16 + crates/perry-runtime/src/regex/site_key.rs | 12 + crates/perry-runtime/src/regex/site_test.rs | 880 ++++++++++++++++++ scripts/gc_runtime_root_holders.json | 80 +- 20 files changed, 1536 insertions(+), 50 deletions(-) create mode 100644 crates/perry-codegen/src/expr/regex_site_test_tests.rs create mode 100644 crates/perry-runtime/src/regex/site_test.rs diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index e530881f23..1facefbe65 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -515,6 +515,7 @@ pub(super) fn compile_closure( captures_new_target, enclosing_class, is_async, + is_generator, is_strict, ) = match closure_expr { perry_hir::Expr::Closure { @@ -525,6 +526,7 @@ pub(super) fn compile_closure( captures_new_target, enclosing_class, is_async, + is_generator, is_strict, .. } => ( @@ -535,6 +537,7 @@ pub(super) fn compile_closure( *captures_new_target, enclosing_class.clone(), *is_async, + *is_generator, *is_strict, ), _ => return Err(anyhow!("compile_closure: expected Expr::Closure")), @@ -556,6 +559,16 @@ pub(super) fn compile_closure( closure_relevant_ids.extend(captures.iter().copied()); let public_llvm_name = format!("perry_closure_{}__{}", module_prefix, func_id); + let regex_factory_identity = (!is_async + && !is_generator + && params.is_empty() + && matches!( + body.as_slice(), + [perry_hir::Stmt::Return(Some( + perry_hir::Expr::RegExp { .. } + ))] + )) + .then(|| public_llvm_name.clone()); let typed_public_trampoline = if cross_module.typed_f64_closures.contains(&func_id) { Some(TypedFunctionTrampolineKind::F64) } else if cross_module.typed_i32_closures.contains(&func_id) { @@ -1053,6 +1066,7 @@ pub(super) fn compile_closure( module_slug: crate::expr::native_region_slug(strings.module_prefix()), source_function: format!("closure_{}", func_id), source_function_slug: crate::expr::native_region_slug(&format!("closure_{}", func_id)), + regex_factory_identity, active_region_id: None, native_facts: &native_facts, locals, diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index 4564eb9046..5d2f5cc498 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -850,6 +850,7 @@ pub(super) fn compile_module_entry( module_slug: crate::expr::native_region_slug(strings.module_prefix()), source_function: "module_init".to_string(), source_function_slug: crate::expr::native_region_slug("module_init"), + regex_factory_identity: None, active_region_id: None, native_facts: &main_native_facts, locals: HashMap::new(), @@ -1640,6 +1641,7 @@ pub(super) fn compile_module_entry( module_slug: crate::expr::native_region_slug(strings.module_prefix()), source_function: "module_init".to_string(), source_function_slug: crate::expr::native_region_slug("module_init"), + regex_factory_identity: None, active_region_id: None, native_facts: &init_native_facts, locals: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 21eb811fb3..0289c4b085 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -5,7 +5,7 @@ use std::collections::{HashMap, HashSet}; use anyhow::{anyhow, Context, Result}; -use perry_hir::Function; +use perry_hir::{Expr, Function, Stmt}; use crate::expr::FnCtx; use crate::module::LlModule; @@ -524,6 +524,11 @@ pub(super) fn compile_function( .get(&f.id) .cloned() .ok_or_else(|| anyhow!("function name not resolved for {}", f.name))?; + let regex_factory_identity = (!f.is_async + && !f.is_generator + && f.params.is_empty() + && matches!(f.body.as_slice(), [Stmt::Return(Some(Expr::RegExp { .. }))])) + .then(|| public_llvm_name.clone()); let guarded_public_plan = if typed_public_trampoline.is_none() && spec_entry.is_none() { cross_module .spec_abi_functions @@ -1023,6 +1028,7 @@ pub(super) fn compile_function( module_slug: crate::expr::native_region_slug(strings.module_prefix()), source_function: f.name.clone(), source_function_slug: crate::expr::native_region_slug(&f.name), + regex_factory_identity, active_region_id: None, native_facts: &native_facts, locals, diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index 4c1a1f061e..c562c1de88 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -469,6 +469,7 @@ pub(super) fn compile_method( "{}.{}", class.name, method.name )), + regex_factory_identity: None, active_region_id: None, native_facts: &native_facts, locals, @@ -1609,6 +1610,7 @@ pub(super) fn compile_static_method( "{}.{}", class.name, f.name )), + regex_factory_identity: None, active_region_id: None, native_facts: &native_facts, locals, diff --git a/crates/perry-codegen/src/expr/calls.rs b/crates/perry-codegen/src/expr/calls.rs index f110590367..d2975c8799 100644 --- a/crates/perry-codegen/src/expr/calls.rs +++ b/crates/perry-codegen/src/expr/calls.rs @@ -13,7 +13,8 @@ use perry_hir::Expr; use crate::lower_call::{lower_call, lower_native_method_call}; use crate::nanbox::double_literal; -use crate::types::DOUBLE; +use crate::rooting; +use crate::types::{DOUBLE, I64}; use super::{ emit_string_literal_global, lower_expr, nanbox_pointer_inline, nanbox_string_inline, @@ -73,6 +74,32 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { args, ), + // `().test(arg)`, including the bundled namespace form + // `ns.default().test(arg)`. The inner call still executes normally; + // a structurally proven zero-argument regex factory consumes the + // active site and may return its rooted header. Any reassignment or + // non-literal body therefore reaches the unchanged generic method + // path, rather than trusting a source-level binding assumption. + Expr::Call { callee, args, .. } + if args.len() == 1 + && matches!( + callee.as_ref(), + Expr::PropertyGet { object, property, .. } + if property == "test" + && matches!( + object.as_ref(), + Expr::Call { callee, args, .. } + if args.is_empty() + // A computed/`with` reference carries + // receiver-binding semantics that the + // site wrapper does not model. + && !matches!(callee.as_ref(), Expr::IndexGet { .. } | Expr::WithGet { .. }) + ) + ) => + { + arm_regexp_factory_site_test(ctx, callee.as_ref(), &args[0]) + } + // #1645: `ReadableStream.from(iterable)` (Node 20+). The HIR lowers // `(ReadableStream as any).from(x)` to a Call whose callee is // `PropertyGet { ExternFuncRef("ReadableStream"), "from" }`; route it to @@ -848,3 +875,76 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { _ => unreachable!("expr/mod.rs dispatched a variant not handled by this submodule"), } } + +fn arm_regexp_factory_site_test( + ctx: &mut FnCtx<'_>, + outer_callee: &Expr, + argument: &Expr, +) -> Result { + let Expr::PropertyGet { object, .. } = outer_callee else { + unreachable!("guarded by the caller") + }; + let Expr::Call { + callee: inner_callee, + args: inner_args, + .. + } = object.as_ref() + else { + unreachable!("guarded by the caller") + }; + debug_assert!(inner_args.is_empty()); + + let slot_ref = super::logical_collections::emit_regexp_site_key(ctx); + let site_key = ctx.block().ptrtoint(&slot_ref, I64); + let receiver = match inner_callee.as_ref() { + Expr::PropertyGet { + object, property, .. + } => { + let object = lower_expr(ctx, object)?; + let key_idx = ctx.strings.intern(property); + let entry = ctx.strings.entry(key_idx); + let key_global = format!("@{}", entry.handle_global); + let key = ctx.block().load(DOUBLE, &key_global); + ctx.block().call( + DOUBLE, + "js_regexp_site_factory_call_method", + &[(I64, &site_key), (DOUBLE, &object), (DOUBLE, &key)], + ) + } + callee => { + let callee = lower_expr(ctx, callee)?; + ctx.block().call( + DOUBLE, + "js_regexp_site_factory_call_value", + &[(I64, &site_key), (DOUBLE, &callee)], + ) + } + }; + + // Property Get for `.test` precedes argument evaluation in ECMAScript. + // A cached/canonical receiver records the builtin as an internal marker; + // a decline resolves the actual property now, so a getter or a patch has + // exactly the generic ordering. + let method = ctx.block().call( + DOUBLE, + "js_regexp_site_test_get_method", + &[(I64, &site_key), (DOUBLE, &receiver)], + ); + rooting::with_rooted_group(ctx, 2, |ctx, roots| { + let receiver = roots.adopt_emitted(ctx, rooting::Repr::Boxed, &receiver, true); + let method = roots.adopt_emitted(ctx, rooting::Repr::Boxed, &method, true); + let argument = lower_expr(ctx, argument)?; + let receiver = roots.reread_emitted(ctx, receiver); + let method = roots.reread_emitted(ctx, method); + Ok(ctx.block().call( + DOUBLE, + "js_regexp_site_test_dispatch", + &[ + (I64, &site_key), + (DOUBLE, &receiver), + (DOUBLE, &method), + (DOUBLE, &argument), + ], + )) + }) +} diff --git a/crates/perry-codegen/src/expr/instance_misc1.rs b/crates/perry-codegen/src/expr/instance_misc1.rs index 9f395cfc5e..d212aed233 100644 --- a/crates/perry-codegen/src/expr/instance_misc1.rs +++ b/crates/perry-codegen/src/expr/instance_misc1.rs @@ -1190,6 +1190,41 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // Receiver is a NaN-tagged i64 RegExpHeader pointer; arg is // a NaN-tagged string. Both must be unboxed before the call. Expr::RegExpTest { regex, string } => { + // A literal used directly as this one receiver cannot escape: the + // HIR node owns the literal expression and publishes only the + // call result. Construct (or fetch) the site's rooted header + // before evaluating the argument, resolving `.test` at the same + // pre-argument point as an ordinary call. This ordering matters + // for `/x/.test(patchPrototype())`: it invokes the method value + // captured before the patch. + if let Expr::RegExp { pattern, flags } = regex.as_ref() { + let (receiver, site_key) = + super::logical_collections::lower_regexp_site_test_receiver( + ctx, pattern, flags, + ); + let method = ctx.block().call( + DOUBLE, + "js_regexp_site_test_get_method", + &[(I64, &site_key), (DOUBLE, &receiver)], + ); + return rooting::with_rooted_group(ctx, 2, |ctx, roots| { + let receiver = roots.adopt_emitted(ctx, rooting::Repr::Boxed, &receiver, true); + let method = roots.adopt_emitted(ctx, rooting::Repr::Boxed, &method, true); + let argument = lower_expr(ctx, string)?; + let receiver = roots.reread_emitted(ctx, receiver); + let method = roots.reread_emitted(ctx, method); + Ok(ctx.block().call( + DOUBLE, + "js_regexp_site_test_dispatch", + &[ + (I64, &site_key), + (DOUBLE, &receiver), + (DOUBLE, &method), + (DOUBLE, &argument), + ], + )) + }); + } // #7154: the receiver is live across BOTH the string operand's own // lowering and the `js_jsvalue_to_string_coerce` below it, and the // coerce is unconditional — it allocates, and on an object argument diff --git a/crates/perry-codegen/src/expr/logical_collections.rs b/crates/perry-codegen/src/expr/logical_collections.rs index 68b3be1a6e..27d46ae0c9 100644 --- a/crates/perry-codegen/src/expr/logical_collections.rs +++ b/crates/perry-codegen/src/expr/logical_collections.rs @@ -58,6 +58,58 @@ use super::{ record_collection_string_key_selected, unbox_str_handle, unbox_to_i64, FnCtx, }; +/// Emit one immortal identity slot for a regex optimization site. +/// +/// The value stored in the slot is irrelevant; only its linker-stable address +/// is used. Keeping this in one helper prevents the allocation-free `.test` +/// paths from inventing a second site-key scheme or hand-writing an ABI +/// constant that can drift from ordinary literal lowering. +pub(crate) fn emit_regexp_site_key(ctx: &mut FnCtx<'_>) -> String { + let site_id = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let prefix = ctx.strings.module_prefix(); + let slot_name = if prefix.is_empty() { + format!("perry_regexp_site_{site_id}") + } else { + format!("perry_regexp_site_{prefix}__{site_id}") + }; + ctx.typed_parse_rodata + .push(format!("@{slot_name} = private global i64 0")); + format!("@{slot_name}") +} + +/// Construct the receiver for the exact non-escaping `/literal/.test(arg)` +/// shape. The returned site key is also consumed by the post-argument +/// dispatch, which revalidates the builtin before it exposes the cached +/// receiver as `this`. +pub(crate) fn lower_regexp_site_test_receiver( + ctx: &mut FnCtx<'_>, + pattern: &str, + flags: &str, +) -> (String, String) { + let pattern_idx = ctx.strings.intern(pattern); + let flags_idx = ctx.strings.intern(flags); + let pattern_global = format!("@{}", ctx.strings.entry(pattern_idx).handle_global); + let flags_global = format!("@{}", ctx.strings.entry(flags_idx).handle_global); + let slot_ref = emit_regexp_site_key(ctx); + let blk = ctx.block(); + let pattern_box = blk.load(DOUBLE, &pattern_global); + let flags_box = blk.load(DOUBLE, &flags_global); + let pattern_handle = unbox_to_i64(blk, &pattern_box); + let flags_handle = unbox_to_i64(blk, &flags_box); + let site_key = blk.ptrtoint(&slot_ref, I64); + let result = blk.call( + I64, + "js_regexp_site_test_new", + &[ + (I64, &pattern_handle), + (I64, &flags_handle), + (I64, &site_key), + ], + ); + (nanbox_pointer_inline(blk, &result), site_key) +} + fn is_static_string_key_map(ctx: &FnCtx<'_>, map: &Expr) -> bool { matches!( map_static_type_args(ctx, map), @@ -1327,34 +1379,37 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // and unenforced: a future early return that drops the artifacts // breaks this site, loudly, at the in-process LLVM parse (`use of // undefined value`) rather than at runtime. - let site_id = ctx.ic_site_counter; - ctx.ic_site_counter += 1; - let slot_name = { - let prefix = ctx.strings.module_prefix(); - if prefix.is_empty() { - format!("perry_regexp_site_{site_id}") - } else { - format!("perry_regexp_site_{prefix}__{site_id}") - } - }; - ctx.typed_parse_rodata - .push(format!("@{slot_name} = private global i64 0")); - let slot_ref = format!("@{slot_name}"); + let slot_ref = emit_regexp_site_key(ctx); + let factory_identity = ctx.regex_factory_identity.clone(); let blk = ctx.block(); let pattern_box = blk.load(DOUBLE, &pattern_global); let flags_box = blk.load(DOUBLE, &flags_global); let pattern_handle = unbox_to_i64(blk, &pattern_box); let flags_handle = unbox_to_i64(blk, &flags_box); let site_key = blk.ptrtoint(&slot_ref, I64); - let result = blk.call( - I64, - "js_regexp_new_site", - &[ - (I64, &pattern_handle), - (I64, &flags_handle), - (I64, &site_key), - ], - ); + let result = if let Some(identity) = factory_identity { + let identity = blk.ptrtoint(&format!("@{identity}"), I64); + blk.call( + I64, + "js_regexp_new_factory_site", + &[ + (I64, &pattern_handle), + (I64, &flags_handle), + (I64, &site_key), + (I64, &identity), + ], + ) + } else { + blk.call( + I64, + "js_regexp_new_site", + &[ + (I64, &pattern_handle), + (I64, &flags_handle), + (I64, &site_key), + ], + ) + }; Ok(nanbox_pointer_inline(blk, &result)) } diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index a360d47452..60b33c7168 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -179,6 +179,8 @@ mod call_spread_short_tests; mod issue7628_rooting_tests; #[cfg(test)] mod readonly_collection_tests; +#[cfg(test)] +mod regex_site_test_tests; pub(crate) mod shadow_slot; #[cfg(test)] mod slice7_rooting_tests; @@ -265,6 +267,13 @@ pub(crate) struct FnCtx<'a> { /// module code uses `module_init`. pub source_function: String, pub source_function_slug: String, + /// Public callable symbol when this body is proven to be exactly + /// `function () { return /literal/flags; }`. The proof is structural at + /// the HIR function boundary (zero parameters, one return statement, no + /// async/generator machinery). Regex literal lowering passes this + /// identity to the runtime only for that shape; ordinary literals retain + /// fresh-object semantics. + pub regex_factory_identity: Option, /// Stable id for the labeled loop currently being lowered. pub active_region_id: Option, /// Full native-region fact graph collected for this lowered HIR region. diff --git a/crates/perry-codegen/src/expr/regex_site_test_tests.rs b/crates/perry-codegen/src/expr/regex_site_test_tests.rs new file mode 100644 index 0000000000..02817ae442 --- /dev/null +++ b/crates/perry-codegen/src/expr/regex_site_test_tests.rs @@ -0,0 +1,206 @@ +//! Allocation-free regex `.test` site lowering. These are IR-shape tests so +//! deleting a specialization while leaving the runtime helpers behind fails. + +use perry_hir::types::Type; +use perry_hir::{Expr, Function, Module, ModuleInitKind, Param, Stmt}; + +fn function( + id: u32, + name: &str, + params: Vec, + body: Vec, + return_type: Type, +) -> Function { + Function { + id, + name: name.to_string(), + type_params: Vec::new(), + params, + return_type, + body, + is_async: false, + is_generator: false, + is_strict: false, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + } +} + +fn param(id: u32, name: &str) -> Param { + Param { + id, + name: name.to_string(), + ty: Type::String, + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + } +} + +fn call(callee: Expr, args: Vec) -> Expr { + Expr::Call { + callee: Box::new(callee), + args, + type_args: Vec::new(), + byte_offset: 0, + } +} + +fn property(object: Expr, property: &str) -> Expr { + Expr::PropertyGet { + object: Box::new(object), + property: property.to_string(), + byte_offset: 0, + } +} + +fn compile(functions: Vec) -> String { + let mut module = Module::new("regex_site_test.ts"); + module.functions = functions; + module.init_kind = ModuleInitKind::Eager; + String::from_utf8( + crate::compile_module(&module, super::class_field_barrier_tests::ir_opts()) + .expect("regex site fixture compiles"), + ) + .expect("LLVM IR is UTF-8") +} + +#[test] +fn direct_literal_test_uses_the_site_header_and_post_get_dispatch() { + let body = vec![Stmt::Return(Some(Expr::RegExpTest { + regex: Box::new(Expr::RegExp { + pattern: "x".to_string(), + flags: "g".to_string(), + }), + string: Box::new(Expr::LocalGet(10)), + }))]; + let ir = compile(vec![function( + 1, + "direct", + vec![param(10, "s")], + body, + Type::Boolean, + )]); + assert!(ir.contains("call i64 @js_regexp_site_test_new("), "{ir}"); + assert!( + ir.contains("call double @js_regexp_site_test_get_method("), + "{ir}" + ); + assert!( + ir.contains("call double @js_regexp_site_test_dispatch("), + "{ir}" + ); +} + +#[test] +fn escaping_literal_is_not_transformed_and_keeps_one_stateful_receiver() { + let body = vec![ + Stmt::Let { + id: 20, + name: "r".to_string(), + ty: Type::Named("RegExp".to_string()), + mutable: false, + init: Some(Expr::RegExp { + pattern: "x".to_string(), + flags: "g".to_string(), + }), + }, + Stmt::Expr(Expr::RegExpTest { + regex: Box::new(Expr::LocalGet(20)), + string: Box::new(Expr::LocalGet(21)), + }), + Stmt::Return(Some(Expr::RegExpTest { + regex: Box::new(Expr::LocalGet(20)), + string: Box::new(Expr::LocalGet(22)), + })), + ]; + let ir = compile(vec![function( + 1, + "escaping", + vec![param(21, "a"), param(22, "b")], + body, + Type::Boolean, + )]); + // The fixture can be emitted in more than one specialized clone. Every + // clone must retain one ordinary construction and two stateful tests. + let constructions = ir.matches("call i64 @js_regexp_new_site(").count(); + assert!(constructions >= 1, "{ir}"); + assert_eq!( + ir.matches("call i64 @js_regexp_site_test_new(").count(), + 0, + "{ir}" + ); + assert_eq!( + ir.matches("call i32 @js_regexp_test(").count(), + constructions * 2, + "{ir}" + ); +} + +fn exact_factory() -> Function { + function( + 1, + "factory", + Vec::new(), + vec![Stmt::Return(Some(Expr::RegExp { + pattern: "x".to_string(), + flags: "g".to_string(), + }))], + Type::Named("RegExp".to_string()), + ) +} + +#[test] +fn direct_factory_call_records_function_identity_and_uses_the_caller_site() { + let inner = call(Expr::FuncRef(1), Vec::new()); + let outer = call(property(inner, "test"), vec![Expr::LocalGet(30)]); + let caller = function( + 2, + "caller", + vec![param(30, "s")], + vec![Stmt::Return(Some(outer))], + Type::Boolean, + ); + let ir = compile(vec![exact_factory(), caller]); + assert!(ir.contains("call i64 @js_regexp_new_factory_site("), "{ir}"); + assert!( + ir.contains("ptrtoint ptr @perry_fn_"), + "factory identity missing: {ir}" + ); + assert!( + ir.contains("call double @js_regexp_site_factory_call_value("), + "{ir}" + ); + assert!( + ir.contains("call double @js_regexp_site_test_dispatch("), + "{ir}" + ); +} + +#[test] +fn namespace_member_factory_call_uses_the_member_wrapper() { + // The runtime wrapper resolves `default` first, then activates the site + // only while invoking the resolved function. `Undefined` is sufficient + // for an IR-shape fixture; runtime tests exercise a real namespace object. + let inner = call(property(Expr::Undefined, "default"), Vec::new()); + let outer = call(property(inner, "test"), vec![Expr::String("x".to_string())]); + let ir = compile(vec![function( + 1, + "member", + Vec::new(), + vec![Stmt::Return(Some(outer))], + Type::Any, + )]); + assert!( + ir.contains("call double @js_regexp_site_factory_call_method("), + "{ir}" + ); + assert!( + ir.contains("call double @js_regexp_site_test_dispatch("), + "{ir}" + ); +} diff --git a/crates/perry-codegen/src/runtime_decls/mod.rs b/crates/perry-codegen/src/runtime_decls/mod.rs index fab34b46eb..99973fc3e1 100644 --- a/crates/perry-codegen/src/runtime_decls/mod.rs +++ b/crates/perry-codegen/src/runtime_decls/mod.rs @@ -264,5 +264,42 @@ mod tests { plain.starts_with("declare i64 @js_regexp_new(i64, i64)"), "got: {plain}" ); + + for (name, signature) in [ + ( + "js_regexp_site_test_new", + "declare i64 @js_regexp_site_test_new(i64, i64, i64)", + ), + ( + "js_regexp_new_factory_site", + "declare i64 @js_regexp_new_factory_site(i64, i64, i64, i64)", + ), + ( + "js_regexp_site_factory_call_value", + "declare double @js_regexp_site_factory_call_value(i64, double)", + ), + ( + "js_regexp_site_factory_call_method", + "declare double @js_regexp_site_factory_call_method(i64, double, double)", + ), + ( + "js_regexp_site_test_get_method", + "declare double @js_regexp_site_test_get_method(i64, double)", + ), + ( + "js_regexp_site_test_dispatch", + "declare double @js_regexp_site_test_dispatch(i64, double, double, double)", + ), + ] { + let line = module + .declaration_lines() + .find(|(candidate, _)| *candidate == name) + .map(|(_, line)| line) + .unwrap_or_else(|| panic!("missing declaration for {name}")); + assert!( + line.starts_with(signature), + "wrong declaration for {name}: {line}" + ); + } } } diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index 911e318a47..2e99cd4db1 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -1319,6 +1319,20 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { // unit tests. `runtime_decls::tests` asserts the name AND the arity: a // wrong arity parses and miscompiles. module.declare_function("js_regexp_new_site", I64, &[I64, I64, I64]); + module.declare_function("js_regexp_new_factory_site", I64, &[I64, I64, I64, I64]); + module.declare_function("js_regexp_site_test_new", I64, &[I64, I64, I64]); + module.declare_function("js_regexp_site_factory_call_value", DOUBLE, &[I64, DOUBLE]); + module.declare_function( + "js_regexp_site_factory_call_method", + DOUBLE, + &[I64, DOUBLE, DOUBLE], + ); + module.declare_function("js_regexp_site_test_get_method", DOUBLE, &[I64, DOUBLE]); + module.declare_function( + "js_regexp_site_test_dispatch", + DOUBLE, + &[I64, DOUBLE, DOUBLE, DOUBLE], + ); // Full ECMAScript RegExp constructor: NaN-boxed pattern + flags in, handles // RegExp/undefined/object patterns and ToString-coerced flags. module.declare_function("js_regexp_construct", I64, &[DOUBLE, DOUBLE]); diff --git a/crates/perry-runtime/src/exception.rs b/crates/perry-runtime/src/exception.rs index a617be1909..63298fd0a2 100644 --- a/crates/perry-runtime/src/exception.rs +++ b/crates/perry-runtime/src/exception.rs @@ -137,6 +137,11 @@ struct ExceptionState { /// evaluating the right-hand side of a guarded private write skips the /// normal consumer, so catch entry must discard the orphaned hint. private_member_access_hint_depths: Box<[usize]>, + /// Active allocation-free regex-factory sites at handler entry. A + /// non-literal replacement callee can throw before the wrapper's normal + /// pop, so catch entry discards the orphaned identity frame. + #[cfg(feature = "regex-engine")] + regex_factory_site_depths: Box<[usize]>, /// #6559: dyn-eval interpreter state (rooted-stack length + interpreter /// call depth, packed) captured when each `try` was pushed. A throw /// `longjmp`s past interpreter Rust frames without running their @@ -168,6 +173,8 @@ impl ExceptionState { private_lexical_brand_depths: vec![0usize; MAX_TRY_DEPTH].into_boxed_slice(), derived_super_binding_depths: vec![0usize; MAX_TRY_DEPTH].into_boxed_slice(), private_member_access_hint_depths: vec![0usize; MAX_TRY_DEPTH].into_boxed_slice(), + #[cfg(feature = "regex-engine")] + regex_factory_site_depths: vec![0usize; MAX_TRY_DEPTH].into_boxed_slice(), #[cfg(feature = "dyn-eval")] dyn_eval_savepoints: vec![0u64; MAX_TRY_DEPTH].into_boxed_slice(), try_depth: 0, @@ -244,6 +251,11 @@ fn try_push_with_kind(kind: HandlerKind) -> *mut i32 { crate::object::derived_super_binding_stack_savepoint(); (*s).private_member_access_hint_depths[depth] = crate::object::private_member_access_hints_savepoint(); + #[cfg(feature = "regex-engine")] + { + (*s).regex_factory_site_depths[depth] = + crate::regex::site_test::active_factory_stack_savepoint(); + } // #6559: capture the dyn-eval interpreter's rooted-stack length + // call depth, so a caught throw restores interpreter state exactly // like the shadow stack. @@ -479,6 +491,10 @@ pub extern "C-unwind" fn js_throw(value: f64) -> ! { crate::object::private_member_access_hints_restore( (*s).private_member_access_hint_depths[depth], ); + #[cfg(feature = "regex-engine")] + crate::regex::site_test::active_factory_stack_restore( + (*s).regex_factory_site_depths[depth], + ); // #6559: restore the dyn-eval interpreter's rooted stack + call depth // (interpreter Rust frames unwound by this longjmp never run their // truncate/decrement epilogues). @@ -845,6 +861,10 @@ pub(crate) fn test_unwind_innermost_shadow_restore() { crate::object::private_member_access_hints_restore( (*s).private_member_access_hint_depths[depth], ); + #[cfg(feature = "regex-engine")] + crate::regex::site_test::active_factory_stack_restore( + (*s).regex_factory_site_depths[depth], + ); }); } diff --git a/crates/perry-runtime/src/gc/census.rs b/crates/perry-runtime/src/gc/census.rs index 743c93fb69..25246b59fe 100644 --- a/crates/perry-runtime/src/gc/census.rs +++ b/crates/perry-runtime/src/gc/census.rs @@ -575,6 +575,8 @@ fn side_tables() -> Vec { rows.extend(crate::module_require::path_registry_census()); rows.extend(crate::timer::timer_tables_census()); rows.push(crate::symbol::symbol_registry_census()); + #[cfg(feature = "regex-engine")] + rows.extend(crate::regex::site_test::side_table_census()); let (masks, typed) = super::layout_tables::per_object_layout_table_sizes(); rows.push(("gc.layout_slot_masks", masks, masks * 24)); rows.push(("gc.typed_layouts", typed, typed * 24)); @@ -586,6 +588,23 @@ fn side_tables() -> Vec { rows } +#[cfg(test)] +mod regex_census_tests { + #[test] + fn regex_side_tables_are_registered_with_the_census_prefix() { + let names: Vec<_> = super::side_tables() + .into_iter() + .filter_map(|(name, _, _)| name.starts_with("regex.").then_some(name)) + .collect(); + assert!(names.contains(&"regex.content_cache"), "rows: {names:?}"); + assert!(names.contains(&"regex.literal_sites"), "rows: {names:?}"); + assert!( + names.contains(&"regex.site_test_headers"), + "rows: {names:?}" + ); + } +} + // --------------------------------------------------------------------------- // Process-level numbers // --------------------------------------------------------------------------- diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 13ae921a77..3cb72ce193 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -1003,6 +1003,8 @@ pub fn gc_init() { reg_scanner!(async_hooks_mutable_root_scanner); reg_scanner!(shape_cache_mutable_root_scanner); reg_scanner!(crate::regex::scan_last_exec_groups_root_mut); + #[cfg(feature = "regex-engine")] + reg_scanner!(crate::regex::site_test::scan_roots_mut); // #7211: the eight interned `typeof` result strings, and JSON.rawJSON's // interned `"rawJSON"` key. Both are thread-local caches of a RAW // `StringHeader*` allocated in the nursery and referenced by nothing else, diff --git a/crates/perry-runtime/src/hot_diag.rs b/crates/perry-runtime/src/hot_diag.rs index 10f063eebe..c68c4bb78a 100644 --- a/crates/perry-runtime/src/hot_diag.rs +++ b/crates/perry-runtime/src/hot_diag.rs @@ -203,6 +203,14 @@ pub struct RegexDiag { /// CONTENT-keyed cache; a site hit never reaches it, so the two are /// disjoint and `site_key_hit + site_hit <= new`. pub new_site_key_hit: u64, + /// `.test` evaluations served by a site-rooted RegExp header instead of a + /// fresh header allocation. + pub site_test_no_alloc: u64, + /// Validation declines, split so a perf run proves which guard fired. + pub site_test_declined: u64, + pub site_test_declined_patched_prototype: u64, + pub site_test_declined_callee_mismatch: u64, + pub site_test_declined_non_literal: u64, #[cfg(test)] test_program_builds: u64, #[cfg(test)] @@ -381,7 +389,8 @@ impl RegexDiag { desc_regexp_probes={} desc_regexp_meta_negative={} \ barrier_taken={} barrier_gated={} header_bytes={} site_verify_bytes={} \ side_table_inserts={} site_key_hit={} ptr_ins={} src_ins={} \ - ptr_rm={} src_rm={} rekeys={}", + ptr_rm={} src_rm={} rekeys={} site_test_no_alloc={} \ + site_test_declined={}(patched_prototype={},callee_mismatch={},non_literal={})", self.new_calls, self.new_validated_hit, self.new_site_hit, @@ -417,6 +426,11 @@ impl RegexDiag { self.pointer_table_removals, self.source_table_removals, self.side_table_rekeys, + self.site_test_no_alloc, + self.site_test_declined, + self.site_test_declined_patched_prototype, + self.site_test_declined_callee_mismatch, + self.site_test_declined_non_literal, ); // Merge by content (prefix, len, flags): distinct literal sites with // the same pattern are one row. diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 12c0ef67b4..d4bcb265ee 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -6,10 +6,8 @@ #[cfg(feature = "regex-engine")] use regex::Regex; use std::cell::RefCell; -// Every use of `HashMap` in this file is inside a `#[cfg(feature = "regex-engine")]` -// block, so an unconditional import is an unused-import error under the -// `warnings` job's `-D warnings` when `perry`'s own binaries pull the runtime -// in without that feature. +// Every `HashMap` use is behind `regex-engine`; gate the import too so the +// feature-off `-D warnings` build does not see it as unused. #[cfg(feature = "regex-engine")] use std::collections::HashMap; use std::ptr; @@ -77,6 +75,8 @@ mod site_cache; #[cfg(feature = "regex-engine")] mod site_key; #[cfg(feature = "regex-engine")] +pub(crate) mod site_test; +#[cfg(feature = "regex-engine")] mod unicode17; #[cfg(feature = "regex-engine")] mod unicode17_data; @@ -111,10 +111,9 @@ pub use properties::{ js_regexp_set_last_index, js_regexp_to_string, }; -/// Class id for `RegExp String Iterator` exotic objects. Referenced by the -/// always-linked iterator-prototype dispatch, so it stays ungated even when -/// the regex engine (which produces these iterators) is compiled out. +/// Class id shared with the always-linked RegExp string-iterator dispatch. pub const REGEXP_STRING_ITERATOR_CLASS_ID: u32 = 0xFFFF_000A; + #[cfg(feature = "regex-engine")] use replace_expand::expand_js_replacement; #[cfg(feature = "regex-engine")] diff --git a/crates/perry-runtime/src/regex/site_cache.rs b/crates/perry-runtime/src/regex/site_cache.rs index 5f6533cfd5..b8081ee907 100644 --- a/crates/perry-runtime/src/regex/site_cache.rs +++ b/crates/perry-runtime/src/regex/site_cache.rs @@ -121,6 +121,22 @@ fn entry_count(cache: &ContentMap) -> usize { cache.values().map(Vec::len).sum() } +pub(super) fn census() -> crate::gc::census::SideTableRow { + SITE_CACHE.with(|cache| { + let cache = cache.borrow(); + let entries = entry_count(&cache); + // The content payload dominates; include its owned pattern/flags bytes + // as well as one entry record. Bucket/control-byte overhead is small + // and deliberately left as an estimate, matching the census contract. + let bytes = cache + .values() + .flatten() + .map(|entry| std::mem::size_of::() + entry.pattern.len() + entry.flags.len()) + .sum(); + ("regex.content_cache", entries, bytes) + }) +} + /// Remove one entry that has no recorded literal site. The scan happens only /// on a distinct-content miss at capacity; literal-site hits never reach it. fn evict_one_dynamic(cache: &mut ContentMap) -> bool { diff --git a/crates/perry-runtime/src/regex/site_key.rs b/crates/perry-runtime/src/regex/site_key.rs index f9737f7ef6..6669775cc5 100644 --- a/crates/perry-runtime/src/regex/site_key.rs +++ b/crates/perry-runtime/src/regex/site_key.rs @@ -147,6 +147,18 @@ fn slot_of(key: usize) -> usize { (key >> 3) & (SLOTS - 1) } +pub(super) fn census() -> crate::gc::census::SideTableRow { + SITE_KEY_TABLE.with(|table| { + let table = table.borrow(); + let entries = table.iter().filter(|entry| entry.is_some()).count(); + ( + "regex.literal_sites", + entries, + table.capacity() * std::mem::size_of::>(), + ) + }) +} + /// The entry recorded for `key`, or `None`. pub(super) fn lookup(key: usize, raw_flags: &str) -> Option { if !enabled() || key == 0 { diff --git a/crates/perry-runtime/src/regex/site_test.rs b/crates/perry-runtime/src/regex/site_test.rs new file mode 100644 index 0000000000..ebc40c80c8 --- /dev/null +++ b/crates/perry-runtime/src/regex/site_test.rs @@ -0,0 +1,880 @@ +//! Allocation-free headers for regex literals whose only observable use at a +//! source site is the receiver of one `.test` call. +//! +//! A cached header is a strong mutable GC root. It is never returned from a +//! transformed expression: direct literals feed it only to the paired test +//! dispatch, and a factory participates only while its structurally proven +//! body (`return /literal/`) is executing under a recorded caller site. The +//! caller still invokes the factory on every evaluation, so reassignment and +//! member lookup retain their ordinary effects. + +use std::cell::RefCell; +use std::collections::HashMap; + +use super::{is_valid_regex_ptr, js_regexp_new_impl, RegExpHeader}; + +struct Entry { + header: *mut RegExpHeader, + /// Zero for a direct literal; otherwise the compiler-emitted public + /// function entry for the exact-return factory body. + factory_identity: usize, + /// Set by the construction/factory entry after this evaluation's + /// canonicality check, consumed by the immediately following property Get. + approved: bool, +} + +#[derive(Clone, Copy)] +struct ActiveFactorySite { + site_key: usize, + /// Native entry resolved from the actual callee value before invocation. + /// A literal in a nested helper must not consume its caller's site. + expected_identity: usize, + handled_by_literal: bool, +} + +crate::perry_thread_local! { + static SITE_TEST_HEADERS: RefCell> = RefCell::new(HashMap::new()); + static ACTIVE_FACTORY_SITES: RefCell> = RefCell::new(Vec::new()); +} + +const BUILTIN_TEST_MARKER: u64 = crate::value::TAG_MARKER; + +#[inline] +fn note_no_alloc() { + #[cfg(test)] + TEST_NO_ALLOC.with(|counter| counter.set(counter.get() + 1)); + if crate::hot_diag::regex_on() { + crate::hot_diag::regex_counters(|d| d.site_test_no_alloc += 1); + } +} + +#[inline] +fn note_declined(reason: DeclineReason) { + #[cfg(test)] + { + TEST_DECLINED.with(|counter| counter.set(counter.get() + 1)); + let bucket = match reason { + DeclineReason::PatchedPrototype => &TEST_DECLINED_PATCHED, + DeclineReason::CalleeMismatch => &TEST_DECLINED_CALLEE, + DeclineReason::NonLiteral => &TEST_DECLINED_NON_LITERAL, + }; + bucket.with(|counter| counter.set(counter.get() + 1)); + } + if crate::hot_diag::regex_on() { + crate::hot_diag::regex_counters(|d| { + d.site_test_declined += 1; + match reason { + DeclineReason::PatchedPrototype => d.site_test_declined_patched_prototype += 1, + DeclineReason::CalleeMismatch => d.site_test_declined_callee_mismatch += 1, + DeclineReason::NonLiteral => d.site_test_declined_non_literal += 1, + } + }); + } +} + +#[cfg(test)] +thread_local! { + static TEST_NO_ALLOC: std::cell::Cell = const { std::cell::Cell::new(0) }; + static TEST_DECLINED: std::cell::Cell = const { std::cell::Cell::new(0) }; + static TEST_DECLINED_PATCHED: std::cell::Cell = const { std::cell::Cell::new(0) }; + static TEST_DECLINED_CALLEE: std::cell::Cell = const { std::cell::Cell::new(0) }; + static TEST_DECLINED_NON_LITERAL: std::cell::Cell = const { std::cell::Cell::new(0) }; + static TEST_ALLOCATIONS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[derive(Clone, Copy)] +enum DeclineReason { + PatchedPrototype, + CalleeMismatch, + NonLiteral, +} + +fn lookup(site_key: usize) -> Option<(*mut RegExpHeader, usize)> { + SITE_TEST_HEADERS.with(|table| { + table + .borrow() + .get(&site_key) + .map(|entry| (entry.header, entry.factory_identity)) + }) +} + +fn install(site_key: usize, header: *mut RegExpHeader, factory_identity: usize) { + if site_key == 0 || header.is_null() { + return; + } + let mut entry = Entry { + header: std::ptr::null_mut(), + factory_identity, + approved: true, + }; + // GC_STORE_AUDIT(ROOT): `entry.header` becomes a mutable raw root when the + // entry is inserted into SITE_TEST_HEADERS; `scan_roots_mut` visits it. + // SAFETY: both pointers are either null or allocator-returned RegExp + // addresses; the destination is the root slot that will own `header`. + unsafe { + crate::gc::runtime_store_root_raw_mut_ptr_slot(&mut entry.header, header); + } + SITE_TEST_HEADERS.with(|table| { + table.borrow_mut().insert(site_key, entry); + }); +} + +#[inline] +fn allocate( + pattern: *const crate::StringHeader, + flags: *const crate::StringHeader, + site_key: usize, +) -> *mut RegExpHeader { + #[cfg(test)] + TEST_ALLOCATIONS.with(|counter| counter.set(counter.get() + 1)); + js_regexp_new_impl(pattern, flags, site_key) +} + +fn approve(site_key: usize, header: *mut RegExpHeader) { + SITE_TEST_HEADERS.with(|table| { + if let Some(entry) = table.borrow_mut().get_mut(&site_key) { + if entry.header == header { + entry.approved = true; + } + } + }); +} + +fn take_approval(site_key: usize, header: *mut RegExpHeader) -> bool { + SITE_TEST_HEADERS.with(|table| { + let mut table = table.borrow_mut(); + let Some(entry) = table.get_mut(&site_key) else { + return false; + }; + if entry.header != header || !entry.approved { + return false; + } + entry.approved = false; + true + }) +} + +fn canonical_rooted_header(header: *mut RegExpHeader) -> Option<*mut RegExpHeader> { + if !is_valid_regex_ptr(header) { + return None; + } + let scope = crate::gc::RuntimeHandleScope::new(); + let rooted = scope.root_raw_mut_ptr(header); + let value = f64::from_bits(crate::value::JSValue::pointer(header.cast::()).bits()); + if !crate::object::regex_proto_thunks::regexp_prototype_test_is_canonical(value) { + return None; + } + Some(rooted.get_raw_mut_ptr()) +} + +/// Direct literal receiver for `/literal/flags.test(arg)`. +#[no_mangle] +pub extern "C" fn js_regexp_site_test_new( + pattern: *const crate::StringHeader, + flags: *const crate::StringHeader, + site_key: i64, +) -> *mut RegExpHeader { + let site_key = site_key as usize; + if let Some((header, factory_identity)) = lookup(site_key) { + if factory_identity == 0 { + if let Some(header) = canonical_rooted_header(header) { + approve(site_key, header); + // Price the construction this function avoided. The cold + // install is deliberately excluded. + note_no_alloc(); + return header; + } + note_declined(DeclineReason::PatchedPrototype); + return allocate(pattern, flags, site_key); + } + } + + let header = allocate(pattern, flags, site_key); + if let Some(header) = canonical_rooted_header(header) { + install(site_key, header, 0); + header + } else { + note_declined(DeclineReason::PatchedPrototype); + header + } +} + +fn mark_active_literal(factory_identity: usize) -> Option { + ACTIVE_FACTORY_SITES.with(|stack| { + let mut stack = stack.borrow_mut(); + let frame = stack.last_mut()?; + if frame.expected_identity != factory_identity { + return None; + } + frame.handled_by_literal = true; + Some(frame.site_key) + }) +} + +/// Literal construction inside a HIR-proven exact regex factory. Outside a +/// transformed caller it is exactly `js_regexp_new_site`; inside one it uses +/// the caller site's `(site, function-entry)` record. +#[no_mangle] +pub extern "C" fn js_regexp_new_factory_site( + pattern: *const crate::StringHeader, + flags: *const crate::StringHeader, + literal_site_key: i64, + factory_identity: i64, +) -> *mut RegExpHeader { + let factory_identity = factory_identity as usize; + let Some(call_site_key) = mark_active_literal(factory_identity) else { + return allocate(pattern, flags, literal_site_key as usize); + }; + if let Some((header, recorded_identity)) = lookup(call_site_key) { + if recorded_identity != factory_identity { + note_declined(DeclineReason::CalleeMismatch); + return allocate(pattern, flags, literal_site_key as usize); + } + if let Some(header) = canonical_rooted_header(header) { + approve(call_site_key, header); + // Price the construction this function avoided. The cold + // install is deliberately excluded. + note_no_alloc(); + return header; + } + note_declined(DeclineReason::PatchedPrototype); + return allocate(pattern, flags, literal_site_key as usize); + } + + let header = allocate(pattern, flags, literal_site_key as usize); + if let Some(header) = canonical_rooted_header(header) { + install(call_site_key, header, factory_identity); + header + } else { + note_declined(DeclineReason::PatchedPrototype); + header + } +} + +struct ActiveFactoryGuard { + depth_before: usize, +} + +impl ActiveFactoryGuard { + fn push(site_key: usize, expected_identity: usize) -> Self { + let depth_before = ACTIVE_FACTORY_SITES.with(|stack| { + let depth_before = stack.borrow().len(); + stack.borrow_mut().push(ActiveFactorySite { + site_key, + expected_identity, + handled_by_literal: false, + }); + depth_before + }); + Self { depth_before } + } + + fn handled(&self) -> bool { + ACTIVE_FACTORY_SITES.with(|stack| { + stack + .borrow() + .get(self.depth_before) + .is_some_and(|frame| frame.handled_by_literal) + }) + } +} + +impl Drop for ActiveFactoryGuard { + fn drop(&mut self) { + ACTIVE_FACTORY_SITES.with(|stack| { + let mut stack = stack.borrow_mut(); + // `js_throw` may already have restored the stack before a system + // unwind runs this Drop. Never pop a still-live outer frame. + if stack.len() > self.depth_before { + stack.truncate(self.depth_before); + } + }); + } +} + +pub(crate) fn active_factory_stack_savepoint() -> usize { + ACTIVE_FACTORY_SITES.with(|stack| stack.borrow().len()) +} + +pub(crate) fn active_factory_stack_restore(depth: usize) { + ACTIVE_FACTORY_SITES.with(|stack| stack.borrow_mut().truncate(depth)); +} + +struct ImplicitThisGuard<'scope> { + previous: crate::gc::RuntimeHandle<'scope>, +} + +impl<'scope> ImplicitThisGuard<'scope> { + fn bind(scope: &'scope crate::gc::RuntimeHandleScope, receiver: f64) -> Self { + Self { + previous: scope.root_nanbox_f64(crate::object::js_implicit_this_set(receiver)), + } + } +} + +impl Drop for ImplicitThisGuard<'_> { + fn drop(&mut self) { + crate::object::js_implicit_this_set(self.previous.get_nanbox_f64()); + } +} + +fn call_value_at_site(site_key: usize, callee: f64, this_value: Option) -> f64 { + let scope = crate::gc::RuntimeHandleScope::new(); + let callee = scope.root_nanbox_f64(callee); + let callee_value = crate::value::JSValue::from_bits(callee.get_nanbox_f64().to_bits()); + let expected_identity = if callee_value.is_pointer() { + let closure = callee_value.as_pointer::(); + crate::closure::get_valid_func_ptr(closure) as usize + } else { + 0 + }; + let this_value = this_value.map(|value| scope.root_nanbox_f64(value)); + let this_guard = this_value + .as_ref() + .map(|value| ImplicitThisGuard::bind(&scope, value.get_nanbox_f64())); + let active = ActiveFactoryGuard::push(site_key, expected_identity); + let result = unsafe { + crate::closure::js_native_call_value(callee.get_nanbox_f64(), std::ptr::null(), 0) + }; + if !active.handled() { + let reason = if lookup(site_key).is_some() { + DeclineReason::CalleeMismatch + } else { + DeclineReason::NonLiteral + }; + note_declined(reason); + } + drop(active); + drop(this_guard); + result +} + +#[cfg(panic = "abort")] +#[no_mangle] +pub extern "C" fn js_regexp_site_factory_call_value(site_key: i64, callee: f64) -> f64 { + call_value_at_site(site_key as usize, callee, None) +} + +#[cfg(not(panic = "abort"))] +#[no_mangle] +pub extern "C-unwind" fn js_regexp_site_factory_call_value(site_key: i64, callee: f64) -> f64 { + call_value_at_site(site_key as usize, callee, None) +} + +#[cfg(panic = "abort")] +#[no_mangle] +pub unsafe extern "C" fn js_regexp_site_factory_call_method( + site_key: i64, + receiver: f64, + method_key: f64, +) -> f64 { + unsafe { site_factory_call_method_impl(site_key, receiver, method_key) } +} + +#[cfg(not(panic = "abort"))] +#[no_mangle] +pub unsafe extern "C-unwind" fn js_regexp_site_factory_call_method( + site_key: i64, + receiver: f64, + method_key: f64, +) -> f64 { + unsafe { site_factory_call_method_impl(site_key, receiver, method_key) } +} + +#[inline(always)] +unsafe fn site_factory_call_method_impl(site_key: i64, receiver: f64, method_key: f64) -> f64 { + // Resolve the member before activating the factory site. An accessor is + // arbitrary user code and must not let an incidental regex construction + // masquerade as the function the call actually selected. + let scope = crate::gc::RuntimeHandleScope::new(); + let receiver = scope.root_nanbox_f64(receiver); + let method_key = scope.root_nanbox_f64(method_key); + let method = + crate::value::js_dyn_index_get(receiver.get_nanbox_f64(), method_key.get_nanbox_f64()); + call_value_at_site(site_key as usize, method, Some(receiver.get_nanbox_f64())) +} + +/// Resolve `.test` at the spec-mandated point (before argument evaluation). +/// The internal marker means the builtin was validated for the cached header; +/// every generic decline returns the actual property value instead. +#[cfg(panic = "abort")] +#[no_mangle] +pub extern "C" fn js_regexp_site_test_get_method(site_key: i64, receiver: f64) -> f64 { + site_test_get_method_impl(site_key, receiver) +} + +#[cfg(not(panic = "abort"))] +#[no_mangle] +pub extern "C-unwind" fn js_regexp_site_test_get_method(site_key: i64, receiver: f64) -> f64 { + site_test_get_method_impl(site_key, receiver) +} + +#[inline(always)] +fn site_test_get_method_impl(site_key: i64, receiver: f64) -> f64 { + let site_key = site_key as usize; + let receiver_value = crate::value::JSValue::from_bits(receiver.to_bits()); + let receiver_ptr = receiver_value + .is_pointer() + .then(|| receiver_value.as_pointer::() as *mut RegExpHeader); + if let (Some(receiver_ptr), Some((header, _))) = (receiver_ptr, lookup(site_key)) { + if receiver_ptr == header { + if take_approval(site_key, header) { + return f64::from_bits(BUILTIN_TEST_MARKER); + } + note_declined(DeclineReason::PatchedPrototype); + } + } + + let scope = crate::gc::RuntimeHandleScope::new(); + let receiver = scope.root_nanbox_f64(receiver); + let key = crate::string::intern_ascii_literal(b"test"); + crate::value::js_dyn_index_get( + receiver.get_nanbox_f64(), + crate::js_nanbox_string(key as i64), + ) +} + +/// Consume the method captured above and the now-evaluated argument. +#[cfg(panic = "abort")] +#[no_mangle] +pub extern "C" fn js_regexp_site_test_dispatch( + _site_key: i64, + receiver: f64, + method: f64, + argument: f64, +) -> f64 { + site_test_dispatch_impl(receiver, method, argument) +} + +#[cfg(not(panic = "abort"))] +#[no_mangle] +pub extern "C-unwind" fn js_regexp_site_test_dispatch( + _site_key: i64, + receiver: f64, + method: f64, + argument: f64, +) -> f64 { + site_test_dispatch_impl(receiver, method, argument) +} + +#[inline(always)] +fn site_test_dispatch_impl(receiver: f64, method: f64, argument: f64) -> f64 { + let scope = crate::gc::RuntimeHandleScope::new(); + let receiver = scope.root_nanbox_f64(receiver); + let argument = scope.root_nanbox_f64(argument); + if method.to_bits() == BUILTIN_TEST_MARKER { + let string = crate::value::js_jsvalue_to_string_coerce(argument.get_nanbox_f64()); + let receiver = receiver.get_nanbox_f64(); + let value = crate::value::JSValue::from_bits(receiver.to_bits()); + if !value.is_pointer() { + return f64::from_bits(crate::value::TAG_FALSE); + } + let header = value.as_pointer::() as *mut RegExpHeader; + if !is_valid_regex_ptr(header) { + return f64::from_bits(crate::value::TAG_FALSE); + } + // A fresh literal begins every evaluation at zero. `test` may write + // the cached header's lastIndex, but no reference to this header leaves + // the transformed expression and the next call resets it again. + unsafe { + (*header).last_index = crate::value::JSValue::number(0.0).bits(); + } + return f64::from_bits( + crate::value::JSValue::bool(super::js_regexp_test(header, string) != 0).bits(), + ); + } + + let method = scope.root_nanbox_f64(method); + let _this_guard = ImplicitThisGuard::bind(&scope, receiver.get_nanbox_f64()); + let args = [argument.get_nanbox_f64()]; + unsafe { crate::closure::js_native_call_value(method.get_nanbox_f64(), args.as_ptr(), 1) } +} + +/// Strong root for every cached header. The visitor rewrites entries in +/// place during evacuation, so later probes never retain a from-space address. +pub(crate) fn scan_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + SITE_TEST_HEADERS.with(|table| { + for entry in table.borrow_mut().values_mut() { + visitor.visit_raw_mut_ptr_slot(&mut entry.header); + } + }); +} + +pub(super) fn census() -> crate::gc::census::SideTableRow { + SITE_TEST_HEADERS.with(|table| { + let table = table.borrow(); + ( + "regex.site_test_headers", + table.len(), + crate::gc::census::map_bytes(&*table), + ) + }) +} + +pub(crate) fn side_table_census() -> Vec { + vec![ + super::site_cache::census(), + super::site_key::census(), + census(), + ] +} + +#[cfg(test)] +pub(super) fn test_reset() { + SITE_TEST_HEADERS.with(|table| table.borrow_mut().clear()); + ACTIVE_FACTORY_SITES.with(|stack| stack.borrow_mut().clear()); + TEST_NO_ALLOC.with(|counter| counter.set(0)); + TEST_DECLINED.with(|counter| counter.set(0)); + TEST_DECLINED_PATCHED.with(|counter| counter.set(0)); + TEST_DECLINED_CALLEE.with(|counter| counter.set(0)); + TEST_DECLINED_NON_LITERAL.with(|counter| counter.set(0)); + TEST_ALLOCATIONS.with(|counter| counter.set(0)); +} + +#[cfg(test)] +pub(super) fn test_header(site_key: usize) -> Option { + lookup(site_key).map(|(header, _)| header as usize) +} + +#[cfg(test)] +mod tests { + use super::*; + + static DIRECT_G: u64 = 1; + static DIRECT_Y: u64 = 2; + static FACTORY_CALL: u64 = 3; + static MEMBER_CALL: u64 = 4; + static NESTED_FACTORY_CALL: u64 = 5; + static NESTED_WRAPPER_CALLS: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + + fn string(text: &str) -> *mut crate::StringHeader { + crate::string::js_string_from_bytes(text.as_ptr(), text.len() as u32) + } + + fn key(slot: &'static u64) -> i64 { + slot as *const u64 as i64 + } + + fn run_direct(site: i64, flags: &str, input: &str) -> (usize, bool) { + let receiver = js_regexp_site_test_new(string("x"), string(flags), site); + let receiver_value = crate::value::js_nanbox_pointer(receiver as i64); + let method = js_regexp_site_test_get_method(site, receiver_value); + let result = js_regexp_site_test_dispatch( + site, + receiver_value, + method, + crate::js_nanbox_string(string(input) as i64), + ); + (receiver as usize, crate::value::js_is_truthy(result) != 0) + } + + fn run_fresh_generic(flags: &str, input: &str) -> bool { + let receiver = super::super::js_regexp_new(string("x"), string(flags)); + super::super::js_regexp_test(receiver, string(input)) != 0 + } + + fn ensure_regexp_builtins() { + let value = crate::object::builtin_prototype_value("RegExp"); + assert!( + crate::value::JSValue::from_bits(value.to_bits()).is_pointer(), + "the RegExp intrinsic must be installed before testing its recorded test site" + ); + } + + #[test] + fn direct_global_site_allocates_one_header_and_resets_last_index() { + let _lock = crate::gc::global_side_table_test_lock(); + test_reset(); + ensure_regexp_builtins(); + let site = key(&DIRECT_G); + let inputs = ["x", "x", "a", "xx"]; + let mut header = None; + for input in inputs { + let want = run_fresh_generic("g", input); + let (current, got) = run_direct(site, "g", input); + assert_eq!(got, want, "site result must equal a fresh /x/g.test(input)"); + assert_eq!(*header.get_or_insert(current), current); + } + assert_eq!(test_header(site as usize), header); + assert_eq!( + TEST_NO_ALLOC.with(std::cell::Cell::get), + inputs.len() as u64 - 1 + ); + assert_eq!( + TEST_ALLOCATIONS.with(std::cell::Cell::get), + 1, + "the site must allocate exactly its one rooted header" + ); + } + + #[test] + fn direct_sticky_site_starts_each_evaluation_at_zero() { + let _lock = crate::gc::global_side_table_test_lock(); + test_reset(); + ensure_regexp_builtins(); + let site = key(&DIRECT_Y); + let first = run_direct(site, "y", "x"); + let second = run_direct(site, "y", "x"); + let offset_only = run_direct(site, "y", "ax"); + assert_eq!(first.0, second.0, "one rooted header serves the site"); + assert!( + first.1 && second.1, + "lastIndex must reset before the second test" + ); + assert!(!offset_only.1, "fresh /x/y is anchored at index zero"); + assert_eq!(offset_only.1, run_fresh_generic("y", "ax")); + assert_eq!(TEST_ALLOCATIONS.with(std::cell::Cell::get), 1); + } + + #[test] + fn escaping_generic_global_header_carries_last_index_between_tests() { + let _lock = crate::gc::global_side_table_test_lock(); + let re = super::super::js_regexp_new(string("x"), string("g")); + let subject = string("xx"); + assert_ne!(super::super::js_regexp_test(re, subject), 0); + assert_eq!( + unsafe { (*re).last_index }, + crate::value::JSValue::number(1.0).bits() + ); + assert_ne!( + super::super::js_regexp_test(re, subject), + 0, + "the second test must start at the first test's lastIndex, not at zero" + ); + assert_eq!( + unsafe { (*re).last_index }, + crate::value::JSValue::number(2.0).bits() + ); + } + + extern "C" fn exact_factory(_closure: *const crate::closure::ClosureHeader) -> f64 { + let re = js_regexp_new_factory_site( + string("x"), + string("g"), + key(&DIRECT_G), + exact_factory as *const u8 as i64, + ); + crate::value::js_nanbox_pointer(re as i64) + } + + extern "C" fn replacement_factory(_closure: *const crate::closure::ClosureHeader) -> f64 { + let object = crate::object::js_object_alloc(0, 0); + crate::object::js_object_set_field_by_name( + object, + string("test"), + closure_with_arity(patched_test as *const u8, 1), + ); + crate::value::js_nanbox_pointer(object as i64) + } + + #[inline(never)] + extern "C" fn nested_factory_wrapper(_closure: *const crate::closure::ClosureHeader) -> f64 { + // Keep this observably distinct from `exact_factory` under release + // function merging while modeling a non-literal wrapper with effects. + NESTED_WRAPPER_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + exact_factory(std::ptr::null()) + } + + fn closure(function: *const u8) -> f64 { + closure_with_arity(function, 0) + } + + fn closure_with_arity(function: *const u8, arity: usize) -> f64 { + crate::closure::js_register_closure_arity(function, arity as u32); + crate::value::js_nanbox_pointer(crate::closure::js_closure_alloc(function, 0) as i64) + } + + fn dispatch_test(site: i64, receiver: f64, input: &str) -> f64 { + let method = js_regexp_site_test_get_method(site, receiver); + js_regexp_site_test_dispatch( + site, + receiver, + method, + crate::js_nanbox_string(string(input) as i64), + ) + } + + #[test] + fn direct_factory_site_reuses_only_the_recorded_callee() { + let _lock = crate::gc::global_side_table_test_lock(); + test_reset(); + ensure_regexp_builtins(); + let site = key(&FACTORY_CALL); + let callee = closure(exact_factory as *const u8); + let first = js_regexp_site_factory_call_value(site, callee); + let second = js_regexp_site_factory_call_value(site, callee); + assert_eq!( + first.to_bits(), + second.to_bits(), + "the exact factory reuses its header" + ); + assert_eq!(TEST_NO_ALLOC.with(std::cell::Cell::get), 1); + + let replacement = closure(replacement_factory as *const u8); + let receiver = js_regexp_site_factory_call_value(site, replacement); + let result = dispatch_test(site, receiver, "does not contain the pattern"); + assert_eq!(result.to_bits(), crate::value::TAG_TRUE); + assert_eq!( + TEST_DECLINED.with(std::cell::Cell::get), + 1, + "the very next call must record the callee mismatch" + ); + assert_eq!(TEST_DECLINED_CALLEE.with(std::cell::Cell::get), 1); + } + + #[test] + fn nested_exact_factory_cannot_claim_a_different_callees_site() { + let _lock = crate::gc::global_side_table_test_lock(); + test_reset(); + ensure_regexp_builtins(); + NESTED_WRAPPER_CALLS.store(0, std::sync::atomic::Ordering::Relaxed); + let site = key(&NESTED_FACTORY_CALL); + let wrapper = closure(nested_factory_wrapper as *const u8); + let scope = crate::gc::RuntimeHandleScope::new(); + let first = scope.root_nanbox_f64(js_regexp_site_factory_call_value(site, wrapper)); + let second = js_regexp_site_factory_call_value(site, wrapper); + assert_ne!( + first.get_nanbox_f64().to_bits(), + second.to_bits(), + "a nested literal must keep fresh-object semantics" + ); + assert_eq!(test_header(site as usize), None); + assert_eq!(TEST_ALLOCATIONS.with(std::cell::Cell::get), 2); + assert_eq!(TEST_DECLINED.with(std::cell::Cell::get), 2); + assert_eq!(TEST_DECLINED_NON_LITERAL.with(std::cell::Cell::get), 2); + assert_eq!( + NESTED_WRAPPER_CALLS.load(std::sync::atomic::Ordering::Relaxed), + 2 + ); + } + + #[test] + fn caught_throw_restores_an_orphaned_factory_site_frame() { + let _lock = crate::gc::global_side_table_test_lock(); + test_reset(); + let base = active_factory_stack_savepoint(); + let _jump_buffer = crate::exception::js_try_push(); + let active = ActiveFactoryGuard::push(key(&FACTORY_CALL) as usize, usize::MAX); + assert_eq!(active_factory_stack_savepoint(), base + 1); + + // A raw throw skips this Drop. Model that transport, then replay the + // exception path's recorded savepoint restoration. + std::mem::forget(active); + crate::exception::test_unwind_innermost_shadow_restore(); + crate::exception::js_try_end(); + assert_eq!(active_factory_stack_savepoint(), base); + } + + #[test] + fn namespace_member_factory_site_is_covered() { + let _lock = crate::gc::global_side_table_test_lock(); + test_reset(); + ensure_regexp_builtins(); + let site = key(&MEMBER_CALL); + let namespace = crate::object::js_object_alloc(0, 0); + let name = string("default"); + crate::object::js_object_set_field_by_name( + namespace, + name, + closure(exact_factory as *const u8), + ); + let namespace_object = namespace; + let namespace = crate::value::js_nanbox_pointer(namespace_object as i64); + let member = crate::js_nanbox_string(name as i64); + let first = unsafe { js_regexp_site_factory_call_method(site, namespace, member) }; + let second = unsafe { js_regexp_site_factory_call_method(site, namespace, member) }; + assert_eq!(first.to_bits(), second.to_bits()); + assert_eq!(TEST_NO_ALLOC.with(std::cell::Cell::get), 1); + assert_eq!( + test_header(site as usize), + Some(crate::value::js_nanbox_get_pointer(first) as usize) + ); + + crate::object::js_object_set_field_by_name( + namespace_object, + name, + closure(replacement_factory as *const u8), + ); + let replacement = unsafe { js_regexp_site_factory_call_method(site, namespace, member) }; + assert_eq!( + dispatch_test(site, replacement, "no match").to_bits(), + crate::value::TAG_TRUE + ); + assert_eq!( + TEST_DECLINED.with(std::cell::Cell::get), + 1, + "rebinding the namespace member must decline on the next call" + ); + assert_eq!(TEST_DECLINED_CALLEE.with(std::cell::Cell::get), 1); + } + + extern "C" fn patched_test(_closure: *const crate::closure::ClosureHeader, _arg: f64) -> f64 { + f64::from_bits(crate::value::TAG_TRUE) + } + + #[test] + fn patched_regexp_prototype_test_declines_on_the_next_call() { + let _lock = crate::gc::global_side_table_test_lock(); + test_reset(); + ensure_regexp_builtins(); + let site = key(&DIRECT_G); + let (warm_header, _) = run_direct(site, "g", "x"); + + let proto_value = crate::object::builtin_prototype_value("RegExp"); + let proto = crate::value::JSValue::from_bits(proto_value.to_bits()) + .as_pointer::() + as *mut crate::object::ObjectHeader; + let test_key = string("test"); + let original = crate::object::js_object_get_field_by_name(proto, test_key); + crate::object::js_object_set_field_by_name( + proto, + test_key, + closure_with_arity(patched_test as *const u8, 1), + ); + + let receiver = js_regexp_site_test_new(string("x"), string("g"), site); + assert_ne!( + receiver as usize, warm_header, + "patched prototype forces a fresh header" + ); + let receiver_value = crate::value::js_nanbox_pointer(receiver as i64); + let method = js_regexp_site_test_get_method(site, receiver_value); + let result = js_regexp_site_test_dispatch( + site, + receiver_value, + method, + crate::js_nanbox_string(string("no match") as i64), + ); + assert_eq!(result.to_bits(), crate::value::TAG_TRUE); + assert_eq!(TEST_DECLINED.with(std::cell::Cell::get), 1); + assert_eq!(TEST_DECLINED_PATCHED.with(std::cell::Cell::get), 1); + + crate::object::js_object_set_field_by_name( + proto, + test_key, + f64::from_bits(original.bits()), + ); + } + + #[test] + fn site_header_root_is_rewritten_by_a_copying_minor() { + let _guard = crate::gc::CopyingNurseryTestGuard::new(0); + test_reset(); + crate::gc::gc_register_mutable_root_scanner(scan_roots_mut); + + let site = key(&DIRECT_G) as usize; + let header = super::super::test_alloc_nursery_regexp_for_move("site-root", "g"); + let old = header as usize; + assert!(crate::arena::pointer_in_nursery(old)); + install(site, header, 0); + + let _ = crate::gc::gc_collect_minor(); + let moved = test_header(site).expect("the site header remains rooted"); + assert_ne!(moved, old, "the scanner must rewrite the cached address"); + assert!(super::super::regex_header_has_magic( + moved as *const RegExpHeader + )); + test_reset(); + } +} diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index abab4f7215..18b265330c 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -266,18 +266,6 @@ "verdict": "not_a_gc_pointer", "why": "Boolean census request latch, set by census_arm and consumed at full-sweep entry; contains no address or JS value." }, - { - "file": "crates/perry-runtime/src/gc/copying.rs", - "name": "LAST_COHORT_SPLIT", - "verdict": "test_only", - "why": "Declared under #[cfg(test)] at crates/perry-runtime/src/gc/copying.rs:1951; this Cell<(usize, usize, usize, usize)> holds only the byte counts of the last survivor-space/fresh-cohort split for the tenuring lock tests. It is absent from shipped binaries." - }, - { - "file": "crates/perry-runtime/src/gc/tenuring.rs", - "name": "SURVIVOR_ROUND_MEASURED", - "verdict": "not_a_gc_pointer", - "why": "Declared at crates/perry-runtime/src/gc/tenuring.rs:212; this Cell records whether any survivor round has been rated on this thread, gating the occupancy rule off its floor. A boolean, never a heap pointer." - }, { "file": "crates/perry-runtime/src/gc/census.rs", "name": "LABEL", @@ -303,9 +291,9 @@ "function": "run_to_completion" }, "sources": { - "crates/perry-runtime/src/gc/census.rs": "388414f9629f196e84673e91bebd04bdcdcabdaa180252d2dfe4b82d1b49ca5a", + "crates/perry-runtime/src/gc/census.rs": "1ddeeec3ca81b792a222dbe165c32b7b995c084f66c760cb6cd3f26baf2cb07a", "crates/perry-runtime/src/gc/cycle.rs": "77eadaf7c4157308c3b14be5e1ff11d7198b84b503a50ff2d244851248d3e800", - "crates/perry-runtime/src/gc/mod.rs": "cf763b4d1743cd4ab5a571aef9b1eddba973ef8a205d343dea8f34775ff2fa8a", + "crates/perry-runtime/src/gc/mod.rs": "cd5b27d4cbc5a0f100eefe8de71e1db6554c5a3e180b9996a658b581d2dd8348", "crates/perry-runtime/src/gc/policy.rs": "dc9242ed40c0aa9c411d1ec0235c0219c6716dd82d56eb4d46578f7e889825d2", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } @@ -323,6 +311,12 @@ "verdict": "test_only", "why": "Declared under cfg(test); holds an explicitly leaked Rust path string used by the isolated census unit tests." }, + { + "file": "crates/perry-runtime/src/gc/copying.rs", + "name": "LAST_COHORT_SPLIT", + "verdict": "test_only", + "why": "Declared under #[cfg(test)] at crates/perry-runtime/src/gc/copying.rs:1951; this Cell<(usize, usize, usize, usize)> holds only the byte counts of the last survivor-space/fresh-cohort split for the tenuring lock tests. It is absent from shipped binaries." + }, { "file": "crates/perry-runtime/src/gc/diag_sites.rs", "name": "BUDGETED", @@ -377,6 +371,12 @@ "verdict": "not_a_gc_pointer", "why": "#9794: monotonic per-minor sequence number used to label survival-origin records. A `Cell`." }, + { + "file": "crates/perry-runtime/src/gc/tenuring.rs", + "name": "SURVIVOR_ROUND_MEASURED", + "verdict": "not_a_gc_pointer", + "why": "Declared at crates/perry-runtime/src/gc/tenuring.rs:212; this Cell records whether any survivor round has been rated on this thread, gating the occupancy rule off its floor. A boolean, never a heap pointer." + }, { "file": "crates/perry-runtime/src/gc/trace.rs", "name": "FORWARDED_STUB_MEMBERSHIP_RECOVERIES", @@ -748,6 +748,54 @@ "verdict": "not_a_gc_pointer", "why": "#9796: the memoized never-match placeholder program installed for a pattern only `fancy-regex` accepts. `Arc` is a Rust-allocator compiled program \u2014 the collector neither traces nor moves it \u2014 and the `Arc` keeps it alive independently." }, + { + "file": "crates/perry-runtime/src/regex/site_test.rs", + "name": "DIRECT_G", + "verdict": "test_only", + "why": "A cfg(test) static whose immortal address supplies a synthetic direct global-regex site key." + }, + { + "file": "crates/perry-runtime/src/regex/site_test.rs", + "name": "MEMBER_CALL", + "verdict": "test_only", + "why": "A cfg(test) static whose immortal address supplies a synthetic namespace-member call site key." + }, + { + "file": "crates/perry-runtime/src/regex/site_test.rs", + "name": "TEST_ALLOCATIONS", + "verdict": "test_only", + "why": "A cfg(test) counter for site-entry header allocations; it contains only a u64 tally." + }, + { + "file": "crates/perry-runtime/src/regex/site_test.rs", + "name": "TEST_DECLINED", + "verdict": "test_only", + "why": "A cfg(test) counter for site-validation declines; it contains only a u64 tally." + }, + { + "file": "crates/perry-runtime/src/regex/site_test.rs", + "name": "TEST_DECLINED_CALLEE", + "verdict": "test_only", + "why": "A cfg(test) counter for callee-identity declines; it contains only a u64 tally." + }, + { + "file": "crates/perry-runtime/src/regex/site_test.rs", + "name": "TEST_DECLINED_NON_LITERAL", + "verdict": "test_only", + "why": "A cfg(test) counter for non-literal factory declines; it contains only a u64 tally." + }, + { + "file": "crates/perry-runtime/src/regex/site_test.rs", + "name": "TEST_DECLINED_PATCHED", + "verdict": "test_only", + "why": "A cfg(test) counter for patched-prototype declines; it contains only a u64 tally." + }, + { + "file": "crates/perry-runtime/src/regex/site_test.rs", + "name": "TEST_NO_ALLOC", + "verdict": "test_only", + "why": "A cfg(test) counter for allocation-free site services; it contains only a u64 tally." + }, { "file": "crates/perry-runtime/src/set.rs", "name": "SET_COMPACTION_LOG", @@ -3432,10 +3480,6 @@ "file": "crates/perry-runtime/src/native_arena.rs", "name": "VIEW_REGISTRY" }, - { - "file": "crates/perry-runtime/src/node_http2_constants.rs", - "name": "SENSITIVE_HEADERS_SYMBOL" - }, { "file": "crates/perry-runtime/src/node_repl.rs", "name": "RECOVERABLE_ERRORS" From 573b0d46938abc6b7df4d8ebdb58d9b9f1a8fbc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 12:07:47 +0200 Subject: [PATCH 10/27] docs(perf): record regex literal-site handoff Document the implementation SHA, source map, validation status, expected diagnostic movement, and the exact full-recompile measurement request. (cherry picked from commit 47370d886193b98de3b301672a140e4102b243cd) --- .../codex/REPORT_regex_literal_site_test.md | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 cc-perf-campaign/codex/REPORT_regex_literal_site_test.md diff --git a/cc-perf-campaign/codex/REPORT_regex_literal_site_test.md b/cc-perf-campaign/codex/REPORT_regex_literal_site_test.md new file mode 100644 index 0000000000..a60af30872 --- /dev/null +++ b/cc-perf-campaign/codex/REPORT_regex_literal_site_test.md @@ -0,0 +1,66 @@ +# Regex literal-site `.test` report + +## Branch and SHA + +- Branch: `perf/regex-literal-site-test` +- Base: `107d40adb9881ff5f91f94124e24907fcfea5796` +- Implementation commit: `2f799c7b0318b683bd0f320705ea6855f73fed85` +- Target remote: `fork/perf/regex-literal-site-test` + +## Map and mechanism + +- Ordinary regex literals still derive an identity from the address of a compiler-emitted private `i64` global, never from a hand-written constant (`crates/perry-codegen/src/expr/logical_collections.rs:61-79`). Ordinary lowering loads the interned pattern/flags handles and calls `js_regexp_new_site(pattern, flags, site)` (`logical_collections.rs:1382-1413`; runtime entry at `crates/perry-runtime/src/regex.rs:988-994`). +- A direct HIR `RegExpTest` whose receiver is exactly `Expr::RegExp` is the non-escaping shape: the literal node is consumed solely as this call's receiver, and only the boolean result is published. It lowers through `js_regexp_site_test_new`, captures `.test` before evaluating the argument, roots receiver and method across argument evaluation, then dispatches (`crates/perry-codegen/src/expr/instance_misc1.rs:1192-1226`). An escaping receiver such as `const r = /x/g; r.test(a); r.test(b)` remains on `js_regexp_new_site` plus the ordinary `js_regexp_test` route. +- `js_regexp_site_test_new` allocates and records one header on the cold evaluation, then reuses it on canonical hits (`crates/perry-runtime/src/regex/site_test.rs:157-200`). The site table owns a strong mutable raw root; its registered visitor marks and rewrites the header during evacuation (`site_test.rs:493-500`; registration at `crates/perry-runtime/src/gc/mod.rs:1003-1005`). The header therefore survives minors and cannot enter regex-death finalization while the site lives. +- Validation is performed on every evaluation. The realm's `RegExp.prototype`, canonical `test` closure, and own-slot index are recorded at intrinsic installation; the two heap values are representation-correct GC roots. The probe rejects a replaced/deleted/accessor `test` slot and any explicit receiver prototype (`crates/perry-runtime/src/object/regex_proto_thunks.rs:319-408`). A decline resolves the actual property and calls it generically (`site_test.rs:397-490`). Property Get remains before argument evaluation, so an argument that patches the prototype still invokes the method captured before that patch. +- Ordinary `js_regexp_test` implements stateful global/sticky `lastIndex` behavior (`crates/perry-runtime/src/regex.rs:1701 onward`). A fresh literal starts with zero on every evaluation, so the allocation-free dispatch resets the private cached header to zero immediately before its test (`site_test.rs:460-484`). The resulting write is unobservable: this exact receiver has no escaping reference, no `this` capture, and only the already-validated builtin sees it. +- The segment-view tier validates the same canonical prototype and calls `regexp_test_str_bounded` on a borrowed segment (`crates/perry-runtime/src/intl/segments_view.rs:350-394`; bounded matcher at `crates/perry-runtime/src/regex.rs:1661-1699`). It deliberately refuses global/sticky regexes and operates only after its receiver exists. Consequently it removed segment materialization but could not remove `g54.default()`'s per-grapheme RegExp construction; generic function-result dispatch is documented at `crates/perry-runtime/src/object/native_call_method/primitive_methods.rs:541-545`. +- For the real bundle shape, codegen recognizes `().test(arg)` and preserves the actual call and member lookup (`crates/perry-codegen/src/expr/calls.rs:77-101,879-950`). Functions/closures are eligible to claim an active caller site only when HIR proves zero parameters, non-async, non-generator, and exactly one `return ` statement (`crates/perry-codegen/src/codegen/function.rs:523-531`; `closure.rs:561-571`). The runtime resolves the actual callee's native entry on every call and pairs it with the site; only the exact factory identity may claim/reuse the header (`site_test.rs:202-252,321-395`). Reassignment, a rebound namespace member, a non-literal body, or a nested helper declines to the generic result path. Active identity frames have exception savepoints so caught throws cannot leave stale authorization. +- `[regex-diag]` now prints `site_test_no_alloc=` and `site_test_declined=(patched_prototype=...,callee_mismatch=...,non_literal=...)` in the whole line (`crates/perry-runtime/src/hot_diag.rs:196-203,374-424`). The no-allocation counter is incremented inside the construction entries that avoid the priced allocation. `new=` falls by the served count because a hit never enters `js_regexp_new_impl`. +- `PERRY_GC_CENSUS` now contains `regex.content_cache`, `regex.literal_sites`, and `regex.site_test_headers` (`crates/perry-runtime/src/gc/census.rs:565-604`; aggregation at `crates/perry-runtime/src/regex/site_test.rs:503-520`). + +## Named correctness and sabotage coverage + +- Codegen: `direct_literal_test_uses_the_site_header_and_post_get_dispatch`, `escaping_literal_is_not_transformed_and_keeps_one_stateful_receiver`, `direct_factory_call_records_function_identity_and_uses_the_caller_site`, and `namespace_member_factory_call_uses_the_member_wrapper` (`crates/perry-codegen/src/expr/regex_site_test_tests.rs:73-207`). +- Runtime: `direct_global_site_allocates_one_header_and_resets_last_index` compares a table with fresh generic `/x/g`; `direct_sticky_site_starts_each_evaluation_at_zero` checks `/x/y` anchoring and reset; `escaping_generic_global_header_carries_last_index_between_tests` proves the untransformed stateful case (`crates/perry-runtime/src/regex/site_test.rs:586-640`). +- Cross-function sabotage: `direct_factory_site_reuses_only_the_recorded_callee`, `nested_exact_factory_cannot_claim_a_different_callees_site`, and `namespace_member_factory_site_is_covered` cover direct `f()`, nested/non-literal decline, and namespace-member rebinding on the next call (`site_test.rs:699-813`). +- Prototype/rooting sabotage: `patched_regexp_prototype_test_declines_on_the_next_call`, `caught_throw_restores_an_orphaned_factory_site_frame`, and `site_header_root_is_rewritten_by_a_copying_minor` cover the next-call patch guard, exception cleanup, and copied-minor root rewriting (`site_test.rs:753-880`). The decline tests assert the individual reason buckets, not only the total. + +## Gates + +Completed after the final edits: + +- Direct `rustfmt` over touched Rust files. +- `git diff --check`. +- `scripts/check_file_size.sh`: passed; `regex.rs` is below the 2,000-line ceiling. +- `python3 -m json.tool scripts/gc_runtime_root_holders.json`: passed. +- `python3 scripts/gc_runtime_root_holders.py --self-test`: passed, 90 planted declarations and 347 inventory entries. +- `python3 scripts/gc_runtime_root_holders.py`: passed, 1,371 holders scanned, 594 reached by registered scanners, 358 classified, 414 frontier-pinned, 152 scanners. +- `python3 scripts/gc_rekeyed_key_tables.py`: passed, 42 rekey sites, 25 registered prunes, 0 gaps. + +Cargo history and disk stop: + +- `cargo test -j4 -p perry-codegen` through `measure_lock.sh --build` passed before the last guard-only codegen edit: 1,452 unit tests passed, 1 ignored, followed by all integration and doc tests passing. The final tree was not rerun. +- `cargo test -j4 -p perry-runtime --release --lib -- --test-threads=1` initially passed (3,265 passed, 4 ignored) before the exception/counter additions. The final-tree attempt compiled successfully and ran 3,266 passing tests plus one new synthetic nested-wrapper test failure; that fixture's trivial Rust wrapper had been release-folded with its callee. The fixture was made observably distinct with an atomic side effect and `inline(never)`, but was not rerun. +- Immediately after that invocation, `df -g /` reported 9 GB available. The binding rule prohibits every further Cargo invocation below 12 GB, so no wait or additional build was attempted. +- Not run on the final tree: the runtime lib gate rerun, the codegen gate rerun, `cargo build --release -p perry-runtime --features wasm-host`, and `cargo build --release -p perry`. +- A fresh archive and `nm` check were not produced because the archive build was prohibited. No local cc CPU/RSS measurement was run. + +## Predictions + +For the supplied I6d 3,300-character reply: + +- `new=`: 1,074,006 -> at most 10,000. +- `site_test_no_alloc=`: approximately 1,068,858 (one cold header means the exact value may be one lower for that site). +- `header_bytes`: 60,144,336 bytes -> approximately 0.3 MB. +- `ptr_ins` / `ptr_rm`: approximately zero at reply scale, apart from cold and unrelated regex objects. +- `test=`: unchanged at approximately 2,139,156. +- Turn CPU at 3,300 characters: -4% to -6%, from removing `js_regexp_new`, regex-death/finalization, and pointer-side-table work. Peak RSS should be lower; +1% to +10% remains acceptable under the campaign goal. + +## Exact perrymaster request + +This commit touches CODEGEN. Build the compiler from `2f799c7b0318b683bd0f320705ea6855f73fed85` on the I6d tree, or on the I7-view tree if all prerequisite picks apply, and perform a full cc bundle recompile; do not reuse the base bundle. From the resulting artefact, use `nm` to prove the new site-test runtime entry symbols are present and report the number of emitted call sites for `js_regexp_site_test_new`, `js_regexp_site_factory_call_value`, and `js_regexp_site_factory_call_method` (including the `g54.default().test(O)` site). + +Run one identical 3,300-character reply and provide the entire `[regex-diag]` line plus the per-pattern table. Confirm the 12,807-byte emoji `/.../g` row is constructed once, built once, and tested approximately 1,068,858 times; report `new`, `site_test_no_alloc`, the three decline buckets, `header_bytes`, `ptr_ins`, `ptr_rm`, `test`, `test_global`, and compile/cache counters. Expected: `new <= 10,000`, `site_test_no_alloc ~= 1,068,858`, `header_bytes ~= 0.3 MB`, `ptr_ins/ptr_rm ~= 0`, and unchanged `test`. + +Then run paired 5x3,300-character and 3x400-character comparisons against the base bundle with identical warmup, environment, inputs, and node-parity stop conditions. Report every turn's CPU and peak RSS. Expected 3,300-character turn CPU improvement is 4% to 6% and peak RSS is lower. Finally capture a perf draw and verify `js_regexp_new`, `regex_header_clear_dead_for_gc`, and the dead-owner regex path have disappeared from the top 25. From 72fae4adbe5d14810aecfcb32c01972fd8923547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 16:46:03 +0200 Subject: [PATCH 11/27] fix(regex): deduplicate CI custody records Keep the merge-train's existing RegExp prototype holder verdicts instead of registering the same three holders twice. Express the literal-site canonical check and its post-call handle reload with the sanctioned across_mut form. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit 54c9373c882fe7a2bb63cde806f563ce5799605d) --- crates/perry-runtime/src/regex/site_test.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/perry-runtime/src/regex/site_test.rs b/crates/perry-runtime/src/regex/site_test.rs index ebc40c80c8..6a2ff3ed68 100644 --- a/crates/perry-runtime/src/regex/site_test.rs +++ b/crates/perry-runtime/src/regex/site_test.rs @@ -161,10 +161,13 @@ fn canonical_rooted_header(header: *mut RegExpHeader) -> Option<*mut RegExpHeade let scope = crate::gc::RuntimeHandleScope::new(); let rooted = scope.root_raw_mut_ptr(header); let value = f64::from_bits(crate::value::JSValue::pointer(header.cast::()).bits()); - if !crate::object::regex_proto_thunks::regexp_prototype_test_is_canonical(value) { + let (canonical, header) = rooted.across_mut::(|| { + crate::object::regex_proto_thunks::regexp_prototype_test_is_canonical(value) + }); + if !canonical { return None; } - Some(rooted.get_raw_mut_ptr()) + Some(header) } /// Direct literal receiver for `/literal/flags.test(arg)`. From 46db1ac4403057e23cfa1d8a5e91df3a5c6bf872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 16:48:28 +0200 Subject: [PATCH 12/27] docs(perf): record regex CI cleanup Append the 2026-09-07 CI-fix handoff with exact fixed code heads, per-item static gate results, disk-skipped cargo gates, and both branch range-diffs. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit a93908a6cc684d511a0af8561b65be4f0536fbd4) --- .../codex/REPORT_regex_literal_site_test.md | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/cc-perf-campaign/codex/REPORT_regex_literal_site_test.md b/cc-perf-campaign/codex/REPORT_regex_literal_site_test.md index a60af30872..a600ec854a 100644 --- a/cc-perf-campaign/codex/REPORT_regex_literal_site_test.md +++ b/cc-perf-campaign/codex/REPORT_regex_literal_site_test.md @@ -64,3 +64,53 @@ This commit touches CODEGEN. Build the compiler from `2f799c7b0318b683bd0f320705 Run one identical 3,300-character reply and provide the entire `[regex-diag]` line plus the per-pattern table. Confirm the 12,807-byte emoji `/.../g` row is constructed once, built once, and tested approximately 1,068,858 times; report `new`, `site_test_no_alloc`, the three decline buckets, `header_bytes`, `ptr_ins`, `ptr_rm`, `test`, `test_global`, and compile/cache counters. Expected: `new <= 10,000`, `site_test_no_alloc ~= 1,068,858`, `header_bytes ~= 0.3 MB`, `ptr_ins/ptr_rm ~= 0`, and unchanged `test`. Then run paired 5x3,300-character and 3x400-character comparisons against the base bundle with identical warmup, environment, inputs, and node-parity stop conditions. Report every turn's CPU and peak RSS. Expected 3,300-character turn CPU improvement is 4% to 6% and peak RSS is lower. Finally capture a perf draw and verify `js_regexp_new`, `regex_header_clear_dead_for_gc`, and the dead-owner regex path have disappeared from the top 25. + +## CI fixes 2026-09-07 + +### Fixed heads + +- #9918 `perf/regex-drop-source-table`: `dd1c5242d2ce87139d33436f347adb7245fe754d` (old head `ce9e12801e8d83fae471e06cc85429257ac10854`). +- #9958 fixed code head, before this final report-only commit: `54c9373c882fe7a2bb63cde806f563ce5799605d` (old head `abb0d907ff8dade12dfcf6b092cbd8f71bfc4233`). The final remote branch head is this report commit, whose hash is necessarily determined after the report contents are committed. + +### Triage items + +1. Formatting: direct `rustfmt` put the test modules in formatter order at `crates/perry-runtime/src/regex.rs:1760-1767`, moving `mod tests_part2;` after `tests_cache` and `tests_header`. Direct `rustfmt --check` passes. `cargo fmt --all --check` was not run: disk (8 GB available, below the binding 12 GB floor). +2. #9918 raw-handle debt: the two new bare reads in the nursery relocation fixture are now scoped `RuntimeHandle::with_const_ptr` stores at `crates/perry-runtime/src/regex.rs:390-395`; the two pre-existing production reads and the ceiling remain unchanged. `python3 scripts/raw_handle_debt.py` passes at 955 sites (baseline 963), and `--self-test` passes. +3. #9918 product warnings: the `Arc` import is feature-gated at `crates/perry-runtime/src/regex.rs:14-15`; `MatcherKind` carries a feature-off `dead_code` allow with the layout-only reason at `regex.rs:500-514`. The workflow's exact product command (`RUSTFLAGS='-D warnings' cargo check -p perry --bins`) was not run: disk. +4. #9918 all-target warnings: the unused `regex_has_repeat_program` import is gone at `crates/perry-runtime/src/regex/tests_part2.rs:5-7`, and the unnecessary `unsafe` block around the safe lazy-build/assertion calls is gone at `tests_part2.rs:600-607`. The workflow's host-compatible `cargo check --workspace --all-targets ...` command was not run: disk. +5. Main's benchmark-freshness, build-cache, and GC-ratchet reds were not touched. +6. #9958 root-holder custody/windows self-test: removed the three duplicate `REGEXP_PROTOTYPE_*_SLOT` entries that the stack re-added; the authoritative #9893 entries remain once each at `scripts/gc_runtime_root_holders.json:647-664`. `python3 scripts/gc_runtime_root_holders.py` passes (1,372 declarations, 596 scanner-reached, 357 inventory-classified, 414 frontier-pinned, 152 scanners), and `--self-test` passes (90 planted declarations, 357 inventory entries). +7. #9958 raw-handle debt: `canonical_rooted_header` now pairs the canonicality call with its post-call reload through `RuntimeHandle::across_mut` at `crates/perry-runtime/src/regex/site_test.rs:164-170`. No per-module ceiling was added; the same debt command and self-test in item 2 pass. +8. `async_hooks_constructors_expose_real_prototype_methods` is in `crates/perry/tests/issue_6764_async_hooks_prototype_metadata.rs`. Not run: disk (8 GB available). The hypothesis that duplicate inventory registration caused the runtime failure remains unverified locally; no codegen bisection or blind patch was performed. +9. Main's unrelated shard and GC reds were not touched. + +Other requested cargo gates were not run: disk: `cargo test -p perry-runtime --release --lib -j4 -- --test-threads=1 regex`, the full runtime lib gate, and the single compiled async-hooks test. `git diff --check`, JSON parsing, both Python audits, and both audit self-tests pass. + +### Range-diffs + +#9918, `git range-diff 616a2cb84..ce9e12801 616a2cb84..dd1c5242d`: + +```text +1: d8daa4fd4 = 1: d8daa4fd4 perf(regex): remove the traced-source side table +2: e2b0a9054 = 2: e2b0a9054 perf(regex): share one program-set handle per header +3: 7a44e5948 = 3: 7a44e5948 perf(regex): tag the selected matcher on each header +4: 6ea7ad9eb = 4: 6ea7ad9eb test(regex): isolate WTF-8 source from matcher parsing +5: c217a231c = 5: c217a231c refactor(regex): split header properties and tests +6: 883d334a6 = 6: 883d334a6 fix(regex): retain canonical flags through allocation +7: ce9e12801 = 7: ce9e12801 perf(regex): preserve live literal programs on eviction +-: --------- > 8: dd1c5242d fix(regex): clear branch-owned CI failures +``` + +All seven measured commits are byte-identical; only the new CI-fix commit is added. + +#9958, `git range-diff ce9e12801..abb0d907f dd1c5242d..54c9373c8`: + +```text +1: f5a2bdb7c = 1: 90e21317a perf(regex): reuse literal headers at test-only sites +2: abb0d907f ! 2: 47370d886 docs(perf): record regex literal-site handoff + The report commit no longer carries the inherited tests_part2 warning cleanup; + that exact hunk is now in #9918's dd1c5242d fix beneath the stack. +-: --------- > 3: 54c9373c8 fix(regex): deduplicate CI custody records +``` + +The measured #9958 implementation commit is patch-identical. The only movement in the report commit is the listed inherited warning cleanup moving to the fixed base; the only new code hunk is the item 6/7 CI-fix commit. From 8d975e80d92648aca1632c6850682152198d3945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 19:56:58 +0200 Subject: [PATCH 13/27] feat(diagnostics): account regex tables in heap census Add requested-only rows for the common regex-owned tables and reconcile their de-duplicated byte estimates with the heap census side-table total. Cover the emitted inventory and the no-construction-cost contract directly. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit 49f551f057c0d52d9e7fc89d4eb16a1c485ec2a5) --- crates/perry-runtime/src/gc/census.rs | 21 +- crates/perry-runtime/src/gc/mod.rs | 1 + crates/perry-runtime/src/gc/regex_census.rs | 153 +++++++++ .../src/object/exotic_expando.rs | 27 ++ crates/perry-runtime/src/regex.rs | 4 + crates/perry-runtime/src/regex/census_rows.rs | 293 ++++++++++++++++++ .../perry-runtime/src/regex/repeat_matcher.rs | 12 + 7 files changed, 502 insertions(+), 9 deletions(-) create mode 100644 crates/perry-runtime/src/gc/regex_census.rs create mode 100644 crates/perry-runtime/src/regex/census_rows.rs diff --git a/crates/perry-runtime/src/gc/census.rs b/crates/perry-runtime/src/gc/census.rs index 25246b59fe..8b7d53b679 100644 --- a/crates/perry-runtime/src/gc/census.rs +++ b/crates/perry-runtime/src/gc/census.rs @@ -557,7 +557,7 @@ pub(crate) fn vec_bytes(v: &Vec) -> usize { v.capacity() * std::mem::size_of::() } -fn side_tables() -> Vec { +pub(super) fn side_tables() -> Vec { let mut rows: Vec = Vec::new(); rows.extend(crate::builtins::function_registries_census()); rows.extend(crate::closure::closure_registry_census()); @@ -811,14 +811,15 @@ fn take_census(label: &str, pass1: Option>) { side_rows.extend(crate::object::shapes::shape_table_liveness_census( &c.live_shape_ids, )); - let side: Vec = side_rows - .into_iter() - .map(|(n, e, b)| serde_json::json!({"table": n, "entries": e, "bytes": b})) - .collect(); - let side_total: usize = side - .iter() - .map(|r| r["bytes"].as_u64().unwrap_or(0) as usize) - .sum(); + let side_snapshot = super::regex_census::side_table_document_from(side_rows); + let side = side_snapshot["rows"].clone(); + let side_total = side_snapshot["side_table_bytes"].as_u64().unwrap_or(0) as usize; + let regex_side_total = side_snapshot["regex_side_table_bytes"] + .as_u64() + .unwrap_or(0) as usize; + let non_regex_side_total = side_snapshot["non_regex_side_table_bytes"] + .as_u64() + .unwrap_or(0) as usize; let live_total: u64 = c.space_live.iter().map(|a| a.bytes).sum(); let dead_total: u64 = c.space_dead.iter().map(|a| a.bytes).sum(); @@ -857,6 +858,8 @@ fn take_census(label: &str, pass1: Option>) { "live_bytes": live_total, "dead_bytes": dead_total, "side_table_bytes": side_total, + "regex_side_table_bytes": regex_side_total, + "non_regex_side_table_bytes": non_regex_side_total, "live_objects": c.space_live.iter().map(|a| a.count).sum::(), "dead_objects": c.space_dead.iter().map(|a| a.count).sum::(), "late_marked_bytes": late_total, diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 3cb72ce193..8ef7ea0294 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -258,6 +258,7 @@ pub(crate) mod census; #[cfg(feature = "diagnostics")] mod heap_snapshot; mod heap_stats; +mod regex_census; pub use census::{census_poll_signal, gc_census_enabled}; #[cfg(feature = "diagnostics")] pub use heap_snapshot::gc_build_v8_heap_snapshot_json; diff --git a/crates/perry-runtime/src/gc/regex_census.rs b/crates/perry-runtime/src/gc/regex_census.rs new file mode 100644 index 0000000000..077f8b61ec --- /dev/null +++ b/crates/perry-runtime/src/gc/regex_census.rs @@ -0,0 +1,153 @@ +//! RegExp side-table serialization for the requested heap census. + +use super::census::SideTableRow; + +/// Serialize ordinary and RegExp-owned registries through one path. The regex +/// attribution is computed independently of the emitted rows so omission is a +/// visible reconciliation failure rather than a silently smaller total. +pub(super) fn side_table_document_from(mut ordinary: Vec) -> serde_json::Value { + // Replace the legacy RegExp tuples with the rich, reconciled rows. + ordinary.retain(|(table, _, _)| !table.starts_with("regex.")); + let non_regex_total = ordinary.iter().map(|(_, _, bytes)| *bytes).sum::(); + let mut rows = ordinary + .drain(..) + .map(|(table, entries, bytes)| { + serde_json::json!({"table": table, "entries": entries, "bytes": bytes}) + }) + .collect::>(); + + #[cfg(feature = "regex-engine")] + let regex_total = { + let snapshot = crate::regex::census_snapshot(); + rows.extend(snapshot.rows.iter().map(crate::regex::RegexCensusRow::json)); + snapshot.attributed_bytes + }; + #[cfg(not(feature = "regex-engine"))] + let regex_total = 0usize; + + serde_json::json!({ + "rows": rows, + "side_table_bytes": non_regex_total + regex_total, + "regex_side_table_bytes": regex_total, + "non_regex_side_table_bytes": non_regex_total, + }) +} + +#[cfg(test)] +fn test_side_table_document() -> serde_json::Value { + let snapshot = side_table_document_from(super::census::side_tables()); + serde_json::json!({ + "totals": { + "side_table_bytes": snapshot["side_table_bytes"], + "regex_side_table_bytes": snapshot["regex_side_table_bytes"], + "non_regex_side_table_bytes": snapshot["non_regex_side_table_bytes"], + }, + "side_tables": snapshot["rows"], + }) +} + +#[cfg(all(test, feature = "regex-engine"))] +mod tests { + use crate::regex::{js_regexp_new, js_regexp_test}; + + fn string(value: &str) -> *mut crate::StringHeader { + crate::string::js_string_from_bytes(value.as_ptr(), value.len() as u32) + } + + fn regex_rows(doc: &serde_json::Value) -> Vec<&serde_json::Value> { + doc["side_tables"] + .as_array() + .expect("side-table array") + .iter() + .filter(|row| { + row["table"] + .as_str() + .is_some_and(|name| name.starts_with("regex.")) + }) + .collect() + } + + #[test] + fn census_prints_regex_rows_that_reconcile_with_side_table_total() { + let _lock = crate::gc::global_side_table_test_lock(); + crate::regex::census_rows::test_reset_tables(); + + const N: usize = 6; + for index in 0..N { + let source = format!("regex-census-{index}"); + let pattern = string(&source); + let header = js_regexp_new(pattern, string("")); + assert_ne!(js_regexp_test(header, string(&source)), 0); + } + + assert_eq!( + crate::regex::census_rows::test_walks(), + 0, + "regex construction and matching must not do census bookkeeping" + ); + let encoded = super::test_side_table_document().to_string(); + let doc: serde_json::Value = serde_json::from_str(&encoded).expect("valid census JSON"); + let rows = regex_rows(&doc); + + let names = rows + .iter() + .map(|row| row["table"].as_str().unwrap()) + .collect::>(); + for expected in [ + "regex.pointers", + "regex.program_cache", + "regex.fancy_cache", + "regex.repeat_cache", + "regex.validated_patterns", + "regex.expando_owners", + "regex.matcher_kinds", + ] { + assert!(names.contains(expected), "missing census row {expected}"); + } + + let pointer = rows + .iter() + .find(|row| row["table"] == "regex.pointers") + .expect("regex.pointers row"); + assert!(pointer["entries"].as_u64().unwrap() >= N as u64); + let row_bytes = rows + .iter() + .map(|row| row["bytes"].as_u64().expect("numeric row bytes")) + .sum::(); + let regex_total = doc["totals"]["regex_side_table_bytes"] + .as_u64() + .expect("regex attribution total"); + let side_total = doc["totals"]["side_table_bytes"].as_u64().unwrap(); + let non_regex_total = doc["totals"]["non_regex_side_table_bytes"] + .as_u64() + .unwrap(); + assert_eq!(row_bytes, regex_total); + assert_eq!(side_total - non_regex_total, regex_total); + + // Sabotage proof: the attribution total is built independently of + // JSON row registration. Omitting any non-zero row from `side_tables` + // makes this exact reconciliation fail. + let omitted = rows + .iter() + .find(|row| row["bytes"].as_u64().unwrap_or(0) != 0) + .unwrap()["bytes"] + .as_u64() + .unwrap(); + assert_ne!(row_bytes - omitted, regex_total); + } + + #[test] + fn census_regex_rows_are_zero_cost_when_not_requested() { + let _lock = crate::gc::global_side_table_test_lock(); + crate::regex::census_rows::test_reset_tables(); + let header = js_regexp_new(string("zero-cost-census"), string("")); + assert_ne!(js_regexp_test(header, string("zero-cost-census")), 0); + assert_eq!( + crate::regex::census_rows::test_walks(), + 0, + "construction must not enter regex census row code" + ); + let _ = super::test_side_table_document(); + assert!(crate::regex::census_rows::test_walks() > 0); + } +} diff --git a/crates/perry-runtime/src/object/exotic_expando.rs b/crates/perry-runtime/src/object/exotic_expando.rs index ff1f71f90e..1037086688 100644 --- a/crates/perry-runtime/src/object/exotic_expando.rs +++ b/crates/perry-runtime/src/object/exotic_expando.rs @@ -639,6 +639,33 @@ pub fn scan_exotic_expando_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor } } +/// `PERRY_GC_CENSUS`: attribute the RegExp-owned share of the mixed exotic +/// expando table. Hash-table storage is divided evenly by owner; each RegExp +/// owner's vector and key buffers are then charged exactly to its row. +#[cfg(feature = "regex-engine")] +pub(crate) fn regex_expando_census() -> (usize, usize, usize) { + let map = crate::state::state().exotic_expando.entries.borrow(); + let regex = map + .iter() + .filter(|(owner, _)| exotic_expando_kind(**owner) == Some(ExoticKind::RegExp)) + .collect::>(); + let owners = regex.len(); + let properties = regex.iter().map(|(_, entries)| entries.len()).sum(); + let shared = if map.is_empty() { + 0 + } else { + crate::gc::census::map_bytes(&*map) * owners / map.len() + }; + let inner = regex + .iter() + .map(|(_, entries)| { + crate::gc::census::vec_bytes(entries) + + entries.iter().map(|(key, _)| key.capacity()).sum::() + }) + .sum::(); + (owners, properties, shared + inner) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index d4bcb265ee..14a77af6a7 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -31,6 +31,8 @@ type CompiledPrograms = site_cache::Programs; #[cfg(not(feature = "regex-engine"))] type CompiledPrograms = (); +#[cfg(feature = "regex-engine")] +pub(crate) mod census_rows; #[cfg(feature = "regex-engine")] mod class_range_validate; #[cfg(feature = "regex-engine")] @@ -45,6 +47,8 @@ mod program_key; #[cfg(feature = "regex-engine")] mod replace_expand_fancy; #[cfg(feature = "regex-engine")] +pub(crate) use census_rows::{census_snapshot, RegexCensusRow}; +#[cfg(feature = "regex-engine")] pub(crate) use program_key::{ProgramKey, NEVER_MATCH_PATTERN}; #[cfg(feature = "regex-engine")] pub use replace_expand_fancy::{ diff --git a/crates/perry-runtime/src/regex/census_rows.rs b/crates/perry-runtime/src/regex/census_rows.rs new file mode 100644 index 0000000000..2b9d1dbc95 --- /dev/null +++ b/crates/perry-runtime/src/regex/census_rows.rs @@ -0,0 +1,293 @@ +//! Diagnostic-only `PERRY_GC_CENSUS` rows for RegExp-owned Rust tables. +//! +//! Nothing in this module is called from construction, matching, collection, +//! or cache maintenance. The census enters it only after a request has armed a +//! synchronous full collection. Engine crates do not expose the size of the +//! heap graph behind their public `Regex` values, so program bytes are an +//! explicitly labelled opaque lower-bound: the `Arc` allocation, public value, +//! and source/capture buffers that can be observed without unsafe layout +//! assumptions. + +use std::sync::Arc; + +use super::*; + +pub(crate) struct RegexCensusRow { + pub(crate) table: &'static str, + pub(crate) entries: usize, + pub(crate) bytes: usize, + fields: serde_json::Map, +} + +impl RegexCensusRow { + fn new(table: &'static str, entries: usize, bytes: usize) -> Self { + Self { + table, + entries, + bytes, + fields: serde_json::Map::new(), + } + } + + fn usize(mut self, name: &'static str, value: usize) -> Self { + self.fields.insert(name.into(), serde_json::json!(value)); + self + } + + fn u64(mut self, name: &'static str, value: u64) -> Self { + self.fields.insert(name.into(), serde_json::json!(value)); + self + } + + fn bool(mut self, name: &'static str, value: bool) -> Self { + self.fields.insert(name.into(), serde_json::json!(value)); + self + } + + fn text(mut self, name: &'static str, value: &'static str) -> Self { + self.fields.insert(name.into(), serde_json::json!(value)); + self + } + + pub(crate) fn json(&self) -> serde_json::Value { + let mut value = serde_json::Map::new(); + value.insert("table".into(), serde_json::json!(self.table)); + value.insert("entries".into(), serde_json::json!(self.entries)); + value.insert("bytes".into(), serde_json::json!(self.bytes)); + value.extend(self.fields.clone()); + serde_json::Value::Object(value) + } +} + +pub(crate) struct RegexCensusSnapshot { + pub(crate) rows: Vec, + /// Built independently of JSON serialization. If a row is accidentally + /// omitted from the emitted array, the reconciliation test sees the gap. + pub(crate) attributed_bytes: usize, +} + +#[cfg(test)] +static TEST_CENSUS_WALKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +#[cfg(test)] +pub(crate) fn test_reset_walks() { + TEST_CENSUS_WALKS.store(0, std::sync::atomic::Ordering::Relaxed); +} + +#[cfg(test)] +pub(crate) fn test_walks() -> usize { + TEST_CENSUS_WALKS.load(std::sync::atomic::Ordering::Relaxed) +} + +#[inline] +fn arc_allocation_bytes() -> usize { + // Two strong/weak counters precede the Arc payload in today's allocator + // representation. This is an estimate, not a promise about Arc layout. + 2 * std::mem::size_of::() + std::mem::size_of::() +} + +fn standard_program_bytes(program: ®ex::Regex) -> usize { + arc_allocation_bytes::() + program.as_str().len() +} + +fn fancy_program_bytes(program: &fancy_regex::Regex) -> usize { + arc_allocation_bytes::() + program.as_str().len() +} + +fn repeat_program_bytes(program: &repeat_matcher::RepeatMatcherRegex) -> usize { + arc_allocation_bytes::() + program.census_buffer_bytes() +} + +fn pointer_row() -> RegexCensusRow { + REGEX_POINTERS.with(|table| { + let table = table.borrow(); + let live_headers = table + .iter() + .filter(|&&addr| unsafe { + crate::value::addr_class::try_read_gc_header(addr).is_some_and(|header| { + header.obj_type == crate::gc::GC_TYPE_REGEXP + && header.gc_flags & (crate::gc::GC_FLAG_MARKED | crate::gc::GC_FLAG_PINNED) + != 0 + }) + }) + .count(); + RegexCensusRow::new( + "regex.pointers", + table.len(), + crate::gc::census::set_bytes(&*table), + ) + .usize("live_headers", live_headers) + }) +} + +fn standard_cache_row() -> RegexCensusRow { + REGEX_CACHE.with(|cache| { + let cache = cache.borrow(); + let programs = cache + .values() + .map(|program| (Arc::as_ptr(program) as usize, program)) + .collect::>(); + let opaque = programs + .values() + .map(|program| standard_program_bytes(program)) + .sum::(); + let mut cleared = 0; + let mut evictions = 0; + if crate::hot_diag::regex_on() { + crate::hot_diag::regex_counters(|diag| { + cleared = diag.cache_clears; + evictions = diag.cache_evictions; + }); + } + RegexCensusRow::new( + "regex.program_cache", + cache.len(), + crate::gc::census::map_bytes(&*cache) + opaque, + ) + .usize("compiled_programs", programs.len()) + .usize("opaque_program_bytes", opaque) + .text("program_bytes_estimate", "opaque_inline_lower_bound") + .bool("program_bytes_inside_side_table_bytes", true) + .u64("cleared", cleared) + .u64("evictions", evictions) + .text("cache_event_scope", "all_regex_caches") + }) +} + +fn fancy_cache_row() -> RegexCensusRow { + FANCY_CACHE.with(|cache| { + let cache = cache.borrow(); + let programs = cache + .values() + .map(|program| (Arc::as_ptr(program) as usize, program)) + .collect::>(); + let opaque = programs + .values() + .map(|program| fancy_program_bytes(program)) + .sum::(); + RegexCensusRow::new( + "regex.fancy_cache", + cache.len(), + crate::gc::census::map_bytes(&*cache) + opaque, + ) + .usize("compiled_programs", programs.len()) + .usize("opaque_program_bytes", opaque) + .text("program_bytes_estimate", "opaque_inline_lower_bound") + .bool("program_bytes_inside_side_table_bytes", true) + }) +} + +fn repeat_cache_row() -> RegexCensusRow { + REPEAT_MATCHER_CACHE.with(|cache| { + let cache = cache.borrow(); + let programs = cache + .values() + .map(|program| (Arc::as_ptr(program) as usize, program)) + .collect::>(); + let opaque = programs + .values() + .map(|program| repeat_program_bytes(program)) + .sum::(); + RegexCensusRow::new( + "regex.repeat_cache", + cache.len(), + crate::gc::census::map_bytes(&*cache) + opaque, + ) + .usize("compiled_programs", programs.len()) + .usize("opaque_program_bytes", opaque) + .text("program_bytes_estimate", "opaque_inline_lower_bound") + .bool("program_bytes_inside_side_table_bytes", true) + }) +} + +fn validation_cache_row() -> RegexCensusRow { + VALIDATED_PATTERNS.with(|cache| { + let cache = cache.borrow(); + let text_bytes = cache + .keys() + .map(|(pattern, flags)| pattern.capacity() + flags.capacity()) + .sum::(); + RegexCensusRow::new( + "regex.validated_patterns", + cache.len(), + crate::gc::census::map_bytes(&*cache) + text_bytes, + ) + .usize("text_bytes", text_bytes) + }) +} + +fn matcher_kind_row() -> RegexCensusRow { + let mut counts = [0usize; 4]; + REGEX_POINTERS.with(|table| { + for &addr in table.borrow().iter() { + let re = addr as *const RegExpHeader; + if !is_valid_regex_ptr(re) { + continue; + } + let index = unsafe { + match (*re).matcher_kind { + MatcherKind::Unbuilt => 0, + MatcherKind::Standard => 1, + MatcherKind::Fancy => 2, + MatcherKind::Repeat => 3, + } + }; + counts[index] += 1; + } + }); + RegexCensusRow::new("regex.matcher_kinds", counts.iter().sum(), 0) + .usize("unbuilt", counts[0]) + .usize("standard", counts[1]) + .usize("fancy", counts[2]) + .usize("repeat", counts[3]) + .bool("bytes_inside_side_table_bytes", false) + .text("storage", "RegExpHeader.matcher_kind") +} + +fn expando_row() -> RegexCensusRow { + let (owners, properties, bytes) = crate::object::exotic_expando::regex_expando_census(); + RegexCensusRow::new("regex.expando_owners", owners, bytes) + .usize("owners", owners) + .usize("properties", properties) +} + +/// Snapshot every RegExp-owned table only when the census requests it. +pub(crate) fn census_snapshot() -> RegexCensusSnapshot { + #[cfg(test)] + TEST_CENSUS_WALKS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + let rows = vec![ + pointer_row(), + standard_cache_row(), + fancy_cache_row(), + repeat_cache_row(), + validation_cache_row(), + expando_row(), + matcher_kind_row(), + ]; + + // Deliberately independent from JSON row registration below: this second + // diagnostic walk is what makes an omitted row fail reconciliation. + let attributed_bytes = pointer_row().bytes + + standard_cache_row().bytes + + fancy_cache_row().bytes + + repeat_cache_row().bytes + + validation_cache_row().bytes + + expando_row().bytes + + matcher_kind_row().bytes; + + RegexCensusSnapshot { + rows, + attributed_bytes, + } +} + +#[cfg(test)] +pub(crate) fn test_reset_tables() { + REGEX_POINTERS.with(|table| table.borrow_mut().clear()); + REGEX_CACHE.with(|cache| cache.borrow_mut().clear()); + FANCY_CACHE.with(|cache| cache.borrow_mut().clear()); + REPEAT_MATCHER_CACHE.with(|cache| cache.borrow_mut().clear()); + VALIDATED_PATTERNS.with(|cache| cache.borrow_mut().clear()); + test_reset_walks(); +} diff --git a/crates/perry-runtime/src/regex/repeat_matcher.rs b/crates/perry-runtime/src/regex/repeat_matcher.rs index da008816de..64884c7d2f 100644 --- a/crates/perry-runtime/src/regex/repeat_matcher.rs +++ b/crates/perry-runtime/src/regex/repeat_matcher.rs @@ -15,6 +15,18 @@ pub(super) struct RepeatMatcherRegex { } impl RepeatMatcherRegex { + /// Census-only visible buffers. `regress::Regex` does not expose its + /// compiled heap graph, so callers label this as an opaque lower bound. + pub(super) fn census_buffer_bytes(&self) -> usize { + self.capture_names.capacity() * std::mem::size_of::>() + + self + .capture_names + .iter() + .flatten() + .map(String::capacity) + .sum::() + } + fn named_group_range( &self, matched: ®ress::Match, From 6f78b4e18207001dd2921a30be98c49adfba7152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 20:00:02 +0200 Subject: [PATCH 14/27] feat(diagnostics): add literal-site regex census rows Measure the #9958 site-rooted headers and their gross pinned program lower bound separately from de-duplicated side-table attribution. Include the content and literal-key tables that provide the other owners. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit 5e807913891d3cb8e18c3d874f7515d1275b22f0) --- crates/perry-runtime/src/gc/regex_census.rs | 31 ++++ crates/perry-runtime/src/regex/census_rows.rs | 151 ++++++++++++++++++ crates/perry-runtime/src/regex/site_cache.rs | 34 +++- crates/perry-runtime/src/regex/site_key.rs | 6 +- crates/perry-runtime/src/regex/site_test.rs | 39 +++++ scripts/gc_runtime_root_holders.json | 6 +- 6 files changed, 256 insertions(+), 11 deletions(-) diff --git a/crates/perry-runtime/src/gc/regex_census.rs b/crates/perry-runtime/src/gc/regex_census.rs index 077f8b61ec..ac6647b72c 100644 --- a/crates/perry-runtime/src/gc/regex_census.rs +++ b/crates/perry-runtime/src/gc/regex_census.rs @@ -48,6 +48,7 @@ fn test_side_table_document() -> serde_json::Value { #[cfg(all(test, feature = "regex-engine"))] mod tests { + use crate::regex::site_test::js_regexp_site_test_new; use crate::regex::{js_regexp_new, js_regexp_test}; fn string(value: &str) -> *mut crate::StringHeader { @@ -80,6 +81,20 @@ mod tests { assert_ne!(js_regexp_test(header, string(&source)), 0); } + // Evaluate one direct literal site twice: the second call must reuse + // the first rooted header and its installed program bundle. + let prototype = crate::object::builtin_prototype_value("RegExp"); + assert!(crate::value::JSValue::from_bits(prototype.to_bits()).is_pointer()); + static SITE: u64 = 0; + let site = std::ptr::addr_of!(SITE) as i64; + let first = js_regexp_site_test_new(string("site-census"), string("g"), site); + assert_ne!(js_regexp_test(first, string("site-census")), 0); + let second = js_regexp_site_test_new(string("site-census"), string("g"), site); + assert_eq!( + first, second, + "the literal site must reuse its rooted header" + ); + assert_eq!( crate::regex::census_rows::test_walks(), 0, @@ -99,6 +114,10 @@ mod tests { "regex.fancy_cache", "regex.repeat_cache", "regex.validated_patterns", + "regex.content_cache", + "regex.literal_sites", + "regex.site_table", + "regex.active_factory_sites", "regex.expando_owners", "regex.matcher_kinds", ] { @@ -110,6 +129,18 @@ mod tests { .find(|row| row["table"] == "regex.pointers") .expect("regex.pointers row"); assert!(pointer["entries"].as_u64().unwrap() >= N as u64); + let site = rows + .iter() + .find(|row| row["table"] == "regex.site_table") + .expect("regex.site_table row"); + assert!(site["sites"].as_u64().unwrap() >= 1); + assert!(site["rooted_headers"].as_u64().unwrap() >= 1); + assert!(site["pinned_programs"].as_u64().unwrap() >= 1); + assert!( + site["pinned_program_bytes"].as_u64().unwrap() + >= site["attributed_program_bytes"].as_u64().unwrap() + ); + assert_eq!(site["pinned_program_bytes_inside_side_table_bytes"], false); let row_bytes = rows .iter() .map(|row| row["bytes"].as_u64().expect("numeric row bytes")) diff --git a/crates/perry-runtime/src/regex/census_rows.rs b/crates/perry-runtime/src/regex/census_rows.rs index 2b9d1dbc95..7418484397 100644 --- a/crates/perry-runtime/src/regex/census_rows.rs +++ b/crates/perry-runtime/src/regex/census_rows.rs @@ -8,6 +8,7 @@ //! and source/capture buffers that can be observed without unsafe layout //! assumptions. +use std::collections::HashSet; use std::sync::Arc; use super::*; @@ -98,6 +99,36 @@ fn repeat_program_bytes(program: &repeat_matcher::RepeatMatcherRegex) -> usize { arc_allocation_bytes::() + program.census_buffer_bytes() } +fn program_bundle_bytes( + bundle_ptrs: impl IntoIterator, + standard_skip: &HashSet, + fancy_skip: &HashSet, + repeat_skip: &HashSet, +) -> usize { + let mut standard_seen = standard_skip.clone(); + let mut fancy_seen = fancy_skip.clone(); + let mut repeat_seen = repeat_skip.clone(); + let mut bytes = 0usize; + for ptr in bundle_ptrs { + let programs = unsafe { &*(ptr as *const site_cache::Programs) }; + bytes += arc_allocation_bytes::(); + if standard_seen.insert(Arc::as_ptr(&programs.std) as usize) { + bytes += standard_program_bytes(&programs.std); + } + if let Some(program) = &programs.fancy { + if fancy_seen.insert(Arc::as_ptr(program) as usize) { + bytes += fancy_program_bytes(program); + } + } + if let Some(program) = &programs.repeat { + if repeat_seen.insert(Arc::as_ptr(program) as usize) { + bytes += repeat_program_bytes(program); + } + } + } + bytes +} + fn pointer_row() -> RegexCensusRow { REGEX_POINTERS.with(|table| { let table = table.borrow(); @@ -244,6 +275,81 @@ fn matcher_kind_row() -> RegexCensusRow { .text("storage", "RegExpHeader.matcher_kind") } +fn content_row( + standard_cached: &HashSet, + fancy_cached: &HashSet, + repeat_cached: &HashSet, +) -> RegexCensusRow { + let (entries, table_bytes, bundle_ptrs) = site_cache::census_parts(); + let opaque = program_bundle_bytes( + bundle_ptrs.iter().copied(), + standard_cached, + fancy_cached, + repeat_cached, + ); + RegexCensusRow::new("regex.content_cache", entries, table_bytes + opaque) + .usize("pinned_programs", bundle_ptrs.len()) + .usize("opaque_program_bytes", opaque) + .text("program_bytes_estimate", "opaque_inline_lower_bound") + .bool("program_bytes_inside_side_table_bytes", true) +} + +fn site_table_row( + content_bundles: &HashSet, + standard_cached: &HashSet, + fancy_cached: &HashSet, + repeat_cached: &HashSet, +) -> RegexCensusRow { + let (sites, table_bytes, header_ptrs, bundle_ptrs) = site_test::census_parts(); + let rooted_headers = header_ptrs.len(); + let exclusive = bundle_ptrs + .iter() + .copied() + .filter(|ptr| !content_bundles.contains(ptr)) + .collect::>(); + let attributed_program_bytes = program_bundle_bytes( + exclusive.iter().copied(), + standard_cached, + fancy_cached, + repeat_cached, + ); + let pinned_program_bytes = program_bundle_bytes( + bundle_ptrs.iter().copied(), + &HashSet::new(), + &HashSet::new(), + &HashSet::new(), + ); + RegexCensusRow::new( + "regex.site_table", + sites, + table_bytes + attributed_program_bytes, + ) + .usize("sites", sites) + .usize("rooted_headers", rooted_headers) + .usize( + "rooted_header_bytes", + rooted_headers * std::mem::size_of::(), + ) + .bool("rooted_header_bytes_inside_side_table_bytes", false) + .usize("pinned_programs", bundle_ptrs.len()) + .usize("exclusively_attributed_programs", exclusive.len()) + .usize("pinned_program_bytes", pinned_program_bytes) + .usize("attributed_program_bytes", attributed_program_bytes) + .text("program_bytes_estimate", "opaque_inline_lower_bound") + .bool("pinned_program_bytes_inside_side_table_bytes", false) + .bool("attributed_program_bytes_inside_side_table_bytes", true) +} + +fn literal_site_row() -> RegexCensusRow { + let (sites, bytes) = site_key::census_parts(); + RegexCensusRow::new("regex.literal_sites", sites, bytes).usize("sites", sites) +} + +fn active_factory_row() -> RegexCensusRow { + let (entries, bytes) = site_test::active_factory_census_parts(); + RegexCensusRow::new("regex.active_factory_sites", entries, bytes) +} + fn expando_row() -> RegexCensusRow { let (owners, properties, bytes) = crate::object::exotic_expando::regex_expando_census(); RegexCensusRow::new("regex.expando_owners", owners, bytes) @@ -256,12 +362,44 @@ pub(crate) fn census_snapshot() -> RegexCensusSnapshot { #[cfg(test)] TEST_CENSUS_WALKS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let standard_cached = REGEX_CACHE.with(|cache| { + cache + .borrow() + .values() + .map(|program| Arc::as_ptr(program) as usize) + .collect::>() + }); + let fancy_cached = FANCY_CACHE.with(|cache| { + cache + .borrow() + .values() + .map(|program| Arc::as_ptr(program) as usize) + .collect::>() + }); + let repeat_cached = REPEAT_MATCHER_CACHE.with(|cache| { + cache + .borrow() + .values() + .map(|program| Arc::as_ptr(program) as usize) + .collect::>() + }); + let content_bundles = site_cache::census_program_ptrs(); + let rows = vec![ pointer_row(), standard_cache_row(), fancy_cache_row(), repeat_cache_row(), validation_cache_row(), + content_row(&standard_cached, &fancy_cached, &repeat_cached), + literal_site_row(), + site_table_row( + &content_bundles, + &standard_cached, + &fancy_cached, + &repeat_cached, + ), + active_factory_row(), expando_row(), matcher_kind_row(), ]; @@ -273,6 +411,16 @@ pub(crate) fn census_snapshot() -> RegexCensusSnapshot { + fancy_cache_row().bytes + repeat_cache_row().bytes + validation_cache_row().bytes + + content_row(&standard_cached, &fancy_cached, &repeat_cached).bytes + + literal_site_row().bytes + + site_table_row( + &content_bundles, + &standard_cached, + &fancy_cached, + &repeat_cached, + ) + .bytes + + active_factory_row().bytes + expando_row().bytes + matcher_kind_row().bytes; @@ -289,5 +437,8 @@ pub(crate) fn test_reset_tables() { FANCY_CACHE.with(|cache| cache.borrow_mut().clear()); REPEAT_MATCHER_CACHE.with(|cache| cache.borrow_mut().clear()); VALIDATED_PATTERNS.with(|cache| cache.borrow_mut().clear()); + site_cache::test_reset(); + site_key::test_reset(); + site_test::test_reset(); test_reset_walks(); } diff --git a/crates/perry-runtime/src/regex/site_cache.rs b/crates/perry-runtime/src/regex/site_cache.rs index b8081ee907..675896af48 100644 --- a/crates/perry-runtime/src/regex/site_cache.rs +++ b/crates/perry-runtime/src/regex/site_cache.rs @@ -122,21 +122,41 @@ fn entry_count(cache: &ContentMap) -> usize { } pub(super) fn census() -> crate::gc::census::SideTableRow { + let (entries, bytes, _) = census_parts(); + ("regex.content_cache", entries, bytes) +} + +pub(super) fn census_parts() -> (usize, usize, Vec) { SITE_CACHE.with(|cache| { let cache = cache.borrow(); let entries = entry_count(&cache); - // The content payload dominates; include its owned pattern/flags bytes - // as well as one entry record. Bucket/control-byte overhead is small - // and deliberately left as an estimate, matching the census contract. - let bytes = cache + let inner = cache + .values() + .map(crate::gc::census::vec_bytes) + .sum::(); + let text = cache + .values() + .flatten() + .map(|entry| entry.pattern.len() + entry.flags.len()) + .sum::(); + let programs = cache .values() .flatten() - .map(|entry| std::mem::size_of::() + entry.pattern.len() + entry.flags.len()) - .sum(); - ("regex.content_cache", entries, bytes) + .filter_map(|entry| entry.programs.as_ref()) + .map(|programs| Arc::as_ptr(programs) as usize) + .collect(); + ( + entries, + crate::gc::census::map_bytes(&*cache) + inner + text, + programs, + ) }) } +pub(super) fn census_program_ptrs() -> std::collections::HashSet { + census_parts().2.into_iter().collect() +} + /// Remove one entry that has no recorded literal site. The scan happens only /// on a distinct-content miss at capacity; literal-site hits never reach it. fn evict_one_dynamic(cache: &mut ContentMap) -> bool { diff --git a/crates/perry-runtime/src/regex/site_key.rs b/crates/perry-runtime/src/regex/site_key.rs index 6669775cc5..70b313da63 100644 --- a/crates/perry-runtime/src/regex/site_key.rs +++ b/crates/perry-runtime/src/regex/site_key.rs @@ -148,11 +148,15 @@ fn slot_of(key: usize) -> usize { } pub(super) fn census() -> crate::gc::census::SideTableRow { + let (entries, bytes) = census_parts(); + ("regex.literal_sites", entries, bytes) +} + +pub(super) fn census_parts() -> (usize, usize) { SITE_KEY_TABLE.with(|table| { let table = table.borrow(); let entries = table.iter().filter(|entry| entry.is_some()).count(); ( - "regex.literal_sites", entries, table.capacity() * std::mem::size_of::>(), ) diff --git a/crates/perry-runtime/src/regex/site_test.rs b/crates/perry-runtime/src/regex/site_test.rs index 6a2ff3ed68..bbf18e1b35 100644 --- a/crates/perry-runtime/src/regex/site_test.rs +++ b/crates/perry-runtime/src/regex/site_test.rs @@ -514,6 +514,45 @@ pub(super) fn census() -> crate::gc::census::SideTableRow { }) } +pub(super) fn census_parts() -> (usize, usize, Vec, Vec) { + SITE_TEST_HEADERS.with(|table| { + let table = table.borrow(); + let headers = table + .values() + .map(|entry| entry.header as usize) + .collect::>(); + let programs = table + .values() + .filter_map(|entry| { + let header = entry.header; + if header.is_null() || !is_valid_regex_ptr(header) { + return None; + } + let programs = unsafe { (*header).programs_ptr }; + (!programs.is_null()).then_some(programs as usize) + }) + .collect::>() + .into_iter() + .collect(); + ( + table.len(), + crate::gc::census::map_bytes(&*table), + headers, + programs, + ) + }) +} + +pub(super) fn active_factory_census_parts() -> (usize, usize) { + ACTIVE_FACTORY_SITES.with(|stack| { + let stack = stack.borrow(); + ( + stack.len(), + stack.capacity() * std::mem::size_of::(), + ) + }) +} + pub(crate) fn side_table_census() -> Vec { vec![ super::site_cache::census(), diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 18b265330c..468bb5ef33 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -276,7 +276,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -291,9 +291,9 @@ "function": "run_to_completion" }, "sources": { - "crates/perry-runtime/src/gc/census.rs": "1ddeeec3ca81b792a222dbe165c32b7b995c084f66c760cb6cd3f26baf2cb07a", + "crates/perry-runtime/src/gc/census.rs": "0fd14b011bdbe0ae561e105d6d1acf0e9cc03e8a86ddef12685e307d1adee55a", "crates/perry-runtime/src/gc/cycle.rs": "77eadaf7c4157308c3b14be5e1ff11d7198b84b503a50ff2d244851248d3e800", - "crates/perry-runtime/src/gc/mod.rs": "cd5b27d4cbc5a0f100eefe8de71e1db6554c5a3e180b9996a658b581d2dd8348", + "crates/perry-runtime/src/gc/mod.rs": "78aca1306d4679a386e670cd96c4fa5f5046a6c28083e54fd6de7abec96da0e0", "crates/perry-runtime/src/gc/policy.rs": "dc9242ed40c0aa9c411d1ec0235c0219c6716dd82d56eb4d46578f7e889825d2", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } From 06ab3ef2be7f3037e6ca6d12c8d309c416abd6c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 20:33:38 +0200 Subject: [PATCH 15/27] docs(diagnostics): report regex census rows Record the row accounting, reconciliation contract, sabotage coverage, mainline cherry-pick proof, local gates, and RX2 perrymaster request. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit fe3804009e3235bce0fb8f4f8dfc69afca915b94) --- .../codex/REPORT_regex_census_rows.md | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 cc-perf-campaign/codex/REPORT_regex_census_rows.md diff --git a/cc-perf-campaign/codex/REPORT_regex_census_rows.md b/cc-perf-campaign/codex/REPORT_regex_census_rows.md new file mode 100644 index 0000000000..604cab6ec6 --- /dev/null +++ b/cc-perf-campaign/codex/REPORT_regex_census_rows.md @@ -0,0 +1,175 @@ +# RegExp heap-census rows + +## Branch and commits + +- Branch: `diag/regex-census-rows` +- Base: `a93908a6cc684d511a0af8561b65be4f0536fbd4` (`fork/perf/regex-literal-site-test`) +- Common implementation: `49f551f05` (`feat(diagnostics): account regex tables in heap census`) +- #9958 site-table integration: `5e8079138` (`feat(diagnostics): add literal-site regex census rows`) + +Both implementation commits are diagnostic-only. No construction, matching, +cache-maintenance, or collection path calls the new walkers. The walkers run +only while an explicitly requested heap census is being assembled. + +## Emitted rows and byte derivation + +Every row is a JSON object in the census's existing `side_tables` array: +`{"table":"regex.","entries":,"bytes":,...}`. + +- `regex.pointers`: `entries` is `REGEX_POINTERS.len()`; `bytes` is the + HashSet bucket/control estimate; `live_headers` is the marked-or-pinned + RegExp-header count at sweep entry. +- `regex.program_cache`: the 512-entry `REGEX_CACHE`; `bytes` is HashMap + storage plus one de-duplicated lower-bound estimate per compiled `regex` + program. `compiled_programs`, `opaque_program_bytes`, `cleared`, and + `evictions` are included. The event counters read the already-existing + `PERRY_REGEX_DIAG` state and are zero when that diagnostic is off. +- `regex.fancy_cache`: the 512-entry `FANCY_CACHE`; HashMap storage plus one + de-duplicated `fancy_regex::Regex` lower bound. +- `regex.repeat_cache`: the 512-entry `REPEAT_MATCHER_CACHE`; HashMap storage, + the public wrapper/Arc lower bound, and visible capture-name Vec/String + buffers. The opaque `regress::Regex` heap graph is not exposed. +- `regex.validated_patterns`: the 512-entry validation map, including map + storage and owned pattern/flag String capacities. +- `regex.content_cache`: the 1,024-entry content map, collision-bucket Vec + capacities, owned pattern/flag text, `Programs` Arc allocations, and matcher + lower bounds not already charged to an engine cache. It reports + `pinned_programs` and `opaque_program_bytes`. +- `regex.literal_sites`: the 1,024-slot literal-key Vec, including its full + `Option` capacity. Its Arc text allocations are shared with the + content table and are not charged twice. +- `regex.site_table`: the #9958 site-to-rooted-header map. `sites` and + `rooted_headers` are exact. `rooted_header_bytes` is reported but explicitly + outside `side_table_bytes`, because those 56-byte headers are already in the + GC live-heap census. `pinned_programs` and `pinned_program_bytes` are gross, + unique program-bundle retention measurements, including programs shared + with content/engine caches; that gross field is explicitly outside the + additive total to prevent double counting. `exclusively_attributed_programs` + and `attributed_program_bytes` are the de-duplicated subset owned nowhere + else and are inside this row's `bytes` and `side_table_bytes`. +- `regex.active_factory_sites`: the transient #9958 authorization-stack Vec + capacity and current entries (normally empty when a synchronous census + runs). +- `regex.expando_owners`: the RegExp owner share of the mixed exotic-expando + HashMap plus each RegExp-owned property Vec and key-string capacity; + `owners` and `properties` are included. +- `regex.matcher_kinds`: counts `unbuilt`, `standard`, `fancy`, and `repeat` + headers. `bytes=0` and `bytes_inside_side_table_bytes=false`, because the tag + resides in each already-counted `RegExpHeader` rather than a side table. + +The regex engines do not expose their complete compiled heap graphs. All +program rows therefore carry +`program_bytes_estimate="opaque_inline_lower_bound"`: Arc counters, the public +wrapper value, exposed source text, and exposed auxiliary buffers are counted; +unexposed automata/program allocations are not guessed. The program-count +fields remain exact and are the decisive signal if the RX2 delta is primarily +opaque engine storage. + +## Reconciliation + +`side_table_bytes` remains an explicit census estimate, not an allocator tag. +The assembly now computes: + +`side_table_bytes = non_regex_side_table_bytes + regex_side_table_bytes`. + +Legacy regex tuples from #9958 are removed from the ordinary row stream before +summing. The rich regex rows are serialized once, while +`regex_side_table_bytes` is built by an independent second diagnostic walk of +the same registered table inventory. Consequently the sum of every emitted +`regex.*` row's `bytes` must equal `regex_side_table_bytes`; omitting a row does +not silently shrink the attributed total. + +Gross `regex.site_table.rooted_header_bytes` and `pinned_program_bytes` are +labelled outside the additive total. The row's table bytes and +`attributed_program_bytes` are inside it. This preserves reconciliation while +still exposing the causal site-pinned working set side by side. + +## Tests and sabotage + +- `census_prints_regex_rows_that_reconcile_with_side_table_total` constructs + six distinct regex sources, executes them, evaluates one direct #9958 + literal site twice, serializes and parses the direct census document, and + asserts pointer entries >= 6, site/root/header/program counts >= 1, the full + row inventory, `sum(regex row bytes) == regex_side_table_bytes`, and + `side_table_bytes - non_regex_side_table_bytes == regex_side_table_bytes`. + Its sabotage assertion removes one non-zero row contribution and proves the + independent attribution total no longer reconciles. Deleting one row from + registration therefore fails the equality assertion. +- `census_regex_rows_are_zero_cost_when_not_requested` resets a cfg(test) + census-walk counter, constructs and matches a regex, proves the counter is + still zero, invokes the census directly, and proves it advances. Adding a + per-construction bookkeeping call makes its zero assertion fail. + +## Gates + +Every reported Cargo test/build invocation used `-j4` and the campaign build +lock, and launched only after `df -g /` reported at least 12 GiB. + +- `cargo test -p perry-runtime --release --lib -j4 -- --test-threads=1 census`: + final tree, 20 passed, 0 failed, 3,261 filtered out. +- `cargo test -p perry-runtime --release --lib -j4 -- --test-threads=1 regex`: + final tree, 130 passed, 0 failed, 3,151 filtered out. +- `cargo test -p perry-runtime --release --lib -j4 -- --test-threads=1`: + the pre-split implementation passed through Cargo (3,276 passed, 0 failed, + 4 ignored); the exact final Cargo-built executable was then run directly + after the source-only commit split and passed 3,277, 0 failed, 4 ignored. + A final no-op Cargo wrapper retry was prohibited because its guarded + precheck reported 10 GiB after another lane's build. +- `cargo build --release -p perry-runtime --features wasm-host -j4`: + the combined implementation before its source-only commit split passed in + 4m34s. A final-tree refresh was prohibited by the same 10 GiB precheck; the + final default-feature release test target compiled without warnings. +- Direct `rustfmt --edition 2021 --check` on every touched Rust file: passed. +- `git diff --check`: passed. +- `scripts/check_file_size.sh`: passed; no Rust source exceeds 2,000 lines. +- `python3 -m json.tool scripts/gc_runtime_root_holders.json`: passed. +- `python3 scripts/gc_runtime_root_holders.py`: passed: 1,372 declarations, + 596 scanner-reached, 357 classified, 414 frontier-pinned, 152 scanners. +- `python3 scripts/gc_runtime_root_holders.py --self-test`: passed: 90 planted + declarations and 357 inventory entries. + +One non-mutating `cargo fmt --all -- --check` attempt was mistakenly allowed +to continue after its chained precheck printed 8 GiB; it reported only the two +formatting changes then applied. The final formatting gate was rerun directly +with `rustfmt --check` on every touched Rust file and passed. No build/test was +started below the floor. + +The GC snapshot pin was re-audited because `gc/census.rs` and the module list +changed. `PASS1_MARKED` is still taken out of TLS before `take_census`; all new +regex walks occur afterward, and neither boundary nor intervening cycle flow +changed. No thread-local was added. + +One initial full-suite run under a PTY-like harness state had 3,275 passing and +one environment-dependent failure in +`tty::tests::columns_undefined_when_not_tty` (it read terminal width 80). The +exact test passed in a non-TTY process, and the subsequent non-TTY full rerun +passed 3,276 with 4 ignored. The final-tree result above supersedes that +diagnostic history. + +## Origin/main application check + +The implementation was split at the stack boundary. In a disposable checkout +of `origin/main` at `8b7dc3342b22fe6270739c8d51585c3d2cdfa618`, +`git cherry-pick --no-commit 49f551f05` completed with no conflicts. The +excluded `5e8079138` commit contains the #9958 rooted-site row plus the +#9918/#9958 content/literal cache-layout accessors and the branch-specific GC +snapshot pins. Thus the common regex-table census applies cleanly to main; +the site-table integration is intentionally the one omitted stack row. + +## PerryMaster request + +On `app-main7rx` and `app-main7` (the still-running RX2 arm and control), run +one 120-second idle interval followed by one SIGUSR2 heap census on each. Send +the complete `regex.*` rows and the three side-table totals side by side: +`side_table_bytes`, `regex_side_table_bytes`, and +`non_regex_side_table_bytes`. + +In particular compare `regex.site_table.sites`, `rooted_headers`, +`pinned_programs`, `pinned_program_bytes`, and `attributed_program_bytes`, plus +the 512-entry engine-cache program counts. The expected explanation for RX2's +88.3 MB versus 82.4 MB (+6 MB) is roughly 550 site-pinned programs beyond the +512-entry engine cache. If that population is present, the gross site-pinned +lower bound/count identifies it even when shared ownership assigns additive +bytes to another regex row. If it is not present, the side-by-side reconciled +rows identify which other regex table grew; if no regex row explains the +delta, that is the finding and the residual belongs outside regex attribution. From ff9f98d980f55615129789da75ef0d9fd98880e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 07:16:33 +0200 Subject: [PATCH 16/27] perf(gc): scope shape and box scanners to young entries Keep minor remembered sets for boxed roots and the shape table's carrier mutations. Compact both sets after each minor while retaining authoritative full-table walks for major collection. Report whole copied-minor pause time and its scanner share together. (cherry picked from commit 64ef925e6bc142cdb6fef14fe841bad37554c620) --- changelog.d/minor-scanner-young-logs.md | 5 + crates/perry-runtime/src/box.rs | 145 +++++++++++ crates/perry-runtime/src/gc/copying.rs | 24 +- .../perry-runtime/src/gc/scanner_profile.rs | 7 +- .../src/gc/tests/young_log_tests.rs | 242 ++++++++++++++++++ crates/perry-runtime/src/gc/young_log.rs | 40 ++- crates/perry-runtime/src/object/shapes.rs | 142 ++++++++-- .../src/object/shapes_test_support.rs | 20 ++ 8 files changed, 586 insertions(+), 39 deletions(-) create mode 100644 changelog.d/minor-scanner-young-logs.md diff --git a/changelog.d/minor-scanner-young-logs.md b/changelog.d/minor-scanner-young-logs.md new file mode 100644 index 0000000000..cf4834a894 --- /dev/null +++ b/changelog.d/minor-scanner-young-logs.md @@ -0,0 +1,5 @@ +Copying-minor scans of shape descriptors and captured-variable boxes now walk +only entries that can still expose non-old GC pointers. This removes the two +largest table-size-dependent root-scan costs, while full collections retain +their authoritative whole-table walks. `PERRY_GC_DIAG=1` also reports the +whole copying-minor pause and its scanner share on each completed-minor line. diff --git a/crates/perry-runtime/src/box.rs b/crates/perry-runtime/src/box.rs index dc65e4f791..44079b511d 100644 --- a/crates/perry-runtime/src/box.rs +++ b/crates/perry-runtime/src/box.rs @@ -117,6 +117,28 @@ crate::perry_thread_local! { 16 * 1024, crate::fast_hash::PtrHasher, )); + /// Box addresses whose JSValue payload may matter to a minor collection. + /// The registry itself is the authoritative full/major root set; this is + /// only its minor remembered set. + static BOX_YOUNG_ROOTS: std::cell::RefCell> = + const { std::cell::RefCell::new(crate::gc::young_log::YoungLog::new()) }; + #[cfg(test)] + static BOX_YOUNG_LOG_SUPPRESSED: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +const BOX_YOUNG_LOG_NAME: &str = "box.roots"; + +/// Arm the box minor-root log before publishing a young payload. +#[inline] +fn note_box_young_root(addr: usize, bits: u64) { + if !crate::gc::young_log::bits_are_minor_relevant(bits) { + return; + } + #[cfg(test)] + if BOX_YOUNG_LOG_SUPPRESSED.with(std::cell::Cell::get) { + return; + } + BOX_YOUNG_ROOTS.with(|log| log.borrow_mut().note(addr)); } /// Number of slots in each registry's direct-mapped positive cache. Eight @@ -680,6 +702,7 @@ pub extern "C" fn js_box_alloc_bits(initial_bits: i64) -> *mut Box { unsafe { (*ptr).value = initial_bits as u64; } + note_box_young_root(addr, initial_bits as u64); BOX_REGISTRY.with(|r| { r.borrow_mut().insert(addr); }); @@ -699,6 +722,7 @@ pub extern "C" fn js_box_alloc_bits(initial_bits: i64) -> *mut Box { return std::ptr::null_mut(); } (*ptr).value = initial_bits as u64; + note_box_young_root(ptr as usize, initial_bits as u64); BOX_REGISTRY.with(|r| { r.borrow_mut().insert(ptr as usize); }); @@ -928,7 +952,14 @@ pub fn scan_box_roots(mark: &mut dyn FnMut(f64)) { } pub fn scan_box_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + if visitor.young_scope() { + scan_box_young_roots_mut(visitor); + return; + } let full_trace = crate::gc::full_trace_active(); + let mut visited = 0u64; + let table_len = BOX_REGISTRY.with(|registry| registry.borrow().len()) as u64; + let mut kept = Vec::new(); ASYNC_PENDING_RELEASES.with(|pending| { let pending = pending.borrow(); BOX_REGISTRY.with(|r| { @@ -957,11 +988,103 @@ pub fn scan_box_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { if addr >= 0x1000 && (addr as u64) < 0x0001_0000_0000_0000 && addr % 8 == 0 { unsafe { visitor.visit_nanbox_u64_raw_slot(&raw mut (*ptr).value); + if crate::gc::young_log::bits_are_minor_relevant((*ptr).value) { + kept.push(addr); + } } + visited += 1; } } }); }); + let kept_len = kept.len() as u64; + BOX_YOUNG_ROOTS.with(|log| { + let mut log = log.borrow_mut(); + let _ = log.take_sorted(); + log.extend(kept); + }); + crate::gc::young_log::note_walk( + BOX_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: false, + logged: visited, + visited, + kept: kept_len, + table_len, + }, + ); +} + +/// Every live box whose current payload a minor can move, mark through, or +/// sweep. This is the authoritative debug re-derivation of the remembered set. +fn relevant_box_roots() -> Vec { + let mut relevant = BOX_REGISTRY.with(|registry| { + registry + .borrow() + .iter() + .copied() + .filter(|&addr| { + let ptr = addr as *mut Box; + is_plausible_box_ptr(ptr) + && unsafe { crate::gc::young_log::bits_are_minor_relevant((*ptr).value) } + }) + .collect::>() + }); + relevant.sort_unstable(); + relevant +} + +/// Minor root scan: price only the logged boxes, and compact the log from the +/// post-visit payloads. The visit counter lives here because this is the work +/// whose fixed cost the counter measures. +fn scan_box_young_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + let table_len = BOX_REGISTRY.with(|registry| registry.borrow().len()) as u64; + #[cfg(any(debug_assertions, test))] + BOX_YOUNG_ROOTS.with(|log| { + let relevant = relevant_box_roots(); + log.borrow() + .debug_assert_logged(BOX_YOUNG_LOG_NAME, &relevant); + }); + + let mut logged = 0u64; + let mut visited = 0u64; + let mut kept = BOX_YOUNG_ROOTS.with(|log| log.borrow_mut().take_spare()); + loop { + let batch = BOX_YOUNG_ROOTS.with(|log| log.borrow_mut().take_sorted()); + if batch.is_empty() { + break; + } + logged += batch.len() as u64; + for addr in batch { + let registered = BOX_REGISTRY.with(|registry| registry.borrow().contains(&addr)); + if !registered { + continue; + } + let ptr = addr as *mut Box; + if !is_plausible_box_ptr(ptr) { + continue; + } + visited += 1; + unsafe { + visitor.visit_nanbox_u64_raw_slot(&raw mut (*ptr).value); + if crate::gc::young_log::bits_are_minor_relevant((*ptr).value) { + kept.push(addr); + } + } + } + } + let kept_len = kept.len() as u64; + BOX_YOUNG_ROOTS.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + BOX_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: true, + logged, + visited, + kept: kept_len, + table_len, + }, + ); } /// Get the raw JSValue bit pattern from a box. @@ -1213,6 +1336,7 @@ pub extern "C" fn js_box_set_bits(ptr: *mut Box, value_bits: i64) { return; } let bits = value_bits as u64; + note_box_young_root(ptr as usize, bits); (*ptr).value = bits; crate::gc::runtime_write_barrier_root_nanbox(bits); } @@ -1232,6 +1356,7 @@ pub extern "C" fn js_box_set_bits(ptr: *mut Box, value_bits: i64) { #[no_mangle] pub unsafe extern "C" fn js_box_set_bits_trusted_no_barrier(ptr: *mut Box, value_bits: i64) { unsafe { + note_box_young_root(ptr as usize, value_bits as u64); (*ptr).value = value_bits as u64; } } @@ -1479,6 +1604,7 @@ pub(crate) fn test_clear_box_registry() { BOX_REGISTRY.with(|r| r.borrow_mut().clear()); I32_BOX_REGISTRY.with(|r| r.borrow_mut().clear()); BOOL_BOX_REGISTRY.with(|r| r.borrow_mut().clear()); + BOX_YOUNG_ROOTS.with(|log| log.borrow_mut().clear()); BOX_FREE_HEAD.with(|h| h.set(0)); I32_BOX_FREE_HEAD.with(|h| h.set(0)); BOOL_BOX_FREE_HEAD.with(|h| h.set(0)); @@ -1501,6 +1627,25 @@ pub(crate) fn test_clear_box_registry() { } } +/// Test-only sabotage of the box write-side arming hook. The production +/// scanner's re-derivation must reject the missing log entry. +#[cfg(test)] +pub(crate) struct TestBoxYoungLogSuppression(bool); + +#[cfg(test)] +impl TestBoxYoungLogSuppression { + pub(crate) fn new() -> Self { + Self(BOX_YOUNG_LOG_SUPPRESSED.with(|cell| cell.replace(true))) + } +} + +#[cfg(test)] +impl Drop for TestBoxYoungLogSuppression { + fn drop(&mut self) { + BOX_YOUNG_LOG_SUPPRESSED.with(|cell| cell.set(self.0)); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index ae667d992d..981f567333 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -1911,9 +1911,23 @@ pub(super) fn run_copied_minor_attempt( collector.stats.eden_copied_bytes, collector.stats.survivor_first_round_live_bytes, ); + if let Some(d) = collector.survival.as_ref() { + d.report(super::survival_diag::next_minor_seq()); + } + crate::arena::alloc_sample::report("minor"); + super::diag_sites::report_primitive_dispatch("minor"); + crate::object::shapes::id_list_report(); + report_forwarding_refusals("copying_minor"); + let scan_us = super::scanner_profile::report_and_reset("copying_minor"); if crate::gc::gc_diag_enabled() { + // This is intentionally the last diagnostic action before returning to + // the mutator: `pause_us` prices the whole copied-minor path, including + // finalization, pruning, policy feedback and the diagnostic work above. + let pause_us = start.elapsed().as_micros() as u64; eprintln!( - "[gc-copy-minor] ran in_place={} untraced={} untraced_cycles={} untraced_objects={} in_place_blocks={} in_place_dead_bytes={} sparse_blocks={} survival_permille={} copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={} tenuring_survivals={} eden_live_bytes={} eden_copied_bytes={} survivor_live_bytes={} survivor_first_round_live_bytes={} trigger={:?} declared_safepoint={}", + "[gc-copy-minor] ran pause_us={} scan_us={} in_place={} untraced={} untraced_cycles={} untraced_objects={} in_place_blocks={} in_place_dead_bytes={} sparse_blocks={} survival_permille={} copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={} tenuring_survivals={} eden_live_bytes={} eden_copied_bytes={} survivor_live_bytes={} survivor_first_round_live_bytes={} trigger={:?} declared_safepoint={}", + pause_us, + scan_us, collector.stats.in_place_promotion, untraced, super::untraced_promotion_cycles(), @@ -1936,14 +1950,6 @@ pub(super) fn run_copied_minor_attempt( super::policy::GC_AT_DECLARED_SAFEPOINT.with(std::cell::Cell::get) ); } - if let Some(d) = collector.survival.as_ref() { - d.report(super::survival_diag::next_minor_seq()); - } - crate::arena::alloc_sample::report("minor"); - super::diag_sites::report_primitive_dispatch("minor"); - crate::object::shapes::id_list_report(); - report_forwarding_refusals("copying_minor"); - super::scanner_profile::report_and_reset("copying_minor"); CopiedMinorAttempt::Done(Some(CopiedMinorFastPathOutcome { freed_bytes, malloc_swept: malloc_sweep_due, diff --git a/crates/perry-runtime/src/gc/scanner_profile.rs b/crates/perry-runtime/src/gc/scanner_profile.rs index 2e95310269..b86871070f 100644 --- a/crates/perry-runtime/src/gc/scanner_profile.rs +++ b/crates/perry-runtime/src/gc/scanner_profile.rs @@ -128,14 +128,14 @@ pub(super) fn note_scanner( /// Print the per-scanner breakdown accumulated since the last report, then /// clear it. Called once per copied minor from the `[gc-copy-minor]` diag site. -pub(super) fn report_and_reset(cycle_label: &str) { +pub(super) fn report_and_reset(cycle_label: &str) -> u64 { if !scanner_profile_enabled() { - return; + return 0; } super::young_log::report_and_reset(cycle_label); let mut rows = SCANNER_PROFILE.with(|rows| std::mem::take(&mut *rows.borrow_mut())); if rows.is_empty() { - return; + return 0; } rows.sort_by(|a, b| b.1.nanos.cmp(&a.1.nanos)); let total_ns: u64 = rows.iter().map(|(_, row)| row.nanos).sum(); @@ -159,4 +159,5 @@ pub(super) fn report_and_reset(cycle_label: &str) { row.rewrites ); } + total_ns / 1000 } diff --git a/crates/perry-runtime/src/gc/tests/young_log_tests.rs b/crates/perry-runtime/src/gc/tests/young_log_tests.rs index e6cf25887d..53eaa899f3 100644 --- a/crates/perry-runtime/src/gc/tests/young_log_tests.rs +++ b/crates/perry-runtime/src/gc/tests/young_log_tests.rs @@ -38,6 +38,10 @@ fn old_closure() -> usize { ptr as usize } +fn old_leaf() -> usize { + crate::arena::arena_alloc_gc_old(32, 8, GC_TYPE_STRING) as usize +} + unsafe fn young_keys_array() -> *mut crate::array::ArrayHeader { let arr = crate::arena::arena_alloc_gc( std::mem::size_of::(), @@ -602,3 +606,241 @@ fn installing_an_external_shape_id_arms_the_family_log() { "the family must have followed the keys array" ); } + +// ------------------------------------------------------ fixed-cost scanners + +/// N old shape families plus k young ones must price exactly k entries in the +/// minor-scoped scanner. Sabotage: make `note_young_keys` a no-op; the +/// re-derivation fails before this count can be observed. +#[test] +fn shape_table_minor_walk_visits_exactly_k_young_entries() { + const N: usize = 96; + const K: usize = 3; + let _guard = CopyingNurseryTestGuard::new(K as u32); + gc_register_mutable_root_scanner(crate::object::shapes::scan_shape_table_rekey_mut); + crate::object::shapes::test_clear_shape_table(); + + for _ in 0..N { + let keys = crate::arena::arena_alloc_gc_old( + std::mem::size_of::(), + std::mem::align_of::(), + GC_TYPE_ARRAY, + ) as *mut crate::array::ArrayHeader; + unsafe { + (*keys).length = 0; + (*keys).capacity = 0; + } + crate::object::shapes::shape_descriptor_ensure(keys, 0, 0).expect("old shape"); + } + for slot in 0..K { + let keys = unsafe { young_keys_array() }; + js_shadow_slot_set(slot as u32, ptr_bits(keys as usize)); + crate::object::shapes::shape_descriptor_ensure(keys, 0, 0).expect("young shape"); + } + + let _ = gc_collect_minor(); + let row = walk("shapes.families+indices"); + assert!(row.partial, "{row:?}"); + assert_eq!( + row.visited, K as u64, + "minor work must be young-sized: {row:?}" + ); + assert!( + row.table_len >= (N + K) as u64, + "fixture did not build N+k: {row:?}" + ); +} + +/// The debug/test authoritative walk is the proof that every shape writer +/// arms the log. This deliberately suppresses the production family funnel; +/// deleting the assertion makes the sabotage go green. +#[test] +fn shape_table_rederivation_rejects_a_suppressed_logging_site() { + let _guard = CopyingNurseryTestGuard::new(0); + crate::object::shapes::test_clear_shape_table(); + let keys = unsafe { young_keys_array() }; + { + let _sabotage = crate::object::shapes::TestShapeYoungLogSuppression::new(); + crate::object::shapes::shape_descriptor_ensure(keys, 0, 0).expect("shape"); + } + let valid = build_valid_pointer_set(); + let mut visitor = RuntimeRootVisitor::for_mark_scoped(&valid, true); + let rejected = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + crate::object::shapes::scan_shape_table_rekey_mut(&mut visitor); + })); + assert!( + rejected.is_err(), + "a missing shape log note must be detected" + ); +} + +/// Promotion removes a shape address from the minor log, without removing the +/// descriptor from the authoritative table used by the next full/major walk. +/// Sabotage: change the post-visit keep predicate back to +/// `addr_is_minor_relevant(from_space)`; `kept` never reaches zero. +#[test] +fn promoted_shape_entry_leaves_young_log_and_remains_in_major_walk() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::object::shapes::scan_shape_table_rekey_mut); + crate::object::shapes::test_clear_shape_table(); + let keys = unsafe { young_keys_array() }; + js_shadow_slot_set(0, ptr_bits(keys as usize)); + let id = crate::object::shapes::shape_descriptor_ensure(keys, 0, 0).expect("shape"); + + for _ in 0..4 { + let _ = gc_collect_minor(); + } + let promoted = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert!( + !crate::arena::pointer_in_nursery(promoted), + "fixture must promote" + ); + assert_eq!(walk("shapes.families+indices").kept, 0); + + let valid = build_valid_pointer_set(); + let mut visitor = RuntimeRootVisitor::for_rewrite(&valid); + crate::object::shapes::scan_shape_table_rekey_mut(&mut visitor); + let row = walk("shapes.families+indices"); + assert!( + !row.partial, + "major/full walk must remain authoritative: {row:?}" + ); + assert!( + row.visited >= 1, + "major/full walk must still see the descriptor" + ); + assert_eq!( + crate::object::shapes::shape_descriptor_by_id(id).map(|d| d.keys), + Some(promoted as u64) + ); +} + +/// An already-Longlived keys array can gain a new nursery key at the same +/// address. Re-stamping the old receiver is the structural publication +/// chokepoint that must re-arm it. +#[test] +fn shape_mutation_to_new_young_key_rearms_minor_log() { + let _guard = CopyingNurseryTestGuard::new(0); + gc_register_mutable_root_scanner(crate::object::shapes::scan_shape_table_rekey_mut); + crate::object::shapes::test_clear_shape_table(); + unsafe { + let bytes = std::mem::size_of::() + 8; + let keys = crate::arena::arena_alloc_gc_longlived(bytes, 8, GC_TYPE_ARRAY) + as *mut crate::array::ArrayHeader; + (*keys).length = 1; + (*keys).capacity = 1; + let slot = + (keys as *mut u8).add(std::mem::size_of::()) as *mut f64; + *slot = f64::from_bits(string_bits(old_leaf())); + let id = crate::object::shapes::shape_descriptor_ensure(keys, 1, 0).expect("shape"); + let (owner, _) = alloc_old_test_object(0); + crate::object::shapes::stamp_object_shape_id_with_carrier_note(owner, id); + let _ = gc_collect_minor(); + assert_eq!(walk("shapes.families+indices").kept, 0); + + let young = young_leaf(); + *slot = f64::from_bits(string_bits(young)); + crate::object::shapes::stamp_object_shape_id_with_carrier_note(owner, id); + let _ = gc_collect_minor(); + let moved = ((*slot).to_bits() & POINTER_MASK) as usize; + assert_ne!( + moved, young, + "the mutation hook must make the new key visible" + ); + assert!(walk("shapes.families+indices").visited >= 1); + } +} + +/// N old box payloads plus k young payloads must price exactly k registry +/// entries. The counter is recorded inside `scan_box_young_roots_mut`. +#[test] +fn box_roots_minor_walk_visits_exactly_k_young_entries() { + const N: usize = 128; + const K: usize = 4; + let _guard = CopyingNurseryTestGuard::new(0); + gc_register_mutable_root_scanner(crate::r#box::scan_box_roots_mut); + for _ in 0..N { + crate::r#box::js_box_alloc_bits(string_bits(old_leaf()) as i64); + } + for _ in 0..K { + crate::r#box::js_box_alloc_bits(string_bits(young_leaf()) as i64); + } + + let _ = gc_collect_minor(); + let row = walk("box.roots"); + assert!(row.partial, "{row:?}"); + assert_eq!( + row.visited, K as u64, + "minor work must be young-sized: {row:?}" + ); + assert_eq!( + row.table_len, + (N + K) as u64, + "fixture registry mismatch: {row:?}" + ); +} + +/// Suppress the real `js_box_set_bits` arming site and prove the full-registry +/// re-derivation catches the omission. +#[test] +fn box_root_rederivation_rejects_a_suppressed_mutation_hook() { + let _guard = CopyingNurseryTestGuard::new(0); + let cell = crate::r#box::js_box_alloc_bits(string_bits(old_leaf()) as i64); + let young = young_leaf(); + { + let _sabotage = crate::r#box::TestBoxYoungLogSuppression::new(); + crate::r#box::js_box_set_bits(cell, string_bits(young) as i64); + } + let valid = build_valid_pointer_set(); + let mut visitor = RuntimeRootVisitor::for_mark_scoped(&valid, true); + let rejected = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + crate::r#box::scan_box_roots_mut(&mut visitor); + })); + assert!( + rejected.is_err(), + "a missing box mutation note must be detected" + ); +} + +#[test] +fn box_mutation_to_new_young_object_is_visited() { + let _guard = CopyingNurseryTestGuard::new(0); + gc_register_mutable_root_scanner(crate::r#box::scan_box_roots_mut); + let cell = crate::r#box::js_box_alloc_bits(string_bits(old_leaf()) as i64); + let young = young_leaf(); + crate::r#box::js_box_set_bits(cell, string_bits(young) as i64); + + let _ = gc_collect_minor(); + let moved = (crate::r#box::js_box_get_bits(cell) as u64 & POINTER_MASK) as usize; + assert_ne!(moved, young, "setter must re-arm a previously old box"); + assert_eq!(walk("box.roots").visited, 1); +} + +#[test] +fn promoted_box_root_leaves_log_and_is_found_by_full_walk() { + let _guard = CopyingNurseryTestGuard::new(0); + gc_register_mutable_root_scanner(crate::r#box::scan_box_roots_mut); + let cell = crate::r#box::js_box_alloc_bits(string_bits(young_leaf()) as i64); + for _ in 0..4 { + let _ = gc_collect_minor(); + } + let promoted_bits = crate::r#box::js_box_get_bits(cell) as u64; + let promoted = (promoted_bits & POINTER_MASK) as usize; + assert!( + !crate::arena::pointer_in_nursery(promoted), + "fixture must promote" + ); + assert_eq!(walk("box.roots").kept, 0); + + let mut seen = false; + crate::r#box::scan_box_roots(&mut |value| { + if value.to_bits() == promoted_bits { + seen = true; + } + }); + assert!( + seen, + "the unchanged full walk must still enumerate promoted roots" + ); + assert!(!walk("box.roots").partial); +} diff --git a/crates/perry-runtime/src/gc/young_log.rs b/crates/perry-runtime/src/gc/young_log.rs index 9f6d43b22f..d83d9bb36c 100644 --- a/crates/perry-runtime/src/gc/young_log.rs +++ b/crates/perry-runtime/src/gc/young_log.rs @@ -155,7 +155,7 @@ impl YoungLog { /// Rule 2: the log must name every key in `relevant`. `relevant` is the /// set the caller re-derived from the authoritative table under /// `debug_assertions`; a miss is a writer that publishes without noting. - #[cfg(debug_assertions)] + #[cfg(any(debug_assertions, test))] pub(crate) fn debug_assert_logged(&self, table: &'static str, relevant: &[K]) where K: std::fmt::Debug, @@ -210,6 +210,44 @@ pub(crate) fn addr_is_minor_relevant(addr: usize) -> bool { } } +/// Can a minor move or reclaim the object at `addr`? +/// +/// This is narrower than [`addr_is_minor_relevant`]: `Longlived` objects must +/// sometimes be traced *through*, but they are never themselves moved or +/// swept. Side tables whose entries name known GC leaves (shape property +/// keys are strings/symbol headers) use this predicate so an immortal leaf +/// does not pin its entry in a young log forever. +#[inline] +pub(crate) fn addr_is_minor_collectible(addr: usize) -> bool { + if addr == 0 { + return false; + } + match crate::arena::classify_heap_space(addr) { + HeapSpace::NurseryEden + | HeapSpace::Survivor0 + | HeapSpace::Survivor1 + | HeapSpace::PromotedYoung => true, + HeapSpace::Old | HeapSpace::Longlived => false, + HeapSpace::Unknown => { + addr > GC_HEADER_SIZE + && super::malloc::gc_malloc_header_is_tracked( + (addr - GC_HEADER_SIZE) as *const super::GcHeader, + ) + } + } +} + +/// [`addr_is_minor_collectible`] for a NaN-boxed value. +#[inline] +pub(crate) fn bits_are_minor_collectible(bits: u64) -> bool { + let tag = bits & TAG_MASK; + if tag == POINTER_TAG || tag == STRING_TAG || tag == BIGINT_TAG { + addr_is_minor_collectible((bits & POINTER_MASK) as usize) + } else { + false + } +} + /// [`addr_is_minor_relevant`] for a NaN-boxed value: only the three /// pointer-carrying tags decode to an address; numbers, booleans, short /// strings and `undefined` are never relevant. diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index b0d47aac53..51724f362b 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -264,6 +264,28 @@ struct ShapeTableInner { const SHAPE_YOUNG_LOG_NAME: &str = "shapes.families+indices"; +crate::perry_thread_local! { + /// Carrier notes can be produced while a GC walk already borrows the shape + /// table. Keep that write-side stream separate and merge it at the next + /// scanner entry rather than re-borrowing `ShapeTableInner` recursively. + static SHAPE_CARRIER_YOUNG_KEYS: RefCell> = + const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; + #[cfg(test)] + static SHAPE_YOUNG_LOG_SUPPRESSED: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +#[inline] +fn note_shape_carrier_candidate(keys: u64) { + if !crate::gc::young_log::addr_is_minor_relevant(keys as usize) { + return; + } + #[cfg(test)] + if SHAPE_YOUNG_LOG_SUPPRESSED.with(std::cell::Cell::get) { + return; + } + SHAPE_CARRIER_YOUNG_KEYS.with(|log| log.borrow_mut().note(keys)); +} + /// Re-export of the id-list operation counters' report, so the collector does /// not have to name a private sibling module. One `[gc-idlist]` line per /// copying minor under `PERRY_GC_DIAG=1`; `elems_moved` is the falsifier for @@ -281,7 +303,11 @@ impl ShapeTableInner { /// call this themselves. #[inline] fn note_young_keys(&mut self, keys: u64) { - if crate::gc::young_log::addr_is_minor_relevant(keys as usize) { + #[cfg(test)] + if SHAPE_YOUNG_LOG_SUPPRESSED.with(std::cell::Cell::get) { + return; + } + if crate::gc::young_log::addr_is_minor_collectible(keys as usize) { self.young_keys.note(keys); } } @@ -692,8 +718,12 @@ pub(crate) unsafe fn note_old_generation_carrier(descriptor: Option) { return; } let record = descriptor.record as *mut ShapeRecord; + let newly_armed = !(*record).has(RECORD_FLAG_CACHE_CARRIER); // GC_STORE_AUDIT(POINTER_FREE): liveness bookkeeping bit, never a heap reference. (*record).set(RECORD_FLAG_CACHE_CARRIER, true); + if newly_armed { + note_shape_carrier_candidate(descriptor.keys); + } } /// The post-birth publication point for a ShapeId into a receiver's header @@ -770,7 +804,15 @@ pub(crate) unsafe fn stamp_object_shape_id_with_carrier_note( ) { (*obj).parent_class_id = id; if !crate::arena::pointer_in_nursery(obj as usize) { - note_old_generation_carrier(shape_descriptor_by_id(id)); + let descriptor = shape_descriptor_by_id(id); + note_old_generation_carrier(descriptor); + // This stamp is the structural-mutation publication funnel. Re-arm + // even when the descriptor was already an old carrier: an owned + // Longlived keys array may have just gained a nursery key at the same + // address, and its carrier flag alone cannot express that transition. + if let Some(descriptor) = descriptor { + note_shape_carrier_candidate(descriptor.keys); + } } } @@ -1930,6 +1972,8 @@ pub(crate) fn prune_dead_shape_keys_young(is_dead_owner: &dyn Fn(usize) -> bool) pub(crate) fn scan_shape_table_rekey_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { let table = &crate::state::state().shapes; let mut inner = table.inner.borrow_mut(); + let carrier_notes = SHAPE_CARRIER_YOUNG_KEYS.with(|log| log.borrow_mut().take_sorted()); + inner.young_keys.extend(carrier_notes); let rewrite_phase = visitor.is_metadata_rewrite_phase(); // #9754: a minor-scoped pass visits only the young-logged keys addresses; // the full walk below rebuilds the log from what it finds. @@ -2045,7 +2089,7 @@ pub(crate) fn scan_shape_table_rekey_mut(visitor: &mut crate::gc::RuntimeRootVis } // A full walk is authoritative: rebuild the young log from the tables. - let kept = relevant_shape_keys(&inner); + let kept = relevant_shape_keys(table, &inner); let kept_len = kept.len() as u64; let _ = inner.young_keys.take_sorted(); inner.young_keys.extend(kept); @@ -2080,33 +2124,82 @@ fn move_shape_family(table: &ShapeTable, inner: &mut ShapeTableInner, old: u64, inner.facts_remove(record.facts_key_with_keys(old), id); inner.facts_push_back(record.facts_key_with_keys(new), id); } - inner.family_push_back(new, id); + // Scanner-internal rekey: the caller keeps `new` from its post-visit + // relevance result (or the full walk rebuilds the log). Re-entering + // the writer funnel here would enqueue the same family mid-walk and + // price it twice in one minor. + inner.families.entry(new).or_default().push_back(id); } } /// Every keys address a minor can act on, re-derived from the authoritative /// tables (families and slot indices whose keys array is not old). -fn relevant_shape_keys(inner: &ShapeTableInner) -> Vec { - use crate::gc::young_log::addr_is_minor_relevant; - let mut relevant: Vec = inner - .families - .keys() - .copied() - .filter(|&keys| keys != 0 && addr_is_minor_relevant(keys as usize)) - .collect(); - relevant.extend( - inner - .indices - .keys() - .copied() - .filter(|&keys| addr_is_minor_relevant(keys)) - .map(|keys| keys as u64), - ); +fn relevant_shape_keys(table: &ShapeTable, inner: &ShapeTableInner) -> Vec { + let mut relevant: Vec = inner.families.keys().copied().collect(); + relevant.extend(inner.indices.keys().copied().map(|keys| keys as u64)); relevant.sort_unstable(); relevant.dedup(); + relevant.retain(|&keys| shape_keys_entry_is_minor_relevant(table, inner, keys)); relevant } +/// Exact minor-work predicate for one shape-table key. +/// +/// Nursery addresses must be rekeyed even for weak metadata entries. Malloc +/// arrays must be rooted when a carrier owns the family. A Longlived keys +/// array never moves or dies, so it matters only while a rooted family exposes +/// a collectible property-key leaf from its payload. Property keys are +/// strings/symbol headers and both are GC leaves; tracing through an immortal +/// key cannot discover a younger grandchild. +fn shape_keys_entry_is_minor_relevant( + table: &ShapeTable, + inner: &ShapeTableInner, + keys: u64, +) -> bool { + if keys == 0 { + return false; + } + let addr = keys as usize; + match crate::arena::classify_heap_space(addr) { + crate::arena::HeapSpace::NurseryEden + | crate::arena::HeapSpace::Survivor0 + | crate::arena::HeapSpace::Survivor1 + | crate::arena::HeapSpace::PromotedYoung => return true, + crate::arena::HeapSpace::Old => return false, + crate::arena::HeapSpace::Unknown => { + return family_has_root_carrier(table, inner, keys) + && crate::gc::young_log::addr_is_minor_collectible(addr); + } + crate::arena::HeapSpace::Longlived => {} + } + if !family_has_root_carrier(table, inner, keys) { + return false; + } + unsafe { + let Some(header) = crate::value::addr_class::try_read_tracked_gc_header(addr) else { + return false; + }; + if (*header.as_ptr()).obj_type != crate::gc::GC_TYPE_ARRAY { + return false; + } + let (slots, len) = super::keys_array_dense_slots(addr as *const ArrayHeader); + (0..len).any(|index| { + crate::gc::young_log::bits_are_minor_collectible((*slots.add(index)).to_bits()) + }) + } +} + +fn family_has_root_carrier(table: &ShapeTable, inner: &ShapeTableInner, keys: u64) -> bool { + inner.families.get(&keys).is_some_and(|ids| { + ids.as_slice().iter().any(|&id| { + table + .slab() + .get(id) + .is_some_and(|record| record.has(RECORD_FLAG_OLD_CARRIER) || record.cache_carrier()) + }) + }) +} + /// The minor-scoped walk (#9754): only the young-logged keys addresses, each /// visited exactly as the full walk visits it — the family's carrier gate, /// the record rewrite, the recycled-address retirement, the slot-index @@ -2118,9 +2211,9 @@ fn scan_shape_table_young( rewrite_phase: bool, ) { let table_len = (inner.families.len() + inner.indices.len()) as u64; - #[cfg(debug_assertions)] + #[cfg(any(debug_assertions, test))] { - let relevant = relevant_shape_keys(inner); + let relevant = relevant_shape_keys(table, inner); inner .young_keys .debug_assert_logged(SHAPE_YOUNG_LOG_NAME, &relevant); @@ -2242,10 +2335,7 @@ fn scan_shape_keys_address( inner.indices.remove(&addr); } } - ( - post, - crate::gc::young_log::addr_is_minor_relevant(post as usize), - ) + (post, shape_keys_entry_is_minor_relevant(table, inner, post)) } // #8112 sabotage switch. Suppressing the descriptor edge proves the fixture's diff --git a/crates/perry-runtime/src/object/shapes_test_support.rs b/crates/perry-runtime/src/object/shapes_test_support.rs index 6cb974c98c..21bc8f7b4a 100644 --- a/crates/perry-runtime/src/object/shapes_test_support.rs +++ b/crates/perry-runtime/src/object/shapes_test_support.rs @@ -44,6 +44,25 @@ pub(crate) struct TestRecycledKeysCheckSuppression { previous: bool, } +/// Suppress both shape-table young-log writer funnels. A test using this guard +/// must be rejected by the scanner's authoritative re-derivation. +#[cfg(test)] +pub(crate) struct TestShapeYoungLogSuppression(bool); + +#[cfg(test)] +impl TestShapeYoungLogSuppression { + pub(crate) fn new() -> Self { + Self(SHAPE_YOUNG_LOG_SUPPRESSED.with(|cell| cell.replace(true))) + } +} + +#[cfg(test)] +impl Drop for TestShapeYoungLogSuppression { + fn drop(&mut self) { + SHAPE_YOUNG_LOG_SUPPRESSED.with(|cell| cell.set(self.0)); + } +} + #[cfg(test)] impl TestRecycledKeysCheckSuppression { pub(crate) fn new() -> Self { @@ -84,6 +103,7 @@ pub(crate) fn test_clear_shape_table() { inner.by_facts.clear(); inner.families.clear(); inner.young_keys.clear(); + SHAPE_CARRIER_YOUNG_KEYS.with(|log| log.borrow_mut().clear()); // SAFETY: test-only reset with no slab reference held. unsafe { table.slab_mut().clear() }; drop(inner); From a3cecd0d88579fccdcabdd2fa6778a9aa5e13bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 07:18:42 +0200 Subject: [PATCH 17/27] docs(perf): report minor scanner young logs Record the scanner map, sabotage-able test coverage, disk-gated validation, predictions, and the exact perrymaster follow-up request. (cherry picked from commit 194fcb67675fe298457e75b3f6b9808f01e9ac64) --- .../codex/REPORT_minor_scanner_young_logs.md | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 cc-perf-campaign/codex/REPORT_minor_scanner_young_logs.md diff --git a/cc-perf-campaign/codex/REPORT_minor_scanner_young_logs.md b/cc-perf-campaign/codex/REPORT_minor_scanner_young_logs.md new file mode 100644 index 0000000000..fe8570d4a8 --- /dev/null +++ b/cc-perf-campaign/codex/REPORT_minor_scanner_young_logs.md @@ -0,0 +1,107 @@ +# Minor scanner young logs + +Implementation SHA: `d399c39ddb638a92b2735a6bacc2aef13def944a` + +## Map and mechanism + +- `object/shapes.rs:1972` scans two address-keyed structures. `families` maps a + keys-array address to every descriptor id whose slab record carries that + address; the descriptor record is the authoritative rewritable `keys` edge. + A family is a strong minor root only when an old receiver or an optimization + cache carries one of its descriptors. `indices` is a weak key-to-slot + accelerator keyed by the same keys-array address and needs only relocation + repair. Shape property-key payloads are strings/symbol headers, both GC + leaves. Nursery keys arrays can move; old arrays cannot; Longlived arrays do + not move or die but can temporarily contain a collectible key leaf. +- Shapes already had #9755's `young_keys` address log and the four + `shapes.indices` arm sites. Its keep predicate was + `addr_is_minor_relevant`, so every Longlived keys array stayed in the log + forever. `object/shapes.rs:2154` now re-derives actual minor work: nursery + addresses remain for relocation, malloc roots remain while carrier-owned, + and a Longlived carrier remains only while its property-key payload contains + a collectible leaf. `object/shapes.rs:271` receives old/cache carrier notes + without recursively borrowing the shape table; `object/shapes.rs:801` is the + enforced structural-publication funnel that re-arms a same-address mutation. + Scanner-internal rekeys do not enqueue a duplicate visit. +- `box.rs:954` previously walked every address in `BOX_REGISTRY`. These are + malloc-allocated mutable-capture/async state cells; the registry address is + not a GC pointer. Only the `Box::value` NaN-box can point into the nursery. + `I32Box` and `BoolBox` registries contain no GC edge and were never part of + this scanner. There was no partial box log. +- `box.rs:123` adds the box remembered set. Both allocation arms and both + mutation ABIs arm it before publishing a minor-relevant payload + (`box.rs:133`, `box.rs:705`, `box.rs:725`, `box.rs:1318`, `box.rs:1357`). + The trusted setter is included because generated boxed-local stores use it; + omitting that silent path would violate the enforced-funnel rule. Release + paths only clear/de-register cells, and scanner rewrites compact their own + entries. `box.rs:1040` owns the priced `visited` counter. +- Both minor walks sort/deduplicate their logged addresses, drop stale keys, + and keep only post-visit non-old entries. Full/major scans still enumerate + the authoritative whole tables and rebuild the logs. Under + `debug_assertions` and in lib tests, each minor scan re-derives the relevant + set from the whole table and asserts that the log is complete. +- `gc/copying.rs:1889` now emits `pause_us=` and `scan_us=` together on every + completed `[gc-copy-minor] ran` line. `pause_us` is sampled as the final + action before the copied-minor returns to the mutator; `scan_us` is the + already-profiled scanner total returned by `gc/scanner_profile.rs:131`. + Timing remains behind the existing cached `PERRY_GC_DIAG` gate. + +## Tests and sabotages + +- `shape_table_minor_walk_visits_exactly_k_young_entries`: N old families and + k young families produce `visited == k`. Sabotage: remove + `note_young_keys`; the completeness re-derivation panics. +- `shape_table_rederivation_rejects_a_suppressed_logging_site`: a test-only + suppression skips the production family arm and the scan must panic. +- `shape_mutation_to_new_young_key_rearms_minor_log`: a Longlived carrier that + gains a new nursery key at the same address must move that key. Sabotage: + remove the re-arm in `stamp_object_shape_id_with_carrier_note`. +- `box_roots_minor_walk_visits_exactly_k_young_entries`: N old payloads and k + young payloads produce `visited == k`. Sabotage: remove either allocator arm. +- `box_root_rederivation_rejects_a_suppressed_mutation_hook`: a test-only + suppression skips `js_box_set_bits` logging and the authoritative registry + walk must panic. +- `box_mutation_to_new_young_object_is_visited`: an old box changed to a new + nursery object is visited. Sabotage: remove the setter hook. +- `promoted_shape_entry_leaves_young_log_and_remains_in_major_walk` and + `promoted_box_root_leaves_log_and_is_found_by_full_walk`: promotion makes + `kept == 0`, while the next authoritative full walk still visits the entry. + Sabotage: retain the pre-visit/from-space classification or scope the full + walk to the log. +- Existing scanner-completeness and moving-witness suites are unchanged and + remain part of the requested runtime-lib gate. + +## Validation + +- `scripts/check_file_size.sh`: PASS. +- `git diff --check`: PASS. +- Cargo gates: NOT RUN. `df -g /` immediately before the first possible Cargo + invocation reported `0` GB available, below the binding 12 GB floor. Per the + task rule, no Cargo command was started and no wait for disk was attempted. +- Not run for the same reason: + `cargo test -p perry-runtime --release --lib -- --test-threads=1`; + `cargo build --release -p perry-runtime --features wasm-host`; + `cargo build --release -p perry`. + +## Predictions and exact perrymaster request + +Predictions: on a zero-live steady minor, +`object::shapes::scan_shape_table_rekey_mut` and +`r#box::scan_box_roots_mut` each fall from about 2 ms to at most 0.2 ms; +steady-minor scanner total falls from 7–8 ms to at most 3 ms; every completed +minor reports `pause_us` and `scan_us`. CPU bound is about -3% at 3300 chars +and larger at 400 chars, where minors are a larger share. RSS should be +unchanged (small retained log capacities only, within the allowed 1–10%). + +Perrymaster request, from pushed SHA: relink on the I7-view tree +(runtime-only), then run the three gates through +`/Users/amlug/projects/perry/secret-tests/cc-perf-campaign/measure_lock.sh --build` +at `-j4` using detached `nohup`: (1) +`cargo test -p perry-runtime --release --lib -- --test-threads=1`, (2) +`cargo build --release -p perry-runtime --features wasm-host`, and (3) +`cargo build --release -p perry`. Because this is GC-adjacent, the coordinator +must apply `run-extended-tests`. After green gates, do one graceful four-turn +3300-char run and one 400-char run with `PERRY_GC_DIAG=1`, preserving complete +`[gc-copy-minor] ran pause_us=... scan_us=...` and +`[gc-scanner-profile] copying_minor` lines. Then run paired 5x3300 + 3x400 +against I7-view for CPU and RSS. From 6b4c1ad868c84839b1487f69124c49145b7e1ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 11:20:56 +0200 Subject: [PATCH 18/27] perf(gc): split copying minor diagnostics by phase Account for the successful copying-minor path with diagnostic-only wall-time buckets that partition the same interval as pause_us. Include per-table from-space finalization and dead-owner prune detail so the remaining fixed minor cost can be localized from one complete ran line. (cherry picked from commit 09846784cf9fdb277fcd83b8b2c27c068dd0f003) --- crates/perry-runtime/src/gc/copying.rs | 101 +++++++-- crates/perry-runtime/src/gc/copying_phase.rs | 204 ++++++++++++++++++ crates/perry-runtime/src/gc/dead_owner.rs | 20 +- crates/perry-runtime/src/gc/mod.rs | 1 + .../src/node_submodules/diagnostics_gc.rs | 4 +- 5 files changed, 309 insertions(+), 21 deletions(-) create mode 100644 crates/perry-runtime/src/gc/copying_phase.rs diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index 981f567333..e6da9db671 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -1,3 +1,7 @@ +use super::copying_phase::{ + finalize_dead_copied_minor_from_space_side_allocations, CopyingMinorPhase as Phase, + CopyingMinorPhaseDiag as PhaseDiag, +}; use super::*; /// Largest object `move_young` will relocate. See its use site for the @@ -1234,6 +1238,7 @@ pub(super) fn run_copied_minor_attempt( let ptrs = eligibility .ptrs .expect("eligible copied-minor decision must carry pointer classifier"); + let mut phase_diag = PhaseDiag::enabled(); let phase_start = trace_phase_start(trace); let from_space_bytes = crate::arena::copying_from_space_in_use_bytes(); @@ -1257,11 +1262,15 @@ pub(super) fn run_copied_minor_attempt( && ptrs.malloc_registry_empty_at_start && untraced_promotion_instrument_veto().is_none() && super::should_attempt_first_cycle_promotion(); + let promotion_phase_start = PhaseDiag::start(&phase_diag); let promotion = if super::should_promote_young_in_place() || speculate_first_cycle { crate::arena::retag_young_for_in_place_promotion(speculate_first_cycle) } else { crate::arena::InPlacePromotion::default() }; + if let Some(diag) = phase_diag.as_mut() { + diag.record(Phase::Promotion, promotion_phase_start); + } // An empty plan (nothing in use to promote) falls back to the ordinary // path, so the from-space reset still runs. let promoting_in_place = !promotion.is_empty(); @@ -1340,8 +1349,13 @@ pub(super) fn run_copied_minor_attempt( "policy (should_promote_young_untraced)" }) }); + let reset_phase_start = PhaseDiag::start(&phase_diag); collector.stats.reset_blocks += crate::arena::copying_prepare_to_space(); + if let Some(diag) = phase_diag.as_mut() { + diag.record(Phase::BlockResetFlip, reset_phase_start); + } + let root_scan_phase_start = PhaseDiag::start(&phase_diag); let native_stack_walk = if untraced { Default::default() } else { @@ -1420,6 +1434,9 @@ pub(super) fn run_copied_minor_attempt( } visit_ffi_mutable_registered_roots_with_sources(&mut visitor, root_sources); } + if let Some(diag) = phase_diag.as_mut() { + diag.record(Phase::RootScan, root_scan_phase_start); + } // On an untraced promotion the dirty SCAN is where the whole per-object // mark pass lived: `retain`'s array store has a young child in every page @@ -1432,6 +1449,7 @@ pub(super) fn run_copied_minor_attempt( // read path for the remembered set, which is where #7187's lazy barrier // arming happens. Skipping it would leave the barrier unarmed for the next // cycle — a missing-edge bug one collection later. + let remembered_phase_start = PhaseDiag::start(&phase_diag); let snapshot = remembered_dirty_snapshot(); // #9754: objects whose every slot the dirty scan visited in-body — the // post-cycle coverage restore skips them (see `scan_dirty_object_slots`). @@ -1449,6 +1467,8 @@ pub(super) fn run_copied_minor_attempt( // reserved bytes, and under-estimating falls back to ordinary growth. let mut dirty_scan_covered = crate::fast_hash::new_ptr_hash_set_with_capacity(previous_dirty_covered_estimate()); + let mut remembered_entries = 0usize; + let mut remembered_slots = 0usize; if !untraced { let _phase = super::pin::CopyingWalkPhaseGuard::enter("remembered_set"); let remembered_stats = scan_remembered_dirty_slots_copying( @@ -1468,11 +1488,21 @@ pub(super) fn run_copied_minor_attempt( if let Some(trace) = trace.as_mut() { trace.remembered_set = remembered_stats; } + remembered_entries = remembered_stats.entries_scanned; + remembered_slots = remembered_stats.dirty_slots_scanned; } + if let Some(diag) = phase_diag.as_mut() { + diag.record(Phase::RememberedSetYoungLogs, remembered_phase_start); + } + let copy_phase_start = PhaseDiag::start(&phase_diag); unsafe { let _phase = super::pin::CopyingWalkPhaseGuard::enter("worklist_drain"); collector.drain(); } + if let Some(diag) = phase_diag.as_mut() { + diag.record(Phase::CopyEvacuation, copy_phase_start); + } + let rewrite_root_scan_phase_start = PhaseDiag::start(&phase_diag); { let scanners: Vec = if untraced { Vec::new() @@ -1504,6 +1534,9 @@ pub(super) fn run_copied_minor_attempt( } visit_ffi_mutable_registered_roots_with_sources(&mut visitor, root_sources); } + if let Some(diag) = phase_diag.as_mut() { + diag.record(Phase::RootScan, rewrite_root_scan_phase_start); + } // #7803 THE FIX: rebuild the promoted-object remembered set AFTER the last // phase that can move an object, not before the drain. // @@ -1528,6 +1561,7 @@ pub(super) fn run_copied_minor_attempt( // the rebuild performs is exact rather than a from-space // over-approximation. Headers still carry GC_FLAG_MARKED (clear_marks // runs later), which the per-object gate requires. + let forwarding_phase_start = PhaseDiag::start(&phase_diag); if !collector.skip_remembering { let promoted_sticky = rebuild_evacuated_old_to_young_remembered_set(&collector.moved_headers); @@ -1553,6 +1587,9 @@ pub(super) fn run_copied_minor_attempt( super::roots::stack_maps_native_slot_verify(untraced, &|addr| { format!("{:?}", collector.ptrs.classify(addr).map(|ptr| ptr.kind)) }); + if let Some(diag) = phase_diag.as_mut() { + diag.record(Phase::ForwardingFixups, forwarding_phase_start); + } trace_phase_record(trace, "copying_nursery", phase_start); // #7937: the attempt's own trace has finished, so the ratio it was missing @@ -1659,8 +1696,16 @@ pub(super) fn run_copied_minor_attempt( // run here, same window as fromspace_scan (after rewrite, before reset). super::native_stack_scan::run_native_stack_scan(); - crate::promise::cleanup_copied_minor_promise_contexts_for_gc(); - finalize_dead_copied_minor_from_space_side_allocations(); + let finalization = finalize_dead_copied_minor_from_space_side_allocations(); + if let Some(diag) = phase_diag.as_mut() { + diag.add_nanos(Phase::DeadOwnerSideTablePruning, finalization.dead_owner_ns); + diag.add_nanos( + Phase::FromSpaceFinalization, + finalization + .total_ns + .saturating_sub(finalization.dead_owner_ns), + ); + } // #7742: on a promoting cycle the young blocks are handed to old-gen // instead of being reset. This MUST stay before `clear_marks` — the finish // walk reads `GC_FLAG_MARKED` to decide which objects to index — and it @@ -1668,6 +1713,7 @@ pub(super) fn run_copied_minor_attempt( // blocks the reset would recycle are the blocks this keeps. let (reset, promotion_stats) = if promoting_in_place { let phase_start = trace_phase_start(trace); + let promotion_phase_start = PhaseDiag::start(&phase_diag); super::note_promoted_young_capacity(promotion.reserved_bytes()); let promotion_stats = crate::arena::finish_in_place_promotion( promotion, @@ -1677,6 +1723,9 @@ pub(super) fn run_copied_minor_attempt( crate::arena::PromotionLiveness::Marked }, ); + if let Some(diag) = phase_diag.as_mut() { + diag.record(Phase::Promotion, promotion_phase_start); + } trace_phase_record(trace, "in_place_promotion", phase_start); ( crate::arena::ArenaResetStats { @@ -1687,10 +1736,12 @@ pub(super) fn run_copied_minor_attempt( promotion_stats, ) } else { - ( - crate::arena::copying_reset_from_spaces_and_flip(), - crate::arena::InPlacePromotionStats::default(), - ) + let reset_phase_start = PhaseDiag::start(&phase_diag); + let reset = crate::arena::copying_reset_from_spaces_and_flip(); + if let Some(diag) = phase_diag.as_mut() { + diag.record(Phase::BlockResetFlip, reset_phase_start); + } + (reset, crate::arena::InPlacePromotionStats::default()) }; collector.stats.reset_blocks += reset.reset_blocks; if untraced { @@ -1715,6 +1766,7 @@ pub(super) fn run_copied_minor_attempt( if let Some(trace) = trace.as_mut() { trace.old_pages = crate::arena::old_page_summary(); } + let remembered_restore_phase_start = PhaseDiag::start(&phase_diag); remembered_set_clear(); collector.sticky.restore(); if !collector.skip_remembering { @@ -1723,6 +1775,12 @@ pub(super) fn run_copied_minor_attempt( // the last line before the kill is the answer. crate::arena::page_class_table_report(); } + if let Some(diag) = phase_diag.as_mut() { + diag.record( + Phase::RememberedSetYoungLogs, + remembered_restore_phase_start, + ); + } // The mechanism, counted rather than assumed: with the pre-size working, // `capacity` is already >= `len` on entry and hashbrown never grows the // table, so `reserve_rehash` disappears from this path. A capacity that @@ -1923,11 +1981,27 @@ pub(super) fn run_copied_minor_attempt( // This is intentionally the last diagnostic action before returning to // the mutator: `pause_us` prices the whole copied-minor path, including // finalization, pruning, policy feedback and the diagnostic work above. - let pause_us = start.elapsed().as_micros() as u64; + let pause_ns = start.elapsed().as_nanos() as u64; + let pause_us = pause_ns / 1000; + let phases = phase_diag + .as_ref() + .expect("PERRY_GC_DIAG phase accounting must be enabled") + .render( + pause_ns, + scan_us, + collector.stats.copied_objects, + collector.stats.copied_bytes, + collector.stats.promoted_objects, + collector.stats.promoted_bytes, + remembered_entries, + remembered_slots, + &finalization, + ); eprintln!( - "[gc-copy-minor] ran pause_us={} scan_us={} in_place={} untraced={} untraced_cycles={} untraced_objects={} in_place_blocks={} in_place_dead_bytes={} sparse_blocks={} survival_permille={} copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={} tenuring_survivals={} eden_live_bytes={} eden_copied_bytes={} survivor_live_bytes={} survivor_first_round_live_bytes={} trigger={:?} declared_safepoint={}", + "[gc-copy-minor] ran pause_us={} scan_us={} phases: {} in_place={} untraced={} untraced_cycles={} untraced_objects={} in_place_blocks={} in_place_dead_bytes={} sparse_blocks={} survival_permille={} copied_objects={} copied_bytes={} promoted_objects={} promoted_bytes={} freed_bytes={} tenuring_survivals={} eden_live_bytes={} eden_copied_bytes={} survivor_live_bytes={} survivor_first_round_live_bytes={} trigger={:?} declared_safepoint={}", pause_us, scan_us, + phases, collector.stats.in_place_promotion, untraced, super::untraced_promotion_cycles(), @@ -1990,14 +2064,3 @@ fn test_record_cohort_split( pub(super) fn test_last_cohort_split() -> (usize, usize, usize, usize) { LAST_COHORT_SPLIT.with(std::cell::Cell::get) } - -fn finalize_dead_copied_minor_from_space_side_allocations() { - crate::map::finalize_dead_copied_minor_from_space_maps(); - crate::set::finalize_dead_copied_minor_from_space_sets(); - crate::node_submodules::diagnostics_gc::finalize_dead_copied_minor_from_space_errors(); - crate::regex::finalize_dead_copied_minor_from_space_regexps(); - // 2026-07-09 GC audit wave 2: the from-space flip runs no per-object - // finalize hooks, so entries keyed by dead from-space owners in the - // object-address-keyed side tables are pruned here (headers still intact). - super::dead_owner::prune_dead_owner_side_tables_copied_minor(); -} diff --git a/crates/perry-runtime/src/gc/copying_phase.rs b/crates/perry-runtime/src/gc/copying_phase.rs new file mode 100644 index 0000000000..5c693f7005 --- /dev/null +++ b/crates/perry-runtime/src/gc/copying_phase.rs @@ -0,0 +1,204 @@ +//! Diagnostic-only phase accounting for the copying minor. +//! +//! The collector's phases are not all contiguous: registered roots are walked +//! once to evacuate and once to repair forwarding addresses, and in-place +//! promotion has an early retag plus a late finish. The accumulator therefore +//! records non-overlapping spans into semantic buckets. Anything outside a +//! priced span is reported as `other`; top-level buckets plus `other` are an +//! exact partition of the same `Instant` interval used for `pause_us`. + +use std::fmt::Write; +use std::time::Instant; + +#[derive(Clone, Copy)] +pub(super) enum CopyingMinorPhase { + RootScan, + CopyEvacuation, + RememberedSetYoungLogs, + Promotion, + DeadOwnerSideTablePruning, + FromSpaceFinalization, + ForwardingFixups, + BlockResetFlip, +} + +#[derive(Default)] +pub(super) struct CopyingMinorPhaseDiag { + root_scan_ns: u64, + copy_evacuation_ns: u64, + remembered_set_young_logs_ns: u64, + promotion_ns: u64, + dead_owner_side_table_pruning_ns: u64, + from_space_finalization_ns: u64, + forwarding_fixups_ns: u64, + block_reset_flip_ns: u64, +} + +impl CopyingMinorPhaseDiag { + #[inline] + pub(super) fn enabled() -> Option { + super::gc_diag_enabled().then(Self::default) + } + + #[inline] + pub(super) fn start(diag: &Option) -> Option { + diag.as_ref().map(|_| Instant::now()) + } + + #[inline] + pub(super) fn record(&mut self, phase: CopyingMinorPhase, start: Option) { + let Some(start) = start else { + return; + }; + self.add_nanos(phase, start.elapsed().as_nanos() as u64); + } + + #[inline] + pub(super) fn add_nanos(&mut self, phase: CopyingMinorPhase, nanos: u64) { + let slot = match phase { + CopyingMinorPhase::RootScan => &mut self.root_scan_ns, + CopyingMinorPhase::CopyEvacuation => &mut self.copy_evacuation_ns, + CopyingMinorPhase::RememberedSetYoungLogs => &mut self.remembered_set_young_logs_ns, + CopyingMinorPhase::Promotion => &mut self.promotion_ns, + CopyingMinorPhase::DeadOwnerSideTablePruning => { + &mut self.dead_owner_side_table_pruning_ns + } + CopyingMinorPhase::FromSpaceFinalization => &mut self.from_space_finalization_ns, + CopyingMinorPhase::ForwardingFixups => &mut self.forwarding_fixups_ns, + CopyingMinorPhase::BlockResetFlip => &mut self.block_reset_flip_ns, + }; + *slot = slot.saturating_add(nanos); + } + + fn named_nanos(&self) -> u64 { + self.root_scan_ns + .saturating_add(self.copy_evacuation_ns) + .saturating_add(self.remembered_set_young_logs_ns) + .saturating_add(self.promotion_ns) + .saturating_add(self.dead_owner_side_table_pruning_ns) + .saturating_add(self.from_space_finalization_ns) + .saturating_add(self.forwarding_fixups_ns) + .saturating_add(self.block_reset_flip_ns) + } + + pub(super) fn render( + &self, + pause_ns: u64, + scan_us: u64, + copied_objects: usize, + copied_bytes: usize, + promoted_objects: usize, + promoted_bytes: usize, + remembered_entries: usize, + remembered_slots: usize, + finalization: &CopiedMinorFinalizationDiag, + ) -> String { + let named_ns = self.named_nanos(); + let other_ns = pause_ns.saturating_sub(named_ns); + let phase_sum_ns = named_ns.saturating_add(other_ns); + let mut out = String::new(); + write!( + out, + "root_scan={}/{} copy_evacuation={}/{}/{} remembered_set_young_logs={}/{}/{} promotion={}/{}/{} dead_owner_side_table_pruning={}{} from_space_finalization={}/map:{}/{}+set:{}/{}+errors:{}/{}+regex:{}/{} forwarding_fixups={} block_reset_flip={} other={} phase_sum_us={}", + self.root_scan_ns / 1000, + scan_us, + self.copy_evacuation_ns / 1000, + copied_objects, + copied_bytes, + self.remembered_set_young_logs_ns / 1000, + remembered_entries, + remembered_slots, + self.promotion_ns / 1000, + promoted_objects, + promoted_bytes, + self.dead_owner_side_table_pruning_ns / 1000, + finalization.dead_owner_detail, + self.from_space_finalization_ns / 1000, + finalization.map_ns / 1000, + finalization.maps, + finalization.set_ns / 1000, + finalization.sets, + finalization.errors_ns / 1000, + finalization.errors, + finalization.regex_ns / 1000, + finalization.regexps, + self.forwarding_fixups_ns / 1000, + self.block_reset_flip_ns / 1000, + other_ns / 1000, + phase_sum_ns / 1000, + ) + .expect("writing phase diagnostics to a String cannot fail"); + out + } +} + +#[derive(Default)] +pub(super) struct CopiedMinorFinalizationDiag { + pub(super) total_ns: u64, + pub(super) map_ns: u64, + pub(super) maps: usize, + pub(super) set_ns: u64, + pub(super) sets: usize, + pub(super) errors_ns: u64, + pub(super) errors: usize, + pub(super) regex_ns: u64, + pub(super) regexps: usize, + pub(super) dead_owner_ns: u64, + pub(super) dead_owner_detail: String, +} + +/// Finalize the side allocations whose from-space owners just died. The +/// clocks live here, beside the calls they price; without `PERRY_GC_DIAG` this +/// performs the original calls without reading the clock or building strings. +pub(super) fn finalize_dead_copied_minor_from_space_side_allocations() -> CopiedMinorFinalizationDiag +{ + let diag = super::gc_diag_enabled(); + let total_start = diag.then(Instant::now); + let mut out = CopiedMinorFinalizationDiag::default(); + + crate::promise::cleanup_copied_minor_promise_contexts_for_gc(); + + let start = diag.then(Instant::now); + out.maps = crate::map::finalize_dead_copied_minor_from_space_maps(); + out.map_ns = start.map_or(0, |start| start.elapsed().as_nanos() as u64); + + let start = diag.then(Instant::now); + out.sets = crate::set::finalize_dead_copied_minor_from_space_sets(); + out.set_ns = start.map_or(0, |start| start.elapsed().as_nanos() as u64); + + let start = diag.then(Instant::now); + out.errors = + crate::node_submodules::diagnostics_gc::finalize_dead_copied_minor_from_space_errors(); + out.errors_ns = start.map_or(0, |start| start.elapsed().as_nanos() as u64); + + let start = diag.then(Instant::now); + out.regexps = crate::regex::finalize_dead_copied_minor_from_space_regexps(); + out.regex_ns = start.map_or(0, |start| start.elapsed().as_nanos() as u64); + + let start = diag.then(Instant::now); + out.dead_owner_detail = super::dead_owner::prune_dead_owner_side_tables_copied_minor(); + out.dead_owner_ns = start.map_or(0, |start| start.elapsed().as_nanos() as u64); + out.total_ns = total_start.map_or(0, |start| start.elapsed().as_nanos() as u64); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn copied_minor_phase_residual_makes_the_partition_exact() { + let mut diag = CopyingMinorPhaseDiag::default(); + diag.add_nanos(CopyingMinorPhase::RootScan, 11_000); + diag.add_nanos(CopyingMinorPhase::CopyEvacuation, 7_000); + diag.add_nanos(CopyingMinorPhase::BlockResetFlip, 3_000); + + let pause_ns: u64 = 29_000; + let other_ns = pause_ns.saturating_sub(diag.named_nanos()); + assert_eq!(diag.named_nanos() + other_ns, pause_ns); + assert_eq!(other_ns, 8_000); + // Sabotage: remove one named bucket from `named_nanos`; this exact + // residual assertion changes and the test fails rather than merely + // checking that phase reporting did not panic. + } +} diff --git a/crates/perry-runtime/src/gc/dead_owner.rs b/crates/perry-runtime/src/gc/dead_owner.rs index 33e2884256..2eeff26bbe 100644 --- a/crates/perry-runtime/src/gc/dead_owner.rs +++ b/crates/perry-runtime/src/gc/dead_owner.rs @@ -241,6 +241,7 @@ pub(super) fn prune_dead_owner_side_tables_post_trace( &|addr| probe.owner_is_dead(addr, Some(GC_TYPE_CLOSURE)), &|addr| probe.owner_is_dead(addr, Some(GC_TYPE_STRING)), /* young_only = */ !full_trace, + None, ); // #6182: drop dead weak-target HOLDERS (WeakRef / FinalizationRegistry / // WeakMap-WeakSet entry — all GC_TYPE_OBJECT) from the registry so the @@ -257,13 +258,23 @@ pub(super) fn prune_dead_owner_side_tables_post_trace( /// Copied-minor fan-out: prune entries owned by dead from-space objects /// before the flip destroys their headers. Nursery-only by construction, so /// the tenured/malloc caveat cannot mis-fire here. -pub(super) fn prune_dead_owner_side_tables_copied_minor() { +pub(super) fn prune_dead_owner_side_tables_copied_minor() -> String { + let mut detail = String::new(); + let diag = super::gc_diag_enabled(); + if diag { + detail.push('['); + } fan_out( &|addr| owner_is_dead_copied_minor_from_space(addr, None), &|addr| owner_is_dead_copied_minor_from_space(addr, Some(GC_TYPE_CLOSURE)), &|addr| owner_is_dead_copied_minor_from_space(addr, Some(GC_TYPE_STRING)), /* young_only = */ true, + diag.then_some(&mut detail), ); + if diag { + detail.push(']'); + } + detail } /// Which of the pass's three deadness predicates a registered prune is handed. @@ -502,6 +513,7 @@ fn fan_out( is_dead_closure: &dyn Fn(usize) -> bool, is_dead_symbol: &dyn Fn(usize) -> bool, young_only: bool, + mut diag: Option<&mut String>, ) { // Interned key pointers cached in the store-plan cache may die in this // collection — flush every cached verdict. Pointer identity only: the @@ -515,9 +527,15 @@ fn fan_out( DeadKeyOwner::Closure => is_dead_closure, DeadKeyOwner::Symbol => is_dead_symbol, }; + let start = diag.as_ref().map(|_| std::time::Instant::now()); match entry.young_prune { Some(young_prune) if young_only => young_prune(is_dead), _ => (entry.prune)(is_dead), } + if let (Some(detail), Some(start)) = (diag.as_deref_mut(), start) { + use std::fmt::Write; + write!(detail, " {:?}:{}", entry.table, start.elapsed().as_micros()) + .expect("writing dead-owner diagnostics to a String cannot fail"); + } } } diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 8ef7ea0294..48de6e37cc 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -157,6 +157,7 @@ mod prefetch; mod copying; mod copying_first_cycle; +mod copying_phase; mod copying_pointer_set; mod diag_sites; pub(crate) use diag_sites::primitive_dispatch as diag_primitive_dispatch; diff --git a/crates/perry-runtime/src/node_submodules/diagnostics_gc.rs b/crates/perry-runtime/src/node_submodules/diagnostics_gc.rs index 8b68e16e92..511a1999ca 100644 --- a/crates/perry-runtime/src/node_submodules/diagnostics_gc.rs +++ b/crates/perry-runtime/src/node_submodules/diagnostics_gc.rs @@ -46,7 +46,7 @@ pub(crate) fn error_side_tables_clear_dead(user_ptr: usize) { /// key is a dead from-space error — unmarked, unforwarded, nursery-space, /// still typed `GC_TYPE_ERROR`. Mirrors /// `finalize_dead_copied_minor_from_space_maps`. -pub(crate) fn finalize_dead_copied_minor_from_space_errors() { +pub(crate) fn finalize_dead_copied_minor_from_space_errors() -> usize { fn is_dead_from_space_error(addr: usize) -> bool { let space = crate::arena::classify_heap_space(addr); if !matches!(space, crate::arena::HeapSpace::NurseryEden) @@ -73,7 +73,9 @@ pub(crate) fn finalize_dead_copied_minor_from_space_errors() { .filter(|addr| is_dead_from_space_error(*addr)) .collect() }); + let count = dead.len(); for addr in dead { error_side_tables_clear_dead(addr); } + count } From 357f8a564ec77b285b86a9fdb7ebd4de13b25c1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 11:57:41 +0200 Subject: [PATCH 19/27] perf(gc): log young roots for remaining minor scanners Make descriptor, closure metadata, template, array named-property, and symbol side-table minor scans proportional to entries that can still move, die, or expose a young strong edge. Keep full scans authoritative and use test-only re-derivation sabotage checks to enforce every write funnel. Narrow descriptor and closure owner retention to collectible metadata keys; long-lived values remain logged only where their transitive edges require it. (cherry picked from commit dae5192966b6f41ffbdc3ad655a3e1ae931c7b65) --- crates/perry-runtime/src/array/header.rs | 64 +-- .../src/array/header/young_roots.rs | 302 +++++++++++++ .../src/closure/dynamic_props.rs | 46 +- .../src/object/descriptor_state.rs | 48 +- .../src/object/descriptor_state/gc_scan.rs | 4 +- .../src/object/descriptor_state/young.rs | 8 +- .../object/native_module/callable_exports.rs | 103 +---- .../builtin_closure_metadata.rs | 201 +++++++++ crates/perry-runtime/src/symbol.rs | 3 + crates/perry-runtime/src/symbol/accessors.rs | 47 +- crates/perry-runtime/src/symbol/gc_roots.rs | 411 ++++++++++++++++-- crates/perry-runtime/src/symbol/properties.rs | 1 + 12 files changed, 1027 insertions(+), 211 deletions(-) create mode 100644 crates/perry-runtime/src/array/header/young_roots.rs create mode 100644 crates/perry-runtime/src/object/native_module/callable_exports/builtin_closure_metadata.rs diff --git a/crates/perry-runtime/src/array/header.rs b/crates/perry-runtime/src/array/header.rs index ec23d3aa85..63e1e059b5 100644 --- a/crates/perry-runtime/src/array/header.rs +++ b/crates/perry-runtime/src/array/header.rs @@ -4,6 +4,10 @@ pub(crate) use super::header_gc_slots::*; +mod young_roots; +pub use young_roots::scan_template_raw_roots_mut; +use young_roots::{note_array_named, note_template_cache, note_template_raw}; + use std::cell::RefCell; use std::collections::HashMap; @@ -184,6 +188,7 @@ unsafe fn register_template_raw_pair(cooked: *mut ArrayHeader, raw: *mut ArrayHe if cooked.is_null() || raw.is_null() { return; } + note_template_raw(cooked as usize, raw); TEMPLATE_RAW_MAP.with(|m| { m.borrow_mut().insert(cooked as usize, raw); }); @@ -254,6 +259,7 @@ pub extern "C" fn js_tagged_template_get_or_init( mark_template_array_frozen(raw); mark_template_array_frozen(cooked); register_template_raw_pair(cooked, raw); + note_template_cache(site_id, cooked, raw); TEMPLATE_OBJECT_CACHE.with(|m| { m.borrow_mut().insert(site_id, (cooked, raw)); }); @@ -294,33 +300,6 @@ pub fn scan_template_raw_roots(mark: &mut dyn FnMut(f64)) { scan_template_raw_roots_mut(&mut visitor); } -pub fn scan_template_raw_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - TEMPLATE_OBJECT_CACHE.with(|m| { - let mut map = m.borrow_mut(); - for (_, (cooked_ptr, raw_ptr)) in map.iter_mut() { - visitor.visit_raw_mut_ptr_slot(cooked_ptr); - visitor.visit_raw_mut_ptr_slot(raw_ptr); - } - }); - TEMPLATE_RAW_MAP.with(|m| { - let mut map = m.borrow_mut(); - let mut moved = Vec::new(); - for (&cooked_addr, raw_ptr) in map.iter_mut() { - let mut new_cooked_addr = cooked_addr; - if visitor.visit_usize_slot(&mut new_cooked_addr) { - moved.push((cooked_addr, new_cooked_addr)); - } - visitor.visit_raw_mut_ptr_slot(raw_ptr); - } - for (old_addr, new_addr) in moved { - if let Some(raw_ptr) = map.remove(&old_addr) { - map.insert(new_addr, raw_ptr); - } - } - }); - scan_array_named_property_roots_mut(visitor); -} - fn barrier_array_named_props(owner: usize, props: &mut [ArrayNamedProperty]) { for prop in props.iter_mut() { crate::gc::runtime_write_barrier_external_slot( @@ -363,28 +342,10 @@ pub(crate) fn transfer_array_named_property_owner(old_owner: usize, new_owner: u ARRAY_NAMED_PROPS.with(|m| { let mut props = m.borrow_mut(); if let Some(old_props) = props.remove(&old_owner) { - merge_array_named_props(&mut props, new_owner, old_props); - } - }); -} - -pub(crate) fn scan_array_named_property_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - ARRAY_NAMED_PROPS.with(|m| { - let mut props = m.borrow_mut(); - let mut moved = Vec::new(); - for (&owner, owner_props) in props.iter_mut() { - let mut new_owner = owner; - if visitor.visit_metadata_usize_slot(&mut new_owner) { - moved.push((owner, new_owner)); - } - for prop in owner_props.iter_mut() { - visitor.visit_nanbox_f64_slot(&mut prop.value); - } - } - for (old_owner, new_owner) in moved { - if let Some(old_props) = props.remove(&old_owner) { - merge_array_named_props(&mut props, new_owner, old_props); + for prop in &old_props { + note_array_named(new_owner, prop.value.to_bits()); } + merge_array_named_props(&mut props, new_owner, old_props); } }); } @@ -405,6 +366,7 @@ pub(crate) fn test_array_named_property_owner_exists(owner: usize) -> bool { #[cfg(test)] pub(crate) fn test_clear_array_named_property_roots() { ARRAY_NAMED_PROPS.with(|m| m.borrow_mut().clear()); + young_roots::clear_named_log(); } unsafe fn string_header_as_str<'a>(key: *const crate::StringHeader) -> Option<&'a str> { @@ -445,6 +407,7 @@ pub(crate) unsafe fn array_named_property_set( }; let owner = arr as usize; note_array_named_props_ever(); + note_array_named(owner, value.to_bits()); ARRAY_NAMED_PROPS.with(|m| { let mut map = m.borrow_mut(); let props = map.entry(owner).or_default(); @@ -478,6 +441,10 @@ pub(crate) unsafe fn array_named_props_install_fresh( return; } let owner = arr as usize; + note_array_named_props_ever(); + for (_, value) in entries { + note_array_named(owner, value.to_bits()); + } ARRAY_NAMED_PROPS.with(|m| { let mut map = m.borrow_mut(); let props = map.entry(owner).or_default(); @@ -634,6 +601,7 @@ pub(crate) unsafe fn array_named_property_delete_by_name( #[cfg(test)] pub(crate) fn test_seed_template_raw_roots(cooked: *mut ArrayHeader, raw: *mut ArrayHeader) { + note_template_raw(cooked as usize, raw); TEMPLATE_RAW_MAP.with(|m| { let mut m = m.borrow_mut(); m.clear(); diff --git a/crates/perry-runtime/src/array/header/young_roots.rs b/crates/perry-runtime/src/array/header/young_roots.rs new file mode 100644 index 0000000000..946e428c0f --- /dev/null +++ b/crates/perry-runtime/src/array/header/young_roots.rs @@ -0,0 +1,302 @@ +//! Young-entry logs for tagged-template and array named-property roots. + +use super::*; + +crate::perry_thread_local! { + static TEMPLATE_CACHE_YOUNG: RefCell> = + const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; + static TEMPLATE_RAW_YOUNG: RefCell> = + const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; + static ARRAY_NAMED_YOUNG: RefCell> = + const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; + #[cfg(test)] + static TEST_SUPPRESS_ARRAY_NAMED_YOUNG_NOTE: std::cell::Cell = + const { std::cell::Cell::new(false) }; + #[cfg(test)] + static TEST_SUPPRESS_TEMPLATE_RAW_YOUNG_NOTE: std::cell::Cell = + const { std::cell::Cell::new(false) }; +} + +const CACHE_LOG: &str = "array.template_object_cache"; +const RAW_LOG: &str = "array.template_raw_map"; +const NAMED_LOG: &str = "array.named_properties"; + +#[inline] +fn ptr_relevant(ptr: *mut ArrayHeader) -> bool { + crate::gc::young_log::addr_is_minor_relevant(ptr as usize) +} + +pub(super) fn note_template_cache(site: u64, cooked: *mut ArrayHeader, raw: *mut ArrayHeader) { + if ptr_relevant(cooked) || ptr_relevant(raw) { + TEMPLATE_CACHE_YOUNG.with(|log| log.borrow_mut().note(site)); + } +} + +pub(super) fn note_template_raw(cooked: usize, raw: *mut ArrayHeader) { + if crate::gc::young_log::addr_is_minor_relevant(cooked) || ptr_relevant(raw) { + #[cfg(test)] + if TEST_SUPPRESS_TEMPLATE_RAW_YOUNG_NOTE.with(std::cell::Cell::get) { + return; + } + TEMPLATE_RAW_YOUNG.with(|log| log.borrow_mut().note(cooked)); + } +} + +pub(super) fn note_array_named(owner: usize, value_bits: u64) { + if !crate::gc::young_log::addr_is_minor_collectible(owner) + && !crate::gc::young_log::bits_are_minor_relevant(value_bits) + { + return; + } + #[cfg(test)] + if TEST_SUPPRESS_ARRAY_NAMED_YOUNG_NOTE.with(std::cell::Cell::get) { + return; + } + ARRAY_NAMED_YOUNG.with(|log| log.borrow_mut().note(owner)); +} + +fn visit_cache_site(visitor: &mut crate::gc::RuntimeRootVisitor<'_>, site: u64) -> bool { + TEMPLATE_OBJECT_CACHE.with(|m| { + let mut map = m.borrow_mut(); + let Some((cooked, raw)) = map.get_mut(&site) else { + return false; + }; + visitor.visit_raw_mut_ptr_slot(cooked); + visitor.visit_raw_mut_ptr_slot(raw); + ptr_relevant(*cooked) || ptr_relevant(*raw) + }) +} + +fn visit_raw_owner(visitor: &mut crate::gc::RuntimeRootVisitor<'_>, owner: usize) -> Option { + TEMPLATE_RAW_MAP.with(|m| { + let mut map = m.borrow_mut(); + let mut raw = map.remove(&owner)?; + let mut new_owner = owner; + visitor.visit_usize_slot(&mut new_owner); + visitor.visit_raw_mut_ptr_slot(&mut raw); + map.insert(new_owner, raw); + (crate::gc::young_log::addr_is_minor_relevant(new_owner) || ptr_relevant(raw)) + .then_some(new_owner) + }) +} + +fn visit_named_owner( + visitor: &mut crate::gc::RuntimeRootVisitor<'_>, + owner: usize, +) -> Option { + ARRAY_NAMED_PROPS.with(|m| { + let mut map = m.borrow_mut(); + let mut props = map.remove(&owner)?; + let mut new_owner = owner; + visitor.visit_metadata_usize_slot(&mut new_owner); + let mut relevant = crate::gc::young_log::addr_is_minor_collectible(new_owner); + for prop in &mut props { + visitor.visit_nanbox_f64_slot(&mut prop.value); + relevant |= crate::gc::young_log::bits_are_minor_relevant(prop.value.to_bits()); + } + merge_array_named_props(&mut map, new_owner, props); + relevant.then_some(new_owner) + }) +} + +#[cfg(any(debug_assertions, test))] +fn relevant_cache_sites() -> Vec { + TEMPLATE_OBJECT_CACHE.with(|m| { + m.borrow() + .iter() + .filter_map(|(&site, &(cooked, raw))| { + (ptr_relevant(cooked) || ptr_relevant(raw)).then_some(site) + }) + .collect() + }) +} + +#[cfg(any(debug_assertions, test))] +fn relevant_raw_owners() -> Vec { + TEMPLATE_RAW_MAP.with(|m| { + m.borrow() + .iter() + .filter_map(|(&owner, &raw)| { + (crate::gc::young_log::addr_is_minor_relevant(owner) || ptr_relevant(raw)) + .then_some(owner) + }) + .collect() + }) +} + +#[cfg(any(debug_assertions, test))] +fn relevant_named_owners() -> Vec { + ARRAY_NAMED_PROPS.with(|m| { + m.borrow() + .iter() + .filter_map(|(&owner, props)| { + (crate::gc::young_log::addr_is_minor_collectible(owner) + || props.iter().any(|prop| { + crate::gc::young_log::bits_are_minor_relevant(prop.value.to_bits()) + })) + .then_some(owner) + }) + .collect() + }) +} + +fn drain_log( + log: &'static crate::tls_hot::HotKey>>, + mut visit: impl FnMut(K) -> Option, +) -> (u64, u64, u64) { + let mut logged = 0; + let mut visited = 0; + let mut kept = log.with(|log| log.borrow_mut().take_spare()); + loop { + let batch = log.with(|log| log.borrow_mut().take_sorted()); + if batch.is_empty() { + break; + } + logged += batch.len() as u64; + for key in batch { + visited += 1; + if let Some(key) = visit(key) { + kept.push(key); + } + } + } + let kept_len = kept.len() as u64; + log.with(|log| log.borrow_mut().extend(kept)); + (logged, visited, kept_len) +} + +fn report(name: &'static str, partial: bool, row: (u64, u64, u64), table_len: usize) { + crate::gc::young_log::note_walk( + name, + crate::gc::young_log::YoungLogWalk { + partial, + logged: row.0, + visited: row.1, + kept: row.2, + table_len: table_len as u64, + }, + ); +} + +pub fn scan_template_raw_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + let cache_len = TEMPLATE_OBJECT_CACHE.with(|m| m.borrow().len()); + let raw_len = TEMPLATE_RAW_MAP.with(|m| m.borrow().len()); + let named_len = ARRAY_NAMED_PROPS.with(|m| m.borrow().len()); + if visitor.young_scope() { + #[cfg(any(debug_assertions, test))] + { + TEMPLATE_CACHE_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(CACHE_LOG, &relevant_cache_sites()) + }); + TEMPLATE_RAW_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(RAW_LOG, &relevant_raw_owners()) + }); + ARRAY_NAMED_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(NAMED_LOG, &relevant_named_owners()) + }); + } + let cache = drain_log(&TEMPLATE_CACHE_YOUNG, |site| { + visit_cache_site(visitor, site).then_some(site) + }); + let raw = drain_log(&TEMPLATE_RAW_YOUNG, |owner| visit_raw_owner(visitor, owner)); + let named = drain_log(&ARRAY_NAMED_YOUNG, |owner| { + visit_named_owner(visitor, owner) + }); + report(CACHE_LOG, true, cache, cache_len); + report(RAW_LOG, true, raw, raw_len); + report(NAMED_LOG, true, named, named_len); + return; + } + + let cache_sites: Vec = + TEMPLATE_OBJECT_CACHE.with(|m| m.borrow().keys().copied().collect()); + let raw_owners: Vec = TEMPLATE_RAW_MAP.with(|m| m.borrow().keys().copied().collect()); + let named_owners: Vec = ARRAY_NAMED_PROPS.with(|m| m.borrow().keys().copied().collect()); + let _ = TEMPLATE_CACHE_YOUNG.with(|log| log.borrow_mut().take_sorted()); + let _ = TEMPLATE_RAW_YOUNG.with(|log| log.borrow_mut().take_sorted()); + let _ = ARRAY_NAMED_YOUNG.with(|log| log.borrow_mut().take_sorted()); + // `drain_log` consumes the log, so seed it with the authoritative keys. + TEMPLATE_CACHE_YOUNG.with(|log| log.borrow_mut().extend(cache_sites)); + let cache = drain_log(&TEMPLATE_CACHE_YOUNG, |site| { + visit_cache_site(visitor, site).then_some(site) + }); + TEMPLATE_RAW_YOUNG.with(|log| log.borrow_mut().extend(raw_owners)); + let raw = drain_log(&TEMPLATE_RAW_YOUNG, |owner| visit_raw_owner(visitor, owner)); + ARRAY_NAMED_YOUNG.with(|log| log.borrow_mut().extend(named_owners)); + let named = drain_log(&ARRAY_NAMED_YOUNG, |owner| { + visit_named_owner(visitor, owner) + }); + report(CACHE_LOG, false, cache, cache_len); + report(RAW_LOG, false, raw, raw_len); + report(NAMED_LOG, false, named, named_len); +} + +#[cfg(test)] +pub(super) fn clear_named_log() { + ARRAY_NAMED_YOUNG.with(|log| log.borrow_mut().clear()); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn alloc_empty_array() -> *mut ArrayHeader { + let arr = crate::arena::arena_alloc_gc( + std::mem::size_of::(), + std::mem::align_of::(), + crate::gc::GC_TYPE_ARRAY, + ) as *mut ArrayHeader; + unsafe { + (*arr).length = 0; + (*arr).capacity = 0; + } + arr + } + + #[test] + fn template_raw_log_rederivation_rejects_a_suppressed_writer() { + let _lock = crate::gc::global_side_table_test_lock(); + let cooked = alloc_empty_array(); + let raw = alloc_empty_array(); + TEST_SUPPRESS_TEMPLATE_RAW_YOUNG_NOTE.with(|flag| flag.set(true)); + test_seed_template_raw_roots(cooked, raw); + TEST_SUPPRESS_TEMPLATE_RAW_YOUNG_NOTE.with(|flag| flag.set(false)); + let missed = std::panic::catch_unwind(|| { + TEMPLATE_RAW_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(RAW_LOG, &relevant_raw_owners()) + }); + }); + TEMPLATE_RAW_MAP.with(|m| m.borrow_mut().clear()); + TEMPLATE_RAW_YOUNG.with(|log| log.borrow_mut().clear()); + assert!( + missed.is_err(), + "sabotage: suppressing the template-raw writer's note must trip completeness" + ); + } + + #[test] + fn array_named_log_rederivation_rejects_a_suppressed_setter() { + let _lock = crate::gc::global_side_table_test_lock(); + let arr = alloc_empty_array(); + let key = crate::string::js_string_from_bytes(b"sabotage".as_ptr(), 8); + TEST_SUPPRESS_ARRAY_NAMED_YOUNG_NOTE.with(|flag| flag.set(true)); + unsafe { array_named_property_set(arr, key, 7.0) }; + TEST_SUPPRESS_ARRAY_NAMED_YOUNG_NOTE.with(|flag| flag.set(false)); + let missed = std::panic::catch_unwind(|| { + ARRAY_NAMED_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(NAMED_LOG, &relevant_named_owners()) + }); + }); + ARRAY_NAMED_PROPS.with(|m| m.borrow_mut().remove(&(arr as usize))); + clear_named_log(); + assert!( + missed.is_err(), + "sabotage: suppressing array_named_property_set's note must trip completeness" + ); + } +} diff --git a/crates/perry-runtime/src/closure/dynamic_props.rs b/crates/perry-runtime/src/closure/dynamic_props.rs index d3ea542188..06e760bd7c 100644 --- a/crates/perry-runtime/src/closure/dynamic_props.rs +++ b/crates/perry-runtime/src/closure/dynamic_props.rs @@ -104,6 +104,9 @@ crate::perry_thread_local! { /// thread's minors can move or free them. See `gc/young_log.rs`. static CLOSURE_YOUNG_OWNERS: std::cell::RefCell> = const { std::cell::RefCell::new(crate::gc::young_log::YoungLog::new()) }; + #[cfg(test)] + static TEST_SUPPRESS_CLOSURE_YOUNG_NOTE: std::cell::Cell = + const { std::cell::Cell::new(false) }; } const CLOSURE_YOUNG_LOG_NAME: &str = "closure.dynamic_props"; @@ -112,13 +115,38 @@ const CLOSURE_YOUNG_LOG_NAME: &str = "closure.dynamic_props"; /// when the owner or the value being stored can matter to a minor. #[inline] fn note_young_closure_owner(owner: usize, value_bits: u64) { - if crate::gc::young_log::addr_is_minor_relevant(owner) + if crate::gc::young_log::addr_is_minor_collectible(owner) || crate::gc::young_log::bits_are_minor_relevant(value_bits) { + #[cfg(test)] + if TEST_SUPPRESS_CLOSURE_YOUNG_NOTE.with(std::cell::Cell::get) { + return; + } CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().note(owner)); } } +#[cfg(test)] +mod young_log_sabotage_tests { + use super::*; + + #[test] + fn closure_log_rederivation_rejects_a_suppressed_setter() { + let _lock = crate::gc::global_side_table_test_lock(); + test_clear_closure_side_tables(); + let owner = crate::closure::js_closure_alloc(std::ptr::null(), 0) as usize; + TEST_SUPPRESS_CLOSURE_YOUNG_NOTE.with(|flag| flag.set(true)); + closure_set_dynamic_prop(owner, "sabotage", 7.0); + TEST_SUPPRESS_CLOSURE_YOUNG_NOTE.with(|flag| flag.set(false)); + let missed = std::panic::catch_unwind(debug_assert_closure_young_log_complete); + test_clear_closure_side_tables(); + assert!( + missed.is_err(), + "sabotage: suppressing closure_set_dynamic_prop's note must trip completeness" + ); + } +} + /// A re-keyed entry keeps whatever values it had, so the new owner is logged /// unconditionally; the next minor-scoped walk drops it if nothing in it is /// relevant any more. @@ -515,7 +543,7 @@ fn scan_closure_side_tables_young(visitor: &mut crate::gc::RuntimeRootVisitor<'_ .unwrap_or(0); (props + prototypes + deleted) as u64 }; - #[cfg(debug_assertions)] + #[cfg(any(debug_assertions, test))] debug_assert_closure_young_log_complete(); let mut logged = 0u64; let mut visited = 0u64; @@ -550,13 +578,13 @@ fn scan_closure_side_tables_young(visitor: &mut crate::gc::RuntimeRootVisitor<'_ /// Rule 2 of `gc/young_log.rs`: re-derive the relevant owners from the three /// tables and require the log to name each one. -#[cfg(debug_assertions)] +#[cfg(any(debug_assertions, test))] fn debug_assert_closure_young_log_complete() { - use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; + use crate::gc::young_log::{addr_is_minor_collectible, bits_are_minor_relevant}; let mut relevant = Vec::new(); if let Ok(props) = get_closure_props().lock() { for (&owner, entry) in props.iter() { - if addr_is_minor_relevant(owner) + if addr_is_minor_collectible(owner) || entry .values .values() @@ -568,14 +596,14 @@ fn debug_assert_closure_young_log_complete() { } if let Ok(prototypes) = get_closure_prototypes().lock() { for (&owner, &proto_bits) in prototypes.iter() { - if addr_is_minor_relevant(owner) || bits_are_minor_relevant(proto_bits) { + if addr_is_minor_collectible(owner) || bits_are_minor_relevant(proto_bits) { relevant.push(owner); } } } if let Ok(deleted) = get_closure_deleted_keys().lock() { for &owner in deleted.keys() { - if addr_is_minor_relevant(owner) { + if addr_is_minor_collectible(owner) { relevant.push(owner); } } @@ -593,7 +621,7 @@ fn scan_closure_owner( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, owner: usize, ) -> (usize, bool) { - use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; + use crate::gc::young_log::{addr_is_minor_collectible, bits_are_minor_relevant}; let mut relevant = false; let mut current_owner = owner; @@ -657,7 +685,7 @@ fn scan_closure_owner( } } - relevant |= addr_is_minor_relevant(current_owner); + relevant |= addr_is_minor_collectible(current_owner); (current_owner, relevant) } diff --git a/crates/perry-runtime/src/object/descriptor_state.rs b/crates/perry-runtime/src/object/descriptor_state.rs index 31d0a54ce4..21966ee59e 100644 --- a/crates/perry-runtime/src/object/descriptor_state.rs +++ b/crates/perry-runtime/src/object/descriptor_state.rs @@ -130,6 +130,11 @@ impl DescriptorTables { const DESCRIPTOR_YOUNG_LOG_NAME: &str = "object.descriptors"; +#[cfg(test)] +thread_local! { + static TEST_SUPPRESS_DESCRIPTOR_YOUNG_NOTE: Cell = const { Cell::new(false) }; +} + mod gc_scan; mod young; pub(crate) use gc_scan::{scan_descriptor_owner, scan_descriptor_roots_mut}; @@ -145,15 +150,54 @@ fn note_young_descriptor_owner( owner: usize, acc: Option<&AccessorDescriptor>, ) { - use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; - if addr_is_minor_relevant(owner) + use crate::gc::young_log::{addr_is_minor_collectible, bits_are_minor_relevant}; + if addr_is_minor_collectible(owner) || acc .is_some_and(|acc| bits_are_minor_relevant(acc.get) || bits_are_minor_relevant(acc.set)) { + #[cfg(test)] + if TEST_SUPPRESS_DESCRIPTOR_YOUNG_NOTE.with(Cell::get) { + return; + } st.descriptors.young_owners.borrow_mut().note(owner); } } +#[cfg(test)] +mod young_log_sabotage_tests { + use super::*; + + #[test] + fn descriptor_log_rederivation_rejects_a_suppressed_setter() { + let _lock = crate::gc::global_side_table_test_lock(); + let owner = crate::object::js_object_alloc(0, 0) as usize; + state().descriptors.young_owners.borrow_mut().clear(); + TEST_SUPPRESS_DESCRIPTOR_YOUNG_NOTE.with(|flag| flag.set(true)); + set_property_attrs( + owner, + "sabotage".to_string(), + PropertyAttrs::new(true, true, true), + ); + TEST_SUPPRESS_DESCRIPTOR_YOUNG_NOTE.with(|flag| flag.set(false)); + let missed = std::panic::catch_unwind(|| { + state() + .descriptors + .young_owners + .borrow() + .debug_assert_logged( + DESCRIPTOR_YOUNG_LOG_NAME, + &relevant_descriptor_owners(state()), + ); + }); + clear_property_attrs(owner, "sabotage"); + state().descriptors.young_owners.borrow_mut().clear(); + assert!( + missed.is_err(), + "sabotage: suppressing set_property_attrs' note must trip completeness" + ); + } +} + /// Record `key` as owned by `owner` in an owner index. Idempotent: a /// `defineProperty` that overwrites an existing descriptor must not push a /// duplicate, or the key would be reported twice by `Object.keys`. diff --git a/crates/perry-runtime/src/object/descriptor_state/gc_scan.rs b/crates/perry-runtime/src/object/descriptor_state/gc_scan.rs index fe0f22ca36..48471ca672 100644 --- a/crates/perry-runtime/src/object/descriptor_state/gc_scan.rs +++ b/crates/perry-runtime/src/object/descriptor_state/gc_scan.rs @@ -137,7 +137,7 @@ pub(crate) fn scan_descriptor_owner( st: &crate::state::RuntimeState, owner: usize, ) -> (usize, bool) { - use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; + use crate::gc::young_log::{addr_is_minor_collectible, bits_are_minor_relevant}; let new_owner = rewrite_descriptor_owner(visitor, owner); let mut relevant = false; let accessor_keys = st @@ -187,7 +187,7 @@ pub(crate) fn scan_descriptor_owner( owner_index_transfer(&st.descriptors.attr_keys_by_owner, owner, new_owner); owner_index_transfer(&st.descriptors.accessor_keys_by_owner, owner, new_owner); } - relevant |= addr_is_minor_relevant(new_owner); + relevant |= addr_is_minor_collectible(new_owner); (new_owner, relevant) } diff --git a/crates/perry-runtime/src/object/descriptor_state/young.rs b/crates/perry-runtime/src/object/descriptor_state/young.rs index 0133fc7549..631abbc548 100644 --- a/crates/perry-runtime/src/object/descriptor_state/young.rs +++ b/crates/perry-runtime/src/object/descriptor_state/young.rs @@ -13,15 +13,15 @@ use super::*; /// authoritative tables: a non-old owner, or an accessor whose getter or /// setter is non-old. pub(super) fn relevant_descriptor_owners(st: &crate::state::RuntimeState) -> Vec { - use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; + use crate::gc::young_log::{addr_is_minor_collectible, bits_are_minor_relevant}; let mut relevant = Vec::new(); for &owner in st.descriptors.attr_keys_by_owner.borrow().keys() { - if addr_is_minor_relevant(owner) { + if addr_is_minor_collectible(owner) { relevant.push(owner); } } for &owner in st.descriptors.accessor_keys_by_owner.borrow().keys() { - if addr_is_minor_relevant(owner) { + if addr_is_minor_collectible(owner) { relevant.push(owner); } } @@ -45,7 +45,7 @@ pub(super) fn scan_descriptor_roots_young( ) { let table_len = st.descriptors.attr_keys_by_owner.borrow().len() as u64 + st.descriptors.accessor_keys_by_owner.borrow().len() as u64; - #[cfg(debug_assertions)] + #[cfg(any(debug_assertions, test))] { let relevant = relevant_descriptor_owners(st); st.descriptors diff --git a/crates/perry-runtime/src/object/native_module/callable_exports.rs b/crates/perry-runtime/src/object/native_module/callable_exports.rs index ea47d3ea81..431382981a 100644 --- a/crates/perry-runtime/src/object/native_module/callable_exports.rs +++ b/crates/perry-runtime/src/object/native_module/callable_exports.rs @@ -1,6 +1,8 @@ use super::callable_export_arity_table::native_callable_export_arity; use super::*; +mod builtin_closure_metadata; mod module_cjs; +pub(crate) use builtin_closure_metadata::*; use module_cjs::attach_module_cjs_constructor_statics; pub(crate) use module_cjs::{ module_builtin_modules_value, module_cjs_cache_value, module_cjs_extensions_value, @@ -1483,107 +1485,6 @@ pub(crate) fn set_bound_native_closure_name( ); } -thread_local! { - /// Per-closure spec `.length` for built-in *prototype methods*. Those - /// methods all share one no-op closure thunk - /// (`global_this_builtin_noop_thunk`), so the func-ptr-keyed - /// the closure body registry can't give `Array.prototype.map.length === 1` - /// while `Array.prototype.slice.length === 2` — the last install would - /// win for every method. Recording the length per *closure instance* here - /// (keyed by the closure pointer, like the user-facing dynamic-prop table - /// but isolated from it so a user `fn.length = x` write can't perturb it) - /// lets the `.length` value-read and `getOwnPropertyDescriptor` agree with - /// the spec count. #3143. - static BUILTIN_CLOSURE_LENGTH: std::cell::RefCell> = - std::cell::RefCell::new(crate::fast_hash::new_ptr_hash_map()); - - /// Built-in method closures are callable but lack ECMAScript - /// `[[Construct]]`. Track the installed closure values so the dynamic - /// `new` / `Reflect.construct` paths can reject them without changing - /// ordinary user closures or global constructor closures. - static BUILTIN_CLOSURE_NON_CONSTRUCTABLE: std::cell::RefCell> = - std::cell::RefCell::new(crate::fast_hash::new_ptr_hash_set()); -} - -/// Record the spec `.length` for a built-in prototype-method closure. See -/// [`BUILTIN_CLOSURE_LENGTH`]. -pub(crate) fn set_builtin_closure_length(closure: usize, length: u32) { - BUILTIN_CLOSURE_LENGTH.with(|m| { - m.borrow_mut().insert(closure, length); - }); -} - -/// Look up the recorded spec `.length` for a built-in prototype-method -/// closure, or `None` if this closure isn't one. See [`BUILTIN_CLOSURE_LENGTH`]. -pub(crate) fn builtin_closure_length(closure: usize) -> Option { - BUILTIN_CLOSURE_LENGTH.with(|m| m.borrow().get(&closure).copied()) -} - -pub(crate) fn set_builtin_closure_non_constructable(closure: usize) { - BUILTIN_CLOSURE_NON_CONSTRUCTABLE.with(|m| { - m.borrow_mut().insert(closure); - }); -} - -pub(crate) fn builtin_closure_is_non_constructable(closure: usize) -> bool { - BUILTIN_CLOSURE_NON_CONSTRUCTABLE.with(|m| m.borrow().contains(&closure)) -} - -/// Rekey per-instance built-in closure metadata after a moving collection. -/// -/// The keys are identities, not roots: prototype/global objects keep live -/// built-in closures reachable, while dead closures must remain collectable. -/// `visit_metadata_usize_slot` therefore only follows forwarding records. -pub(crate) fn scan_builtin_closure_metadata_roots_mut( - visitor: &mut crate::gc::RuntimeRootVisitor<'_>, -) { - BUILTIN_CLOSURE_LENGTH.with(|lengths| { - let mut lengths = lengths.borrow_mut(); - let mut moved = Vec::new(); - for old_owner in lengths.keys().copied() { - let mut new_owner = old_owner; - if visitor.visit_metadata_usize_slot(&mut new_owner) && new_owner != old_owner { - moved.push((old_owner, new_owner)); - } - } - for (old_owner, new_owner) in moved { - if let Some(length) = lengths.remove(&old_owner) { - lengths.insert(new_owner, length); - } - } - }); - - BUILTIN_CLOSURE_NON_CONSTRUCTABLE.with(|non_constructable| { - let mut non_constructable = non_constructable.borrow_mut(); - let mut moved = Vec::new(); - for old_owner in non_constructable.iter().copied() { - let mut new_owner = old_owner; - if visitor.visit_metadata_usize_slot(&mut new_owner) && new_owner != old_owner { - moved.push((old_owner, new_owner)); - } - } - for (old_owner, new_owner) in moved { - non_constructable.remove(&old_owner); - non_constructable.insert(new_owner); - } - }); -} - -/// Drop metadata for closures proved dead by the collector before their arena -/// addresses can be recycled for unrelated objects. -pub(crate) fn prune_dead_builtin_closure_metadata_owners(is_dead_owner: &dyn Fn(usize) -> bool) { - BUILTIN_CLOSURE_LENGTH.with(|lengths| { - lengths - .borrow_mut() - .retain(|owner, _| !is_dead_owner(*owner)); - }); - BUILTIN_CLOSURE_NON_CONSTRUCTABLE.with(|non_constructable| { - non_constructable - .borrow_mut() - .retain(|owner| !is_dead_owner(*owner)); - }); -} - pub(crate) fn builtin_closure_is_non_constructable_value(value: f64) -> bool { let jv = JSValue::from_bits(value.to_bits()); if !jv.is_pointer() { diff --git a/crates/perry-runtime/src/object/native_module/callable_exports/builtin_closure_metadata.rs b/crates/perry-runtime/src/object/native_module/callable_exports/builtin_closure_metadata.rs new file mode 100644 index 0000000000..fe1a2f75c6 --- /dev/null +++ b/crates/perry-runtime/src/object/native_module/callable_exports/builtin_closure_metadata.rs @@ -0,0 +1,201 @@ +//! Young-scoped GC maintenance for per-instance built-in closure metadata. + +thread_local! { + static BUILTIN_CLOSURE_LENGTH: std::cell::RefCell> = + std::cell::RefCell::new(crate::fast_hash::new_ptr_hash_map()); + static BUILTIN_CLOSURE_NON_CONSTRUCTABLE: std::cell::RefCell> = + std::cell::RefCell::new(crate::fast_hash::new_ptr_hash_set()); + static BUILTIN_CLOSURE_YOUNG: std::cell::RefCell> = + const { std::cell::RefCell::new(crate::gc::young_log::YoungLog::new()) }; + #[cfg(test)] + static TEST_SUPPRESS_BUILTIN_CLOSURE_YOUNG_NOTE: std::cell::Cell = + const { std::cell::Cell::new(false) }; +} + +const LOG_NAME: &str = "object.builtin_closure_metadata"; + +#[inline] +fn note(closure: usize) { + if !crate::gc::young_log::addr_is_minor_collectible(closure) { + return; + } + #[cfg(test)] + if TEST_SUPPRESS_BUILTIN_CLOSURE_YOUNG_NOTE.with(std::cell::Cell::get) { + return; + } + BUILTIN_CLOSURE_YOUNG.with(|log| log.borrow_mut().note(closure)); +} + +pub(crate) fn set_builtin_closure_length(closure: usize, length: u32) { + note(closure); + BUILTIN_CLOSURE_LENGTH.with(|m| { + m.borrow_mut().insert(closure, length); + }); +} + +pub(crate) fn builtin_closure_length(closure: usize) -> Option { + BUILTIN_CLOSURE_LENGTH.with(|m| m.borrow().get(&closure).copied()) +} + +pub(crate) fn set_builtin_closure_non_constructable(closure: usize) { + note(closure); + BUILTIN_CLOSURE_NON_CONSTRUCTABLE.with(|m| { + m.borrow_mut().insert(closure); + }); +} + +pub(crate) fn builtin_closure_is_non_constructable(closure: usize) -> bool { + BUILTIN_CLOSURE_NON_CONSTRUCTABLE.with(|m| m.borrow().contains(&closure)) +} + +#[cfg(any(debug_assertions, test))] +fn relevant_owners() -> Vec { + let mut owners = Vec::new(); + BUILTIN_CLOSURE_LENGTH.with(|m| { + owners.extend( + m.borrow() + .keys() + .copied() + .filter(|owner| crate::gc::young_log::addr_is_minor_collectible(*owner)), + ); + }); + BUILTIN_CLOSURE_NON_CONSTRUCTABLE.with(|m| { + owners.extend( + m.borrow() + .iter() + .copied() + .filter(|owner| crate::gc::young_log::addr_is_minor_collectible(*owner)), + ); + }); + owners.sort_unstable(); + owners.dedup(); + owners +} + +fn visit_owner(visitor: &mut crate::gc::RuntimeRootVisitor<'_>, owner: usize) -> Option { + let mut new_owner = owner; + visitor.visit_metadata_usize_slot(&mut new_owner); + BUILTIN_CLOSURE_LENGTH.with(|lengths| { + let mut lengths = lengths.borrow_mut(); + if new_owner != owner { + if let Some(length) = lengths.remove(&owner) { + lengths.insert(new_owner, length); + } + } + }); + BUILTIN_CLOSURE_NON_CONSTRUCTABLE.with(|set| { + let mut set = set.borrow_mut(); + if new_owner != owner && set.remove(&owner) { + set.insert(new_owner); + } + }); + crate::gc::young_log::addr_is_minor_collectible(new_owner).then_some(new_owner) +} + +pub(crate) fn scan_builtin_closure_metadata_roots_mut( + visitor: &mut crate::gc::RuntimeRootVisitor<'_>, +) { + let table_len = BUILTIN_CLOSURE_LENGTH.with(|m| m.borrow().len()) + + BUILTIN_CLOSURE_NON_CONSTRUCTABLE.with(|m| m.borrow().len()); + if visitor.young_scope() { + #[cfg(any(debug_assertions, test))] + BUILTIN_CLOSURE_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(LOG_NAME, &relevant_owners()) + }); + let mut logged = 0u64; + let mut visited = 0u64; + let mut kept = BUILTIN_CLOSURE_YOUNG.with(|log| log.borrow_mut().take_spare()); + loop { + let batch = BUILTIN_CLOSURE_YOUNG.with(|log| log.borrow_mut().take_sorted()); + if batch.is_empty() { + break; + } + logged += batch.len() as u64; + for owner in batch { + visited += 1; + if let Some(owner) = visit_owner(visitor, owner) { + kept.push(owner); + } + } + } + let kept_len = kept.len() as u64; + BUILTIN_CLOSURE_YOUNG.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: true, + logged, + visited, + kept: kept_len, + table_len: table_len as u64, + }, + ); + return; + } + + let mut owners = Vec::new(); + BUILTIN_CLOSURE_LENGTH.with(|m| owners.extend(m.borrow().keys().copied())); + BUILTIN_CLOSURE_NON_CONSTRUCTABLE.with(|m| owners.extend(m.borrow().iter().copied())); + owners.sort_unstable(); + owners.dedup(); + let visited = owners.len() as u64; + let _ = BUILTIN_CLOSURE_YOUNG.with(|log| log.borrow_mut().take_sorted()); + let mut kept = BUILTIN_CLOSURE_YOUNG.with(|log| log.borrow_mut().take_spare()); + for owner in owners { + if let Some(owner) = visit_owner(visitor, owner) { + kept.push(owner); + } + } + let kept_len = kept.len() as u64; + BUILTIN_CLOSURE_YOUNG.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: false, + logged: visited, + visited, + kept: kept_len, + table_len: table_len as u64, + }, + ); +} + +pub(crate) fn prune_dead_builtin_closure_metadata_owners(is_dead_owner: &dyn Fn(usize) -> bool) { + BUILTIN_CLOSURE_LENGTH.with(|lengths| { + lengths + .borrow_mut() + .retain(|owner, _| !is_dead_owner(*owner)); + }); + BUILTIN_CLOSURE_NON_CONSTRUCTABLE.with(|non_constructable| { + non_constructable + .borrow_mut() + .retain(|owner| !is_dead_owner(*owner)); + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builtin_closure_log_rederivation_rejects_a_suppressed_writer() { + let _lock = crate::gc::global_side_table_test_lock(); + let closure = crate::closure::js_closure_alloc(std::ptr::null(), 0) as usize; + TEST_SUPPRESS_BUILTIN_CLOSURE_YOUNG_NOTE.with(|flag| flag.set(true)); + set_builtin_closure_length(closure, 3); + TEST_SUPPRESS_BUILTIN_CLOSURE_YOUNG_NOTE.with(|flag| flag.set(false)); + let missed = std::panic::catch_unwind(|| { + BUILTIN_CLOSURE_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(LOG_NAME, &relevant_owners()) + }); + }); + BUILTIN_CLOSURE_LENGTH.with(|m| m.borrow_mut().remove(&closure)); + BUILTIN_CLOSURE_YOUNG.with(|log| log.borrow_mut().clear()); + assert!( + missed.is_err(), + "sabotage: suppressing the setter's note must trip completeness" + ); + } +} diff --git a/crates/perry-runtime/src/symbol.rs b/crates/perry-runtime/src/symbol.rs index 6725d3a010..767c43d240 100644 --- a/crates/perry-runtime/src/symbol.rs +++ b/crates/perry-runtime/src/symbol.rs @@ -584,6 +584,7 @@ pub(crate) fn register_symbol_pointer(ptr: usize) { SYMBOL_EVER_REGISTERED.arm(); // Admit before the insert, for the same reason. admit_symbol_pointer(ptr); + gc_roots::note_symbol_pointer(ptr); let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); if guard.is_none() { *guard = Some(new_ptr_hash_set()); @@ -1036,6 +1037,7 @@ pub(crate) fn store_object_symbol_property_root( value_bits: u64, ) -> bool { note_symbol_key_installed(sym_key); + gc_roots::note_symbol_property_root(obj_key, sym_key, value_bits); { let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); if guard.is_none() { @@ -1070,6 +1072,7 @@ pub(crate) static CLASS_STATIC_SYMBOLS_LATCH: crate::registry_latch::RegistryLat pub(crate) fn store_class_static_symbol_root(class_id: u32, sym_key: usize, value_bits: u64) { note_symbol_key_installed(sym_key); + gc_roots::note_class_static_symbol(class_id, sym_key, value_bits); CLASS_STATIC_SYMBOLS_LATCH.arm(); let symbol_id = unsafe { (*(sym_key as *const SymbolHeader)).id }; let created; diff --git a/crates/perry-runtime/src/symbol/accessors.rs b/crates/perry-runtime/src/symbol/accessors.rs index 9ceeddbe3c..9fb977e553 100644 --- a/crates/perry-runtime/src/symbol/accessors.rs +++ b/crates/perry-runtime/src/symbol/accessors.rs @@ -68,6 +68,7 @@ pub(crate) fn test_symbol_accessor_property_count() -> usize { #[cfg(test)] pub(crate) fn test_seed_symbol_accessor_property(obj_key: usize, sym_key: usize, get_bits: u64) { + super::gc_roots::note_symbol_accessor(obj_key, sym_key, get_bits, TAG_UNDEFINED); let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_ACCESSOR_PROPERTIES); guard.get_or_insert_with(HashMap::new).insert( (obj_key, sym_key), @@ -90,6 +91,8 @@ pub(crate) unsafe fn set_symbol_accessor_property( return; } crate::symbol::note_symbol_key_installed(sym_key); + super::gc_roots::note_symbol_property_root(obj_key, sym_key, crate::value::TAG_UNDEFINED); + super::gc_roots::note_symbol_accessor(obj_key, sym_key, get_bits, set_bits); { // `SYMBOL_PROPERTIES` is the only insertion-ordered record of symbol // property CREATION order, which `[[OwnPropertyKeys]]` must report @@ -219,6 +222,29 @@ pub(super) fn accessor_property_keys() -> Vec<(usize, usize)> { .unwrap_or_default() } +pub(super) fn accessor_property_count() -> usize { + let guard = crate::gc::lock_gc_root_registry(&SYMBOL_ACCESSOR_PROPERTIES); + guard.as_ref().map_or(0, HashMap::len) +} + +pub(super) fn relevant_accessor_property_keys() -> Vec<(usize, usize)> { + let guard = crate::gc::lock_gc_root_registry(&SYMBOL_ACCESSOR_PROPERTIES); + guard + .as_ref() + .map(|map| { + map.iter() + .filter_map(|(&(owner, sym_key), acc)| { + (crate::gc::young_log::addr_is_minor_collectible(owner) + || crate::gc::young_log::addr_is_minor_relevant(sym_key) + || crate::gc::young_log::bits_are_minor_relevant(acc.get) + || crate::gc::young_log::bits_are_minor_relevant(acc.set)) + .then_some((owner, sym_key)) + }) + .collect() + }) + .unwrap_or_default() +} + /// Step twin of `scan_symbol_accessor_roots_mut` for one snapshot key: /// strong-visits the get/set closures and rekeys owner/sym on a move. /// Cycle-based collections run ONLY the step scanner, so before this @@ -228,13 +254,13 @@ pub(super) fn scan_symbol_accessor_root_slot( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, owner: usize, sym_key: usize, -) { +) -> Option<(usize, usize)> { let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_ACCESSOR_PROPERTIES); let Some(map) = guard.as_mut() else { - return; + return None; }; let Some(acc) = map.get_mut(&(owner, sym_key)) else { - return; + return None; }; let mut new_owner = owner; let mut new_sym_key = sym_key; @@ -251,6 +277,21 @@ pub(super) fn scan_symbol_accessor_root_slot( map.insert((new_owner, new_sym_key), acc); } } + symbol_accessor_root_relevant_in(map, new_owner, new_sym_key) + .then_some((new_owner, new_sym_key)) +} + +fn symbol_accessor_root_relevant_in( + map: &HashMap<(usize, usize), SymbolAccessorDescriptor>, + owner: usize, + sym_key: usize, +) -> bool { + map.get(&(owner, sym_key)).is_some_and(|acc| { + crate::gc::young_log::addr_is_minor_collectible(owner) + || crate::gc::young_log::addr_is_minor_relevant(sym_key) + || crate::gc::young_log::bits_are_minor_relevant(acc.get) + || crate::gc::young_log::bits_are_minor_relevant(acc.set) + }) } pub(super) fn has_own_symbol_accessor(obj_key: usize, sym_key: usize) -> bool { diff --git a/crates/perry-runtime/src/symbol/gc_roots.rs b/crates/perry-runtime/src/symbol/gc_roots.rs index 8c05be3014..7fec611384 100644 --- a/crates/perry-runtime/src/symbol/gc_roots.rs +++ b/crates/perry-runtime/src/symbol/gc_roots.rs @@ -21,11 +21,16 @@ pub fn scan_symbol_side_table_roots(mark: &mut dyn FnMut(f64)) { } pub fn scan_symbol_side_table_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + if visitor.young_scope() { + scan_young_symbol_side_table_roots_mut(visitor); + return; + } scan_symbol_property_roots_mut(visitor); scan_symbol_property_attrs_mut(visitor); accessors::scan_symbol_accessor_roots_mut(visitor); scan_class_static_symbol_roots_mut(visitor); scan_symbol_pointer_metadata_roots_mut(visitor); + rebuild_symbol_young_log(); } fn scan_symbol_property_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { @@ -131,7 +136,7 @@ fn scan_symbol_pointer_metadata_roots_mut(visitor: &mut crate::gc::RuntimeRootVi } } -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] enum SymbolSideTableRootSlot { SymbolPropertyOwner { owner: usize }, SymbolPropertyEntry { owner: usize, sym_key: usize }, @@ -141,15 +146,83 @@ enum SymbolSideTableRootSlot { SymbolPointer { ptr: usize }, } +crate::perry_thread_local! { + static SYMBOL_SIDE_TABLE_YOUNG: RefCell> = + const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; + #[cfg(test)] + static TEST_SUPPRESS_SYMBOL_YOUNG_NOTE: std::cell::Cell = + const { std::cell::Cell::new(false) }; +} + +const SYMBOL_YOUNG_LOG_NAME: &str = "symbol.side_tables"; + +#[inline] +fn note_symbol_slot(slot: SymbolSideTableRootSlot) { + #[cfg(test)] + if TEST_SUPPRESS_SYMBOL_YOUNG_NOTE.with(std::cell::Cell::get) { + return; + } + SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().note(slot)); +} + +pub(super) fn note_symbol_property_root(owner: usize, sym_key: usize, value_bits: u64) { + if crate::gc::young_log::addr_is_minor_collectible(owner) { + note_symbol_slot(SymbolSideTableRootSlot::SymbolPropertyOwner { owner }); + } + if crate::gc::young_log::addr_is_minor_relevant(sym_key) + || crate::gc::young_log::bits_are_minor_relevant(value_bits) + { + note_symbol_slot(SymbolSideTableRootSlot::SymbolPropertyEntry { owner, sym_key }); + } +} + +pub(super) fn note_symbol_property_attrs(owner: usize, sym_key: usize) { + if crate::gc::young_log::addr_is_minor_collectible(owner) + || crate::gc::young_log::addr_is_minor_relevant(sym_key) + { + note_symbol_slot(SymbolSideTableRootSlot::SymbolPropertyAttrs { owner, sym_key }); + } +} + +pub(super) fn note_symbol_accessor(owner: usize, sym_key: usize, get_bits: u64, set_bits: u64) { + if crate::gc::young_log::addr_is_minor_collectible(owner) + || crate::gc::young_log::addr_is_minor_relevant(sym_key) + || crate::gc::young_log::bits_are_minor_relevant(get_bits) + || crate::gc::young_log::bits_are_minor_relevant(set_bits) + { + note_symbol_slot(SymbolSideTableRootSlot::SymbolAccessorProperty { owner, sym_key }); + } +} + +pub(super) fn note_class_static_symbol(class_id: u32, sym_key: usize, value_bits: u64) { + if crate::gc::young_log::addr_is_minor_relevant(sym_key) + || crate::gc::young_log::bits_are_minor_relevant(value_bits) + { + note_symbol_slot(SymbolSideTableRootSlot::ClassStaticSymbol { class_id, sym_key }); + } +} + +pub(super) fn note_symbol_pointer(ptr: usize) { + if crate::gc::young_log::addr_is_minor_collectible(ptr) { + note_symbol_slot(SymbolSideTableRootSlot::SymbolPointer { ptr }); + } +} + pub(crate) struct SymbolSideTableRootScanState { - slots: Vec, + slots: Option>, + kept: Vec, cursor: usize, + young: bool, + table_len: usize, } pub(crate) fn new_symbol_side_table_root_scan_state() -> Box { Box::new(SymbolSideTableRootScanState { - slots: symbol_side_table_root_snapshot(), + slots: None, + kept: Vec::new(), cursor: 0, + young: false, + table_len: 0, }) } @@ -161,12 +234,49 @@ pub(crate) fn scan_symbol_side_table_roots_mut_step( let state = state .downcast_mut::() .expect("symbol side-table root scanner state type"); - while *remaining > 0 && state.cursor < state.slots.len() { - scan_symbol_side_table_root_slot(visitor, state.slots[state.cursor]); + if state.slots.is_none() { + state.young = visitor.young_scope(); + if state.young { + state.table_len = symbol_side_table_root_len(); + #[cfg(any(debug_assertions, test))] + SYMBOL_SIDE_TABLE_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(SYMBOL_YOUNG_LOG_NAME, &relevant_symbol_slots()) + }); + state.slots = Some(SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().take_sorted())); + } else { + let authoritative = symbol_side_table_root_snapshot(); + state.table_len = authoritative.len(); + let _ = SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().take_sorted()); + state.slots = Some(authoritative); + } + } + let slots = state.slots.as_ref().expect("symbol slots initialized"); + while *remaining > 0 && state.cursor < slots.len() { + if let Some(slot) = scan_symbol_side_table_root_slot(visitor, slots[state.cursor]) { + state.kept.push(slot); + } state.cursor += 1; *remaining -= 1; } - state.cursor >= state.slots.len() + let done = state.cursor >= slots.len(); + if done { + let logged = slots.len() as u64; + let kept = std::mem::take(&mut state.kept); + let kept_len = kept.len() as u64; + SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + SYMBOL_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: state.young, + logged, + visited: state.cursor as u64, + kept: kept_len, + table_len: state.table_len as u64, + }, + ); + } + done } fn symbol_side_table_root_snapshot() -> Vec { @@ -218,13 +328,163 @@ fn symbol_side_table_root_snapshot() -> Vec { slots } +fn symbol_side_table_root_len() -> usize { + // Exact only for diagnostics: counting the property vectors is itself a + // whole-table walk, which the release minor must not pay merely to report + // how much work it skipped. + if !crate::gc::gc_diag_enabled() { + return 0; + } + let properties = { + let guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); + guard.as_ref().map_or(0, |map| { + map.len() + map.values().map(Vec::len).sum::() + }) + }; + let attrs = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTY_ATTRS) + .as_ref() + .map_or(0, |map| map.len()); + let class_statics = crate::gc::lock_gc_root_registry(&CLASS_STATIC_SYMBOLS) + .as_ref() + .map_or(0, |map| map.len()); + let pointers = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS) + .as_ref() + .map_or(0, |set| set.len()); + properties + attrs + accessors::accessor_property_count() + class_statics + pointers +} + +fn collect_relevant_symbol_slots() -> Vec { + let mut slots = Vec::new(); + { + let guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); + if let Some(map) = guard.as_ref() { + for (&owner, entries) in map { + if crate::gc::young_log::addr_is_minor_collectible(owner) { + slots.push(SymbolSideTableRootSlot::SymbolPropertyOwner { owner }); + } + for &(sym_key, value_bits) in entries { + if crate::gc::young_log::addr_is_minor_relevant(sym_key) + || crate::gc::young_log::bits_are_minor_relevant(value_bits) + { + slots.push(SymbolSideTableRootSlot::SymbolPropertyEntry { owner, sym_key }); + } + } + } + } + } + { + let guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTY_ATTRS); + if let Some(map) = guard.as_ref() { + slots.extend(map.keys().filter_map(|&(owner, sym_key)| { + (crate::gc::young_log::addr_is_minor_collectible(owner) + || crate::gc::young_log::addr_is_minor_relevant(sym_key)) + .then_some(SymbolSideTableRootSlot::SymbolPropertyAttrs { owner, sym_key }) + })); + } + } + slots.extend( + accessors::relevant_accessor_property_keys() + .into_iter() + .map( + |(owner, sym_key)| SymbolSideTableRootSlot::SymbolAccessorProperty { + owner, + sym_key, + }, + ), + ); + { + let guard = crate::gc::lock_gc_root_registry(&CLASS_STATIC_SYMBOLS); + if let Some(map) = guard.as_ref() { + slots.extend( + map.iter() + .filter_map(|(&(class_id, sym_key), &value_bits)| { + (crate::gc::young_log::addr_is_minor_relevant(sym_key) + || crate::gc::young_log::bits_are_minor_relevant(value_bits)) + .then_some(SymbolSideTableRootSlot::ClassStaticSymbol { class_id, sym_key }) + }), + ); + } + } + { + let guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); + if let Some(set) = guard.as_ref() { + slots.extend(set.iter().filter_map(|&ptr| { + crate::gc::young_log::addr_is_minor_collectible(ptr) + .then_some(SymbolSideTableRootSlot::SymbolPointer { ptr }) + })); + } + } + slots +} + +#[cfg(any(debug_assertions, test))] +fn relevant_symbol_slots() -> Vec { + collect_relevant_symbol_slots() +} + +fn rebuild_symbol_young_log() { + let table_len = symbol_side_table_root_len(); + let relevant = collect_relevant_symbol_slots(); + let _ = SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().take_sorted()); + let kept = relevant.len() as u64; + SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().extend(relevant)); + crate::gc::young_log::note_walk( + SYMBOL_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: false, + logged: table_len as u64, + visited: table_len as u64, + kept, + table_len: table_len as u64, + }, + ); +} + +fn scan_young_symbol_side_table_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + let table_len = symbol_side_table_root_len(); + #[cfg(any(debug_assertions, test))] + SYMBOL_SIDE_TABLE_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(SYMBOL_YOUNG_LOG_NAME, &relevant_symbol_slots()) + }); + let mut kept = SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().take_spare()); + let mut logged = 0_u64; + loop { + let batch = SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().take_sorted()); + if batch.is_empty() { + break; + } + logged += batch.len() as u64; + for slot in batch { + if let Some(slot) = scan_symbol_side_table_root_slot(visitor, slot) { + kept.push(slot); + } + } + } + let kept_len = kept.len() as u64; + SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + SYMBOL_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: true, + logged, + visited: logged, + kept: kept_len, + table_len: table_len as u64, + }, + ); +} + fn scan_symbol_side_table_root_slot( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, slot: SymbolSideTableRootSlot, -) { +) -> Option { match slot { SymbolSideTableRootSlot::SymbolPropertyOwner { owner } => { - rewrite_symbol_property_owner_if_forwarded(visitor, owner); + rewrite_symbol_property_owner_if_forwarded(visitor, owner).and_then(|owner| { + crate::gc::young_log::addr_is_minor_collectible(owner) + .then_some(SymbolSideTableRootSlot::SymbolPropertyOwner { owner }) + }) } SymbolSideTableRootSlot::SymbolPropertyEntry { owner, sym_key } => { // The preceding budget slice may already have rekeyed this @@ -235,7 +495,7 @@ fn scan_symbol_side_table_root_slot( visitor.visit_metadata_usize_slot(&mut healed_owner); let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); let Some(map) = guard.as_mut() else { - return; + return None; }; let lookup_owner = if map.contains_key(&healed_owner) { healed_owner @@ -246,22 +506,43 @@ fn scan_symbol_side_table_root_slot( .get_mut(&lookup_owner) .and_then(|entries| entries.iter_mut().find(|entry| entry.0 == sym_key)) else { - return; + return None; }; visitor.visit_usize_slot(entry_sym); visitor.visit_nanbox_u64_slot(value_bits); + (crate::gc::young_log::addr_is_minor_relevant(*entry_sym) + || crate::gc::young_log::bits_are_minor_relevant(*value_bits)) + .then_some(SymbolSideTableRootSlot::SymbolPropertyEntry { + owner: lookup_owner, + sym_key: *entry_sym, + }) } SymbolSideTableRootSlot::SymbolAccessorProperty { owner, sym_key } => { - accessors::scan_symbol_accessor_root_slot(visitor, owner, sym_key); + accessors::scan_symbol_accessor_root_slot(visitor, owner, sym_key).map( + |(owner, sym_key)| SymbolSideTableRootSlot::SymbolAccessorProperty { + owner, + sym_key, + }, + ) } SymbolSideTableRootSlot::SymbolPropertyAttrs { owner, sym_key } => { - rewrite_symbol_property_attrs_if_forwarded(visitor, owner, sym_key); + rewrite_symbol_property_attrs_if_forwarded(visitor, owner, sym_key).and_then( + |(owner, sym_key)| { + (crate::gc::young_log::addr_is_minor_collectible(owner) + || crate::gc::young_log::addr_is_minor_relevant(sym_key)) + .then_some(SymbolSideTableRootSlot::SymbolPropertyAttrs { owner, sym_key }) + }, + ) } SymbolSideTableRootSlot::ClassStaticSymbol { class_id, sym_key } => { - rewrite_class_static_symbol_entry_if_forwarded(visitor, class_id, sym_key); + rewrite_class_static_symbol_entry_if_forwarded(visitor, class_id, sym_key) + .map(|sym_key| SymbolSideTableRootSlot::ClassStaticSymbol { class_id, sym_key }) } SymbolSideTableRootSlot::SymbolPointer { ptr } => { - rewrite_symbol_pointer_metadata_if_forwarded(visitor, ptr); + rewrite_symbol_pointer_metadata_if_forwarded(visitor, ptr).and_then(|ptr| { + crate::gc::young_log::addr_is_minor_collectible(ptr) + .then_some(SymbolSideTableRootSlot::SymbolPointer { ptr }) + }) } } } @@ -269,37 +550,41 @@ fn scan_symbol_side_table_root_slot( fn rewrite_symbol_property_owner_if_forwarded( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, owner: usize, -) { +) -> Option { let mut new_owner = owner; - if !visitor.visit_metadata_usize_slot(&mut new_owner) || new_owner == owner { - return; - } - let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); - if let Some(map) = guard.as_mut() { - if let Some(entries) = map.remove(&owner) { - match map.entry(new_owner) { - std::collections::hash_map::Entry::Occupied(mut entry) => { - merge_symbol_property_entries(entry.get_mut(), entries); - } - std::collections::hash_map::Entry::Vacant(entry) => { - entry.insert(entries); + if visitor.visit_metadata_usize_slot(&mut new_owner) && new_owner != owner { + let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); + if let Some(map) = guard.as_mut() { + if let Some(entries) = map.remove(&owner) { + match map.entry(new_owner) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + merge_symbol_property_entries(entry.get_mut(), entries); + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(entries); + } } } } } + let guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); + guard + .as_ref() + .is_some_and(|map| map.contains_key(&new_owner)) + .then_some(new_owner) } fn rewrite_symbol_property_attrs_if_forwarded( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, owner: usize, sym_key: usize, -) { +) -> Option<(usize, usize)> { let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTY_ATTRS); let Some(map) = guard.as_mut() else { - return; + return None; }; if !map.contains_key(&(owner, sym_key)) { - return; + return None; } let mut new_owner = owner; let mut new_sym_key = sym_key; @@ -310,19 +595,21 @@ fn rewrite_symbol_property_attrs_if_forwarded( map.insert((new_owner, new_sym_key), attrs); } } + map.contains_key(&(new_owner, new_sym_key)) + .then_some((new_owner, new_sym_key)) } fn rewrite_class_static_symbol_entry_if_forwarded( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, class_id: u32, sym_key: usize, -) { +) -> Option { let mut guard = crate::gc::lock_gc_root_registry(&CLASS_STATIC_SYMBOLS); let Some(map) = guard.as_mut() else { - return; + return None; }; let Some(value_bits) = map.get_mut(&(class_id, sym_key)) else { - return; + return None; }; let mut new_sym_key = sym_key; let moved = visitor.visit_usize_slot(&mut new_sym_key); @@ -332,27 +619,37 @@ fn rewrite_class_static_symbol_entry_if_forwarded( map.insert((class_id, new_sym_key), value_bits); } } + map.get(&(class_id, new_sym_key)).and_then(|value_bits| { + (crate::gc::young_log::addr_is_minor_relevant(new_sym_key) + || crate::gc::young_log::bits_are_minor_relevant(*value_bits)) + .then_some(new_sym_key) + }) } fn rewrite_symbol_pointer_metadata_if_forwarded( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, ptr: usize, -) { +) -> Option { let mut new_ptr = ptr; - if !visitor.visit_metadata_usize_slot(&mut new_ptr) || new_ptr == ptr { - return; - } - let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); - if let Some(set) = guard.as_mut() { - set.remove(&ptr); - if new_ptr != 0 { - insert_symbol_pointer_in_set(set, new_ptr); + if visitor.visit_metadata_usize_slot(&mut new_ptr) && new_ptr != ptr { + let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); + if let Some(set) = guard.as_mut() { + set.remove(&ptr); + if new_ptr != 0 { + insert_symbol_pointer_in_set(set, new_ptr); + } } } + let guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); + guard + .as_ref() + .is_some_and(|set| set.contains(&new_ptr)) + .then_some(new_ptr) } #[cfg(test)] pub(crate) fn test_clear_symbol_side_table_roots() { + SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().clear()); *crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES) = None; *crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTY_ATTRS) = None; *crate::gc::lock_gc_root_registry(&CLASS_STATIC_SYMBOLS) = None; @@ -379,6 +676,7 @@ pub(crate) fn test_clear_symbol_side_table_roots() { } else { let mut set = new_ptr_hash_set(); for ptr in persistent { + note_symbol_pointer(ptr); insert_symbol_pointer_in_set(&mut set, ptr); } *guard = Some(set); @@ -425,6 +723,7 @@ pub(crate) fn test_seed_class_static_symbol_root(class_id: u32, sym_key: usize, // an unaligned sentinel. Seed only the root table they exercise; // production registration additionally reads SymbolHeader::id for // [[OwnPropertyKeys]] ordering and therefore requires a real Symbol. + note_class_static_symbol(class_id, sym_key, value_bits); CLASS_STATIC_SYMBOLS_LATCH.arm(); let mut guard = crate::gc::lock_gc_root_registry(&CLASS_STATIC_SYMBOLS); if guard.is_none() { @@ -474,3 +773,31 @@ pub(crate) fn test_symbol_pointer_root_contains(ptr: usize) -> bool { let guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); guard.as_ref().is_some_and(|set| set.contains(&ptr)) } + +#[cfg(test)] +mod young_log_sabotage_tests { + use super::*; + + #[test] + fn symbol_log_rederivation_rejects_a_suppressed_property_writer() { + let _lock = crate::gc::global_side_table_test_lock(); + test_clear_symbol_side_table_roots(); + let owner = crate::object::js_object_alloc(0, 0) as usize; + let sym_bits = unsafe { crate::symbol::js_symbol_new_empty() }.to_bits(); + let sym_key = (sym_bits & POINTER_MASK) as usize; + TEST_SUPPRESS_SYMBOL_YOUNG_NOTE.with(|flag| flag.set(true)); + store_object_symbol_property_root(owner, sym_key, 7.0_f64.to_bits()); + TEST_SUPPRESS_SYMBOL_YOUNG_NOTE.with(|flag| flag.set(false)); + let missed = std::panic::catch_unwind(|| { + SYMBOL_SIDE_TABLE_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(SYMBOL_YOUNG_LOG_NAME, &relevant_symbol_slots()) + }); + }); + test_clear_symbol_side_table_roots(); + assert!( + missed.is_err(), + "sabotage: suppressing the property-store note must trip completeness" + ); + } +} diff --git a/crates/perry-runtime/src/symbol/properties.rs b/crates/perry-runtime/src/symbol/properties.rs index ddcbcf4f32..49adaada16 100644 --- a/crates/perry-runtime/src/symbol/properties.rs +++ b/crates/perry-runtime/src/symbol/properties.rs @@ -90,6 +90,7 @@ pub(crate) fn set_symbol_property_attrs( return; } super::note_symbol_key_installed(sym_key); + super::gc_roots::note_symbol_property_attrs(owner, sym_key); let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTY_ATTRS); if guard.is_none() { *guard = Some(crate::fast_hash::new_fast_key_hash_map()); From 0dec476b5d06317d67d75eaf9e2555d3dabd0d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 12:00:51 +0200 Subject: [PATCH 20/27] docs(perf): report minor phases and scanner logs Record the phase instrument, young-log mappings and sabotage coverage, shape residual analysis, validation results, performance predictions, and the exact perrymaster relink and measurement request. (cherry picked from commit de6bfd5c5d88140dcac198e6bddcae109501a881) --- .../codex/REPORT_minor_phases_and_logs.md | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 cc-perf-campaign/codex/REPORT_minor_phases_and_logs.md diff --git a/cc-perf-campaign/codex/REPORT_minor_phases_and_logs.md b/cc-perf-campaign/codex/REPORT_minor_phases_and_logs.md new file mode 100644 index 0000000000..6fac981962 --- /dev/null +++ b/cc-perf-campaign/codex/REPORT_minor_phases_and_logs.md @@ -0,0 +1,206 @@ +# Copying-minor phases and remaining scanner young logs + +Phase-instrument commit: `b444f0251221d8c40dd645c741367dcf5276dff9` + +Scanner-log implementation commit: `364ed3f07c54e365e65bbb5e23bd703a9fc54a0e` + +Branch: `perf/minor-phases-and-logs`, based on +`e2eee113d486b5208c56ae1e3f0f4d0dffcbf2b2`. + +## Copying-minor phase instrument + +- `crates/perry-runtime/src/gc/copying_phase.rs:26` is the diagnostic-only + accumulator. It uses the same `Instant` clock as `pause_us` and records + non-overlapping spans for `root_scan`, `copy_evacuation`, + `remembered_set_young_logs`, `promotion`, + `dead_owner_side_table_pruning`, `from_space_finalization`, + `forwarding_fixups`, and `block_reset_flip`. `other` is the exact residual + between those named spans and the whole pause, and `phase_sum_us` is formed + from the nanosecond partition before conversion, so it equals `pause_us` + apart from the shared sub-microsecond truncation (well inside 2%). +- `crates/perry-runtime/src/gc/copying.rs:1220-1749` starts and records the + counters in the functions whose work they price. The two registered-root + passes accumulate in `root_scan`; the transitive worklist drain is + `copy_evacuation`; remembered snapshot/dirty scan and post-cycle restore are + accumulated together; promotion covers retag plus finish; forwarding covers + promoted-edge rebuild and verification/fixup work; reset covers to-space + preparation plus the final reset/flip. +- `crates/perry-runtime/src/gc/copying_phase.rs:84` renders counts where the + collector already owns them: copied/promoted objects and bytes, remembered + entries and dirty slots, and finalized map/set/error/regexp owners. The + dead-owner fan-out at `crates/perry-runtime/src/gc/dead_owner.rs:261` clocks + every registry table separately and appends those table names and + microseconds to the same field. Those prune callbacks expose no removed-row + count, so no invented count is printed. +- `crates/perry-runtime/src/gc/copying.rs:1962` appends `phases:` to every + completed `[gc-copy-minor] ran` line. `PERRY_GC_DIAG` off creates no phase + accumulator, takes no phase clocks, and builds no detail strings. +- The sabotage unit + `copied_minor_phase_residual_makes_the_partition_exact` removes a named + bucket from the expected arithmetic if the partition is widened or omitted. + +## Scanner map and young-entry logs + +### `scan_descriptor_roots_mut` + +This walks string-keyed property-attribute and accessor tables plus their two +owner indexes. Owner addresses are metadata-only and need a minor visit only +while movable/reclaimable; accessor get/set NaN-boxes are strong roots and may +require tracing through Longlived values. A #9754 owner log already existed. +The write funnel at `object/descriptor_state.rs:148` is present at all five +publication/transfer sites (`:985`, `:1208`, `:1277`, `:1394`, `:1428`). This +change narrows the metadata-key half from `addr_is_minor_relevant` to +`addr_is_minor_collectible`; the re-derivation and post-visit keep predicate +use the same rule at `object/descriptor_state/young.rs:42` and +`object/descriptor_state/gc_scan.rs:17`. The full walk is unchanged and +rebuilds the log. + +### `scan_closure_dynamic_props_roots_mut` + +This walks `CLOSURE_PROPS` values, `CLOSURE_STATIC_PROTOTYPES` values, and the +metadata-only owners of those tables and `CLOSURE_DELETED_KEYS`. A #9754 owner +log already existed. The enforced write funnels are +`closure/dynamic_props.rs:117`, `:186`, `:241`, `:388`, and `:1068`. Owner +retention is now collectible-only; property/prototype values keep the broader +transitive predicate. The minor path is `:533`; the full path remains whole +table and rebuilds the log. + +### `scan_builtin_closure_metadata_roots_mut` + +This walks two owner-keyed, pointer-free metadata tables: closure arity and the +non-constructable set. Only the closure address can move or die. There was no +partial log. The tables and their complete setters were extracted to +`object/native_module/callable_exports/builtin_closure_metadata.rs`; `:18` +arms the owner log before either setter publishes, `:95` drains only logged +collectible owners on a minor, and the unchanged full walk visits all owners +and rebuilds the log. + +### `scan_template_raw_roots_mut` + +This scanner actually owns three tables: call-site to cooked/raw template +arrays, cooked to raw template arrays, and array named properties. Template +array keys/values are strong roots, so they deliberately keep +`addr_is_minor_relevant`: a Longlived template can contain transitive GC +edges. Array named-property owners are metadata-only and collectible-only; +their NaN-box values remain broad strong roots. No partial log existed. +`array/header/young_roots.rs:29`, `:35`, and `:45` define the three logs; +every publication and array-growth transfer arms before publish at +`array/header.rs:191`, `:262`, `:346`, `:410`, `:446`, and test seeding at +`:604`. The minor scanner is `array/header/young_roots.rs:181`; the full walk +still visits every entry and rebuilds all three logs. + +### `scan_symbol_side_table_roots_mut` + +This walks six slot shapes: `SYMBOL_PROPERTIES` owner metadata and strong +symbol/value pairs, `SYMBOL_PROPERTY_ATTRS` owner metadata and strong symbol +keys, symbol accessors plus get/set roots, class-static symbol/value pairs, +and metadata-only `SYMBOL_POINTERS`. There was no partial log. A typed slot log +at `symbol/gc_roots.rs:146-209` records exactly the slot shape that can matter +to a minor. Production funnels arm before publication in `symbol.rs:582`, +`:1030`, `:1065`, `symbol/properties.rs:93`, and +`symbol/accessors.rs:94-95`; direct test seeders follow the same contract. The +direct minor path at `symbol/gc_roots.rs:443` and the budgeted step path at +`:229` take only logged slots. Property-owner slots sort before their entries, +so owner rekeying precedes entry lookup; entry scans heal a snapshot owner +through forwarding. Full direct and step walks still take authoritative +whole-table snapshots and rebuild the log. + +All five scanners emit their existing `[gc-young-log]` accounting with +logged/visited/kept/table size. Release minors do not enumerate a whole table +to obtain the symbol table size: that exact diagnostic count is itself gated +on `PERRY_GC_DIAG`. + +## Sabotage tests + +- `descriptor_log_rederivation_rejects_a_suppressed_setter`: suppresses the + real property-attrs funnel; re-derivation must report the missing owner. +- `closure_log_rederivation_rejects_a_suppressed_setter`: suppresses the real + closure dynamic-property funnel; re-derivation must report the missing + owner. +- `builtin_closure_log_rederivation_rejects_a_suppressed_writer`: suppresses + the arity setter; re-derivation must report the missing closure. +- `template_raw_log_rederivation_rejects_a_suppressed_writer`: suppresses the + cooked/raw publication funnel; re-derivation must report the missing pair. + `array_named_log_rederivation_rejects_a_suppressed_setter` independently + covers the third table owned by that scanner. +- `symbol_log_rederivation_rejects_a_suppressed_property_writer`: suppresses + the production symbol-property store; re-derivation must report its missing + typed slots. + +Each completeness check is compiled under `debug_assertions` and `test`. In +the release lib run below, every named sabotage test passed. + +## Shape residual + +The residual is real young work, not another whole-table leak. The exact keep +predicate is `object/shapes.rs:2154`: + +- Nursery Eden, either survivor half, and `PromotedYoung` keys arrays stay + logged because their table keys must be rewritten if they move. +- Malloc-GC keys arrays stay only when an old/cache carrier makes the family a + root and the allocation remains minor-collectible. +- Longlived keys arrays stay only when an old/cache carrier roots the family + **and** at least one property-key leaf in the array is collectible. Longlived + non-carriers and carriers whose leaves are all old/Longlived drop out. +- Old keys arrays always drop out. + +There is one intentional transient duplicate at `object/shapes.rs:2244`: the +mark pass may move a family before the metadata-only slot index is repaired in +the rewrite pass, so both the post-copy address and stale index address must +survive between the passes. Tightening any of these remaining cases would +skip relocation, collection of malloc keys, a strong carrier edge, or the +between-pass index repair. This explains why shape time appears only on the +steady minors that create/grow a burst of genuinely young shape-key arrays; +there is no sound additional predicate tightening in this change. + +## Validation + +- `git diff --check`: PASS. +- `scripts/check_file_size.sh`: PASS (all Rust files at most 2,000 lines). +- `scripts/gc_runtime_root_holders.py`: PASS. +- `scripts/gc_rekeyed_key_tables.py`: PASS. +- `cargo test -p perry-runtime --release --lib -j4 -- --test-threads=1` via + `measure_lock.sh --build`: NOT GREEN solely because it was run detached with + a PTY. Compilation completed and 3,273 tests passed (including every new + sabotage), 4 were ignored, and the sole failure was + `tty::tests::columns_undefined_when_not_tty`, whose assertion correctly saw + the allocated PTY. Two earlier attempts stopped at compile diagnostics in + the newly extracted module; those visibility/TLS/null-pointer/type issues + were fixed before this complete run. +- The required non-PTY rerun was NOT RUN: immediately afterward `df -g /` + reported 7 GB free, below the binding 12 GB floor. Per the task rule, no + further Cargo command was started and no disk wait was attempted. +- `cargo build --release -p perry-runtime --features wasm-host -j4`: NOT RUN, + same 7 GB disk stop. +- `cargo build --release -p perry -j4`: NOT RUN, same 7 GB disk stop. + +## Predictions and exact perrymaster request + +Predictions: on a zero-live steady minor, each of +`scan_descriptor_roots_mut`, `scan_closure_dynamic_props_roots_mut`, +`scan_builtin_closure_metadata_roots_mut`, `scan_template_raw_roots_mut`, and +`scan_symbol_side_table_roots_mut` is at most **0.3 ms**. Steady scanner total +is at most **5 ms**. This removes roughly 10 ms from a representative steady +minor when the five logs are empty; the phase table, not that estimate, must +name the next non-scanner lever. RSS changes should be small retained log +buffers and remain inside Ralph's allowed +1-10% band. + +Exact perrymaster request: fetch pushed branch `perf/minor-phases-and-logs` and +relink this runtime-only change on main's cache. Run the three required gates +through +`/Users/amlug/projects/perry/secret-tests/cc-perf-campaign/measure_lock.sh --build` +detached, using exactly: + +1. `cargo test -p perry-runtime --release --lib -j4 -- --test-threads=1` +2. `cargo build --release -p perry-runtime --features wasm-host -j4` +3. `cargo build --release -p perry -j4` + +Then run one +graceful four-turn 3300-character cc workload and one 400-character workload +with `PERRY_GC_DIAG=1`, printing and preserving **every complete** +`[gc-copy-minor] ran` line. The phase table for a steady minor is the +deliverable that names the next lever. Confirm all five named scanners are at +most 0.3 ms on zero-live steady minors and steady scanner total is at most 5 +ms. Finally run paired **5x3300 + 3x400** against both main and #9950's runtime, +reporting cc turn CPU and peak RSS; target node/bun CPU parity, allowing only ++1-10% RSS. From a8547f9cedbb22b51716d827c2adda47d5381b51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 15:05:09 +0200 Subject: [PATCH 21/27] perf(gc): drop slower template and symbol young logs Restore the dense full-table walkers for template roots and symbol side tables after MP measurements showed that their keyed young paths cost more. Remove the associated publication upkeep and rederivation tests while leaving the three measured wins and descriptor narrowing intact. Move the built-in closure young log onto hot TLS and update the thread-local and rekey policy inventories for the callable-exports module split. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit 151680fa0f6ffbf6ebb7d8166e2424fc0d031b49) --- .../codex/REPORT_minor_phases_and_logs.md | 150 ++++--- crates/perry-runtime/src/array/header.rs | 64 ++- .../src/array/header/young_roots.rs | 302 ------------- .../builtin_closure_metadata.rs | 8 +- crates/perry-runtime/src/symbol.rs | 3 - crates/perry-runtime/src/symbol/accessors.rs | 47 +- crates/perry-runtime/src/symbol/gc_roots.rs | 411 ++---------------- crates/perry-runtime/src/symbol/properties.rs | 1 - scripts/gc_rekeyed_key_tables.json | 4 +- scripts/thread_local_cold_allowlist.json | 5 +- 10 files changed, 193 insertions(+), 802 deletions(-) delete mode 100644 crates/perry-runtime/src/array/header/young_roots.rs diff --git a/cc-perf-campaign/codex/REPORT_minor_phases_and_logs.md b/cc-perf-campaign/codex/REPORT_minor_phases_and_logs.md index 6fac981962..80af5e11b4 100644 --- a/cc-perf-campaign/codex/REPORT_minor_phases_and_logs.md +++ b/cc-perf-campaign/codex/REPORT_minor_phases_and_logs.md @@ -1,11 +1,11 @@ # Copying-minor phases and remaining scanner young logs -Phase-instrument commit: `b444f0251221d8c40dd645c741367dcf5276dff9` +Phase-instrument commit: `09846784c` -Scanner-log implementation commit: `364ed3f07c54e365e65bbb5e23bd703a9fc54a0e` +Scanner-log implementation commit: `dae519296` Branch: `perf/minor-phases-and-logs`, based on -`e2eee113d486b5208c56ae1e3f0f4d0dffcbf2b2`. +`8b7dc3342`. ## Copying-minor phase instrument @@ -77,38 +77,70 @@ and rebuilds the log. ### `scan_template_raw_roots_mut` -This scanner actually owns three tables: call-site to cooked/raw template -arrays, cooked to raw template arrays, and array named properties. Template -array keys/values are strong roots, so they deliberately keep -`addr_is_minor_relevant`: a Longlived template can contain transitive GC -edges. Array named-property owners are metadata-only and collectible-only; -their NaN-box values remain broad strong roots. No partial log existed. -`array/header/young_roots.rs:29`, `:35`, and `:45` define the three logs; -every publication and array-growth transfer arms before publish at -`array/header.rs:191`, `:262`, `:346`, `:410`, `:446`, and test seeding at -`:604`. The minor scanner is `array/header/young_roots.rs:181`; the full walk -still visits every entry and rebuilds all three logs. +This scanner owns three small tables: call-site to cooked/raw template arrays, +cooked to raw template arrays, and array named properties. The attempted young +logs were reverted after MP measurement showed 2.76 ms for the keyed path +against 1.83 ms for the original full walk. The scanner again walks the three +authoritative tables directly, with no insert-side log upkeep. ### `scan_symbol_side_table_roots_mut` This walks six slot shapes: `SYMBOL_PROPERTIES` owner metadata and strong symbol/value pairs, `SYMBOL_PROPERTY_ATTRS` owner metadata and strong symbol keys, symbol accessors plus get/set roots, class-static symbol/value pairs, -and metadata-only `SYMBOL_POINTERS`. There was no partial log. A typed slot log -at `symbol/gc_roots.rs:146-209` records exactly the slot shape that can matter -to a minor. Production funnels arm before publication in `symbol.rs:582`, -`:1030`, `:1065`, `symbol/properties.rs:93`, and -`symbol/accessors.rs:94-95`; direct test seeders follow the same contract. The -direct minor path at `symbol/gc_roots.rs:443` and the budgeted step path at -`:229` take only logged slots. Property-owner slots sort before their entries, -so owner rekeying precedes entry lookup; entry scans heal a snapshot owner -through forwarding. Full direct and step walks still take authoritative -whole-table snapshots and rebuild the log. - -All five scanners emit their existing `[gc-young-log]` accounting with -logged/visited/kept/table size. Release minors do not enumerate a whole table -to obtain the symbol table size: that exact diagnostic count is itself gated -on `PERRY_GC_DIAG`. +and metadata-only `SYMBOL_POINTERS`. The attempted typed-slot young log was +reverted after MP measurement showed 2.47 ms for the keyed path against +1.74 ms for the original full walk. Direct scans again iterate the +authoritative tables, and budgeted scans again use the pre-existing full slot +snapshot; none of the symbol writers pays young-log upkeep. + +The retained descriptor, closure-dynamic-property, built-in-closure-metadata, +and shape-cache young logs continue to emit `[gc-young-log]` accounting with +logged/visited/kept/table size. + +## MP measurement and the two reverted/fixed logs + +Perrymaster measured MP-stage medians over 16 steady 3300-character minor +collections, comparing app-m6mp (the five scanner changes) with app-m6ms +(without them): + +| scanner | m6ms full walk | m6mp young log | delta | +|---|---:|---:|---:| +| descriptor_roots | 4.12 ms | 4.02 ms | -0.1 ms | +| closure_dynamic_props | 3.75 ms | 3.06 ms | **-0.7 ms** | +| builtin_closure_metadata | 1.42 ms | 0.93 ms | **-0.5 ms** | +| shape_cache | 0.68 ms | 0.27 ms | **-0.4 ms** | +| **template_raw_roots** | 1.83 ms | **2.76 ms** | **+0.9 ms** | +| **symbol_side_table** | 1.74 ms | **2.47 ms** | **+0.7 ms** | +| transition_cache / intern / class_side / singleton_closure / box | flat | flat | flat | +| **total** | **15.5 ms** | **15.3 ms** | **-0.2 ms** | + +Both regressions are case (b): the logged path made each visited entry more +expensive than the dense full walk. They are not duplicate-log failures: +`YoungLog::take_sorted` sorts and globally deduplicates every batch, and each +writer tests the young/relevant predicate before noting a key. + +- `template_raw_roots`: the full scanner streams each map directly and only + removes/reinserts keys that actually move. The young path sorted its keys, + performed a hash lookup for every cache entry, and unconditionally removed + and reinserted every logged raw-map and named-property owner even when the + owner did not move. A pointer or index cannot safely retain the full walk's + per-entry cost because insertion and rekeying can relocate these `HashMap` + entries. The three logs, their publication hooks, and their two rederivation + tests were therefore removed. +- `symbol_side_table`: the full scanner streams the maps and their property + vectors. The typed-slot path sorted its keys, then recovered every property + entry through an owner hash lookup plus a linear search of that owner's + vector; the other slot shapes also paid keyed table lookups. Hash-map + rekeying and vector growth make raw entry pointers or indices unstable, so a + safe O(young) path with the full walk's per-entry cost would require a + structural table redesign. The typed log, all writer hooks, and its + rederivation test were therefore removed. + +Re-measurement falsifier: on perrymaster, `template_raw_roots` must be at most +**1.83 ms** and `symbol_side_table` at most **1.74 ms** at the median, the +three improved scanners must remain unchanged, and total scanner time must be +at most **14 ms**. ## Sabotage tests @@ -119,13 +151,11 @@ on `PERRY_GC_DIAG`. owner. - `builtin_closure_log_rederivation_rejects_a_suppressed_writer`: suppresses the arity setter; re-derivation must report the missing closure. -- `template_raw_log_rederivation_rejects_a_suppressed_writer`: suppresses the - cooked/raw publication funnel; re-derivation must report the missing pair. - `array_named_log_rederivation_rejects_a_suppressed_setter` independently - covers the third table owned by that scanner. -- `symbol_log_rederivation_rejects_a_suppressed_property_writer`: suppresses - the production symbol-property store; re-derivation must report its missing - typed slots. +- `template_raw_log_rederivation_rejects_a_suppressed_writer`, + `array_named_log_rederivation_rejects_a_suppressed_setter`, and + `symbol_log_rederivation_rejects_a_suppressed_property_writer` were removed + with the two reverted logs; their enforced-writer invariant no longer + exists. Each completeness check is compiled under `debug_assertions` and `test`. In the release lib run below, every named sabotage test passed. @@ -157,33 +187,28 @@ there is no sound additional predicate tightening in this change. - `git diff --check`: PASS. - `scripts/check_file_size.sh`: PASS (all Rust files at most 2,000 lines). -- `scripts/gc_runtime_root_holders.py`: PASS. -- `scripts/gc_rekeyed_key_tables.py`: PASS. +- `cargo fmt --all -- --check`: PASS. +- `scripts/check_thread_locals.py --self-test`: PASS in all seven directions. +- `scripts/check_thread_locals.py`: PASS, 411 hot declarations and 273 cold + declarations in 84 recorded files, below the 768-slot hot capacity. +- `scripts/gc_rekeyed_key_tables.py`: PASS, 42 sites and 25 registered prunes + classified with zero gaps. The split child now owns the `visit_owner` + inventory entry. - `cargo test -p perry-runtime --release --lib -j4 -- --test-threads=1` via - `measure_lock.sh --build`: NOT GREEN solely because it was run detached with - a PTY. Compilation completed and 3,273 tests passed (including every new - sabotage), 4 were ignored, and the sole failure was - `tty::tests::columns_undefined_when_not_tty`, whose assertion correctly saw - the allocated PTY. Two earlier attempts stopped at compile diagnostics in - the newly extracted module; those visibility/TLS/null-pointer/type issues - were fixed before this complete run. -- The required non-PTY rerun was NOT RUN: immediately afterward `df -g /` - reported 7 GB free, below the binding 12 GB floor. Per the task rule, no - further Cargo command was started and no disk wait was attempted. -- `cargo build --release -p perry-runtime --features wasm-host -j4`: NOT RUN, - same 7 GB disk stop. -- `cargo build --release -p perry -j4`: NOT RUN, same 7 GB disk stop. + `measure_lock.sh --build`: PASS, 3,271 passed, 0 failed, 4 ignored. The three + rederivation tests tied to the reverted logs were explicitly removed; the + retained sabotage tests passed. +- `cargo build --release -p perry-runtime --features wasm-host -j4` via + `measure_lock.sh --build`: PASS. ## Predictions and exact perrymaster request -Predictions: on a zero-live steady minor, each of -`scan_descriptor_roots_mut`, `scan_closure_dynamic_props_roots_mut`, -`scan_builtin_closure_metadata_roots_mut`, `scan_template_raw_roots_mut`, and -`scan_symbol_side_table_roots_mut` is at most **0.3 ms**. Steady scanner total -is at most **5 ms**. This removes roughly 10 ms from a representative steady -minor when the five logs are empty; the phase table, not that estimate, must -name the next non-scanner lever. RSS changes should be small retained log -buffers and remain inside Ralph's allowed +1-10% band. +Prediction after the MP follow-up: the two reverted scanners return to their +measured full-walk medians or better, the retained closure, built-in closure, +and shape-cache improvements remain, and steady scanner total is at most +**14 ms**. The phase table, not an estimate, must name the next non-scanner +lever. RSS should fall slightly because the two reverted logs and their +retained buffers are gone. Exact perrymaster request: fetch pushed branch `perf/minor-phases-and-logs` and relink this runtime-only change on main's cache. Run the three required gates @@ -199,8 +224,9 @@ Then run one graceful four-turn 3300-character cc workload and one 400-character workload with `PERRY_GC_DIAG=1`, printing and preserving **every complete** `[gc-copy-minor] ran` line. The phase table for a steady minor is the -deliverable that names the next lever. Confirm all five named scanners are at -most 0.3 ms on zero-live steady minors and steady scanner total is at most 5 -ms. Finally run paired **5x3300 + 3x400** against both main and #9950's runtime, +deliverable that names the next lever. Confirm `template_raw_roots` is at most +1.83 ms, `symbol_side_table` is at most 1.74 ms, the three improved scanner +medians are unchanged, and total scanner time is at most 14 ms. Finally run +paired **5x3300 + 3x400** against both main and #9950's runtime, reporting cc turn CPU and peak RSS; target node/bun CPU parity, allowing only +1-10% RSS. diff --git a/crates/perry-runtime/src/array/header.rs b/crates/perry-runtime/src/array/header.rs index 63e1e059b5..ec23d3aa85 100644 --- a/crates/perry-runtime/src/array/header.rs +++ b/crates/perry-runtime/src/array/header.rs @@ -4,10 +4,6 @@ pub(crate) use super::header_gc_slots::*; -mod young_roots; -pub use young_roots::scan_template_raw_roots_mut; -use young_roots::{note_array_named, note_template_cache, note_template_raw}; - use std::cell::RefCell; use std::collections::HashMap; @@ -188,7 +184,6 @@ unsafe fn register_template_raw_pair(cooked: *mut ArrayHeader, raw: *mut ArrayHe if cooked.is_null() || raw.is_null() { return; } - note_template_raw(cooked as usize, raw); TEMPLATE_RAW_MAP.with(|m| { m.borrow_mut().insert(cooked as usize, raw); }); @@ -259,7 +254,6 @@ pub extern "C" fn js_tagged_template_get_or_init( mark_template_array_frozen(raw); mark_template_array_frozen(cooked); register_template_raw_pair(cooked, raw); - note_template_cache(site_id, cooked, raw); TEMPLATE_OBJECT_CACHE.with(|m| { m.borrow_mut().insert(site_id, (cooked, raw)); }); @@ -300,6 +294,33 @@ pub fn scan_template_raw_roots(mark: &mut dyn FnMut(f64)) { scan_template_raw_roots_mut(&mut visitor); } +pub fn scan_template_raw_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + TEMPLATE_OBJECT_CACHE.with(|m| { + let mut map = m.borrow_mut(); + for (_, (cooked_ptr, raw_ptr)) in map.iter_mut() { + visitor.visit_raw_mut_ptr_slot(cooked_ptr); + visitor.visit_raw_mut_ptr_slot(raw_ptr); + } + }); + TEMPLATE_RAW_MAP.with(|m| { + let mut map = m.borrow_mut(); + let mut moved = Vec::new(); + for (&cooked_addr, raw_ptr) in map.iter_mut() { + let mut new_cooked_addr = cooked_addr; + if visitor.visit_usize_slot(&mut new_cooked_addr) { + moved.push((cooked_addr, new_cooked_addr)); + } + visitor.visit_raw_mut_ptr_slot(raw_ptr); + } + for (old_addr, new_addr) in moved { + if let Some(raw_ptr) = map.remove(&old_addr) { + map.insert(new_addr, raw_ptr); + } + } + }); + scan_array_named_property_roots_mut(visitor); +} + fn barrier_array_named_props(owner: usize, props: &mut [ArrayNamedProperty]) { for prop in props.iter_mut() { crate::gc::runtime_write_barrier_external_slot( @@ -342,14 +363,32 @@ pub(crate) fn transfer_array_named_property_owner(old_owner: usize, new_owner: u ARRAY_NAMED_PROPS.with(|m| { let mut props = m.borrow_mut(); if let Some(old_props) = props.remove(&old_owner) { - for prop in &old_props { - note_array_named(new_owner, prop.value.to_bits()); - } merge_array_named_props(&mut props, new_owner, old_props); } }); } +pub(crate) fn scan_array_named_property_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + ARRAY_NAMED_PROPS.with(|m| { + let mut props = m.borrow_mut(); + let mut moved = Vec::new(); + for (&owner, owner_props) in props.iter_mut() { + let mut new_owner = owner; + if visitor.visit_metadata_usize_slot(&mut new_owner) { + moved.push((owner, new_owner)); + } + for prop in owner_props.iter_mut() { + visitor.visit_nanbox_f64_slot(&mut prop.value); + } + } + for (old_owner, new_owner) in moved { + if let Some(old_props) = props.remove(&old_owner) { + merge_array_named_props(&mut props, new_owner, old_props); + } + } + }); +} + /// Remove named-property entries whose array owners are provably dead under /// the centralized collection-specific liveness policy. pub(crate) fn prune_dead_array_named_property_owners(is_dead_owner: &dyn Fn(usize) -> bool) { @@ -366,7 +405,6 @@ pub(crate) fn test_array_named_property_owner_exists(owner: usize) -> bool { #[cfg(test)] pub(crate) fn test_clear_array_named_property_roots() { ARRAY_NAMED_PROPS.with(|m| m.borrow_mut().clear()); - young_roots::clear_named_log(); } unsafe fn string_header_as_str<'a>(key: *const crate::StringHeader) -> Option<&'a str> { @@ -407,7 +445,6 @@ pub(crate) unsafe fn array_named_property_set( }; let owner = arr as usize; note_array_named_props_ever(); - note_array_named(owner, value.to_bits()); ARRAY_NAMED_PROPS.with(|m| { let mut map = m.borrow_mut(); let props = map.entry(owner).or_default(); @@ -441,10 +478,6 @@ pub(crate) unsafe fn array_named_props_install_fresh( return; } let owner = arr as usize; - note_array_named_props_ever(); - for (_, value) in entries { - note_array_named(owner, value.to_bits()); - } ARRAY_NAMED_PROPS.with(|m| { let mut map = m.borrow_mut(); let props = map.entry(owner).or_default(); @@ -601,7 +634,6 @@ pub(crate) unsafe fn array_named_property_delete_by_name( #[cfg(test)] pub(crate) fn test_seed_template_raw_roots(cooked: *mut ArrayHeader, raw: *mut ArrayHeader) { - note_template_raw(cooked as usize, raw); TEMPLATE_RAW_MAP.with(|m| { let mut m = m.borrow_mut(); m.clear(); diff --git a/crates/perry-runtime/src/array/header/young_roots.rs b/crates/perry-runtime/src/array/header/young_roots.rs deleted file mode 100644 index 946e428c0f..0000000000 --- a/crates/perry-runtime/src/array/header/young_roots.rs +++ /dev/null @@ -1,302 +0,0 @@ -//! Young-entry logs for tagged-template and array named-property roots. - -use super::*; - -crate::perry_thread_local! { - static TEMPLATE_CACHE_YOUNG: RefCell> = - const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; - static TEMPLATE_RAW_YOUNG: RefCell> = - const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; - static ARRAY_NAMED_YOUNG: RefCell> = - const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; - #[cfg(test)] - static TEST_SUPPRESS_ARRAY_NAMED_YOUNG_NOTE: std::cell::Cell = - const { std::cell::Cell::new(false) }; - #[cfg(test)] - static TEST_SUPPRESS_TEMPLATE_RAW_YOUNG_NOTE: std::cell::Cell = - const { std::cell::Cell::new(false) }; -} - -const CACHE_LOG: &str = "array.template_object_cache"; -const RAW_LOG: &str = "array.template_raw_map"; -const NAMED_LOG: &str = "array.named_properties"; - -#[inline] -fn ptr_relevant(ptr: *mut ArrayHeader) -> bool { - crate::gc::young_log::addr_is_minor_relevant(ptr as usize) -} - -pub(super) fn note_template_cache(site: u64, cooked: *mut ArrayHeader, raw: *mut ArrayHeader) { - if ptr_relevant(cooked) || ptr_relevant(raw) { - TEMPLATE_CACHE_YOUNG.with(|log| log.borrow_mut().note(site)); - } -} - -pub(super) fn note_template_raw(cooked: usize, raw: *mut ArrayHeader) { - if crate::gc::young_log::addr_is_minor_relevant(cooked) || ptr_relevant(raw) { - #[cfg(test)] - if TEST_SUPPRESS_TEMPLATE_RAW_YOUNG_NOTE.with(std::cell::Cell::get) { - return; - } - TEMPLATE_RAW_YOUNG.with(|log| log.borrow_mut().note(cooked)); - } -} - -pub(super) fn note_array_named(owner: usize, value_bits: u64) { - if !crate::gc::young_log::addr_is_minor_collectible(owner) - && !crate::gc::young_log::bits_are_minor_relevant(value_bits) - { - return; - } - #[cfg(test)] - if TEST_SUPPRESS_ARRAY_NAMED_YOUNG_NOTE.with(std::cell::Cell::get) { - return; - } - ARRAY_NAMED_YOUNG.with(|log| log.borrow_mut().note(owner)); -} - -fn visit_cache_site(visitor: &mut crate::gc::RuntimeRootVisitor<'_>, site: u64) -> bool { - TEMPLATE_OBJECT_CACHE.with(|m| { - let mut map = m.borrow_mut(); - let Some((cooked, raw)) = map.get_mut(&site) else { - return false; - }; - visitor.visit_raw_mut_ptr_slot(cooked); - visitor.visit_raw_mut_ptr_slot(raw); - ptr_relevant(*cooked) || ptr_relevant(*raw) - }) -} - -fn visit_raw_owner(visitor: &mut crate::gc::RuntimeRootVisitor<'_>, owner: usize) -> Option { - TEMPLATE_RAW_MAP.with(|m| { - let mut map = m.borrow_mut(); - let mut raw = map.remove(&owner)?; - let mut new_owner = owner; - visitor.visit_usize_slot(&mut new_owner); - visitor.visit_raw_mut_ptr_slot(&mut raw); - map.insert(new_owner, raw); - (crate::gc::young_log::addr_is_minor_relevant(new_owner) || ptr_relevant(raw)) - .then_some(new_owner) - }) -} - -fn visit_named_owner( - visitor: &mut crate::gc::RuntimeRootVisitor<'_>, - owner: usize, -) -> Option { - ARRAY_NAMED_PROPS.with(|m| { - let mut map = m.borrow_mut(); - let mut props = map.remove(&owner)?; - let mut new_owner = owner; - visitor.visit_metadata_usize_slot(&mut new_owner); - let mut relevant = crate::gc::young_log::addr_is_minor_collectible(new_owner); - for prop in &mut props { - visitor.visit_nanbox_f64_slot(&mut prop.value); - relevant |= crate::gc::young_log::bits_are_minor_relevant(prop.value.to_bits()); - } - merge_array_named_props(&mut map, new_owner, props); - relevant.then_some(new_owner) - }) -} - -#[cfg(any(debug_assertions, test))] -fn relevant_cache_sites() -> Vec { - TEMPLATE_OBJECT_CACHE.with(|m| { - m.borrow() - .iter() - .filter_map(|(&site, &(cooked, raw))| { - (ptr_relevant(cooked) || ptr_relevant(raw)).then_some(site) - }) - .collect() - }) -} - -#[cfg(any(debug_assertions, test))] -fn relevant_raw_owners() -> Vec { - TEMPLATE_RAW_MAP.with(|m| { - m.borrow() - .iter() - .filter_map(|(&owner, &raw)| { - (crate::gc::young_log::addr_is_minor_relevant(owner) || ptr_relevant(raw)) - .then_some(owner) - }) - .collect() - }) -} - -#[cfg(any(debug_assertions, test))] -fn relevant_named_owners() -> Vec { - ARRAY_NAMED_PROPS.with(|m| { - m.borrow() - .iter() - .filter_map(|(&owner, props)| { - (crate::gc::young_log::addr_is_minor_collectible(owner) - || props.iter().any(|prop| { - crate::gc::young_log::bits_are_minor_relevant(prop.value.to_bits()) - })) - .then_some(owner) - }) - .collect() - }) -} - -fn drain_log( - log: &'static crate::tls_hot::HotKey>>, - mut visit: impl FnMut(K) -> Option, -) -> (u64, u64, u64) { - let mut logged = 0; - let mut visited = 0; - let mut kept = log.with(|log| log.borrow_mut().take_spare()); - loop { - let batch = log.with(|log| log.borrow_mut().take_sorted()); - if batch.is_empty() { - break; - } - logged += batch.len() as u64; - for key in batch { - visited += 1; - if let Some(key) = visit(key) { - kept.push(key); - } - } - } - let kept_len = kept.len() as u64; - log.with(|log| log.borrow_mut().extend(kept)); - (logged, visited, kept_len) -} - -fn report(name: &'static str, partial: bool, row: (u64, u64, u64), table_len: usize) { - crate::gc::young_log::note_walk( - name, - crate::gc::young_log::YoungLogWalk { - partial, - logged: row.0, - visited: row.1, - kept: row.2, - table_len: table_len as u64, - }, - ); -} - -pub fn scan_template_raw_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - let cache_len = TEMPLATE_OBJECT_CACHE.with(|m| m.borrow().len()); - let raw_len = TEMPLATE_RAW_MAP.with(|m| m.borrow().len()); - let named_len = ARRAY_NAMED_PROPS.with(|m| m.borrow().len()); - if visitor.young_scope() { - #[cfg(any(debug_assertions, test))] - { - TEMPLATE_CACHE_YOUNG.with(|log| { - log.borrow() - .debug_assert_logged(CACHE_LOG, &relevant_cache_sites()) - }); - TEMPLATE_RAW_YOUNG.with(|log| { - log.borrow() - .debug_assert_logged(RAW_LOG, &relevant_raw_owners()) - }); - ARRAY_NAMED_YOUNG.with(|log| { - log.borrow() - .debug_assert_logged(NAMED_LOG, &relevant_named_owners()) - }); - } - let cache = drain_log(&TEMPLATE_CACHE_YOUNG, |site| { - visit_cache_site(visitor, site).then_some(site) - }); - let raw = drain_log(&TEMPLATE_RAW_YOUNG, |owner| visit_raw_owner(visitor, owner)); - let named = drain_log(&ARRAY_NAMED_YOUNG, |owner| { - visit_named_owner(visitor, owner) - }); - report(CACHE_LOG, true, cache, cache_len); - report(RAW_LOG, true, raw, raw_len); - report(NAMED_LOG, true, named, named_len); - return; - } - - let cache_sites: Vec = - TEMPLATE_OBJECT_CACHE.with(|m| m.borrow().keys().copied().collect()); - let raw_owners: Vec = TEMPLATE_RAW_MAP.with(|m| m.borrow().keys().copied().collect()); - let named_owners: Vec = ARRAY_NAMED_PROPS.with(|m| m.borrow().keys().copied().collect()); - let _ = TEMPLATE_CACHE_YOUNG.with(|log| log.borrow_mut().take_sorted()); - let _ = TEMPLATE_RAW_YOUNG.with(|log| log.borrow_mut().take_sorted()); - let _ = ARRAY_NAMED_YOUNG.with(|log| log.borrow_mut().take_sorted()); - // `drain_log` consumes the log, so seed it with the authoritative keys. - TEMPLATE_CACHE_YOUNG.with(|log| log.borrow_mut().extend(cache_sites)); - let cache = drain_log(&TEMPLATE_CACHE_YOUNG, |site| { - visit_cache_site(visitor, site).then_some(site) - }); - TEMPLATE_RAW_YOUNG.with(|log| log.borrow_mut().extend(raw_owners)); - let raw = drain_log(&TEMPLATE_RAW_YOUNG, |owner| visit_raw_owner(visitor, owner)); - ARRAY_NAMED_YOUNG.with(|log| log.borrow_mut().extend(named_owners)); - let named = drain_log(&ARRAY_NAMED_YOUNG, |owner| { - visit_named_owner(visitor, owner) - }); - report(CACHE_LOG, false, cache, cache_len); - report(RAW_LOG, false, raw, raw_len); - report(NAMED_LOG, false, named, named_len); -} - -#[cfg(test)] -pub(super) fn clear_named_log() { - ARRAY_NAMED_YOUNG.with(|log| log.borrow_mut().clear()); -} - -#[cfg(test)] -mod tests { - use super::*; - - fn alloc_empty_array() -> *mut ArrayHeader { - let arr = crate::arena::arena_alloc_gc( - std::mem::size_of::(), - std::mem::align_of::(), - crate::gc::GC_TYPE_ARRAY, - ) as *mut ArrayHeader; - unsafe { - (*arr).length = 0; - (*arr).capacity = 0; - } - arr - } - - #[test] - fn template_raw_log_rederivation_rejects_a_suppressed_writer() { - let _lock = crate::gc::global_side_table_test_lock(); - let cooked = alloc_empty_array(); - let raw = alloc_empty_array(); - TEST_SUPPRESS_TEMPLATE_RAW_YOUNG_NOTE.with(|flag| flag.set(true)); - test_seed_template_raw_roots(cooked, raw); - TEST_SUPPRESS_TEMPLATE_RAW_YOUNG_NOTE.with(|flag| flag.set(false)); - let missed = std::panic::catch_unwind(|| { - TEMPLATE_RAW_YOUNG.with(|log| { - log.borrow() - .debug_assert_logged(RAW_LOG, &relevant_raw_owners()) - }); - }); - TEMPLATE_RAW_MAP.with(|m| m.borrow_mut().clear()); - TEMPLATE_RAW_YOUNG.with(|log| log.borrow_mut().clear()); - assert!( - missed.is_err(), - "sabotage: suppressing the template-raw writer's note must trip completeness" - ); - } - - #[test] - fn array_named_log_rederivation_rejects_a_suppressed_setter() { - let _lock = crate::gc::global_side_table_test_lock(); - let arr = alloc_empty_array(); - let key = crate::string::js_string_from_bytes(b"sabotage".as_ptr(), 8); - TEST_SUPPRESS_ARRAY_NAMED_YOUNG_NOTE.with(|flag| flag.set(true)); - unsafe { array_named_property_set(arr, key, 7.0) }; - TEST_SUPPRESS_ARRAY_NAMED_YOUNG_NOTE.with(|flag| flag.set(false)); - let missed = std::panic::catch_unwind(|| { - ARRAY_NAMED_YOUNG.with(|log| { - log.borrow() - .debug_assert_logged(NAMED_LOG, &relevant_named_owners()) - }); - }); - ARRAY_NAMED_PROPS.with(|m| m.borrow_mut().remove(&(arr as usize))); - clear_named_log(); - assert!( - missed.is_err(), - "sabotage: suppressing array_named_property_set's note must trip completeness" - ); - } -} diff --git a/crates/perry-runtime/src/object/native_module/callable_exports/builtin_closure_metadata.rs b/crates/perry-runtime/src/object/native_module/callable_exports/builtin_closure_metadata.rs index fe1a2f75c6..a3d1ec5ee3 100644 --- a/crates/perry-runtime/src/object/native_module/callable_exports/builtin_closure_metadata.rs +++ b/crates/perry-runtime/src/object/native_module/callable_exports/builtin_closure_metadata.rs @@ -5,9 +5,15 @@ thread_local! { std::cell::RefCell::new(crate::fast_hash::new_ptr_hash_map()); static BUILTIN_CLOSURE_NON_CONSTRUCTABLE: std::cell::RefCell> = std::cell::RefCell::new(crate::fast_hash::new_ptr_hash_set()); +} + +crate::perry_thread_local! { static BUILTIN_CLOSURE_YOUNG: std::cell::RefCell> = const { std::cell::RefCell::new(crate::gc::young_log::YoungLog::new()) }; - #[cfg(test)] +} + +#[cfg(test)] +thread_local! { static TEST_SUPPRESS_BUILTIN_CLOSURE_YOUNG_NOTE: std::cell::Cell = const { std::cell::Cell::new(false) }; } diff --git a/crates/perry-runtime/src/symbol.rs b/crates/perry-runtime/src/symbol.rs index 767c43d240..6725d3a010 100644 --- a/crates/perry-runtime/src/symbol.rs +++ b/crates/perry-runtime/src/symbol.rs @@ -584,7 +584,6 @@ pub(crate) fn register_symbol_pointer(ptr: usize) { SYMBOL_EVER_REGISTERED.arm(); // Admit before the insert, for the same reason. admit_symbol_pointer(ptr); - gc_roots::note_symbol_pointer(ptr); let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); if guard.is_none() { *guard = Some(new_ptr_hash_set()); @@ -1037,7 +1036,6 @@ pub(crate) fn store_object_symbol_property_root( value_bits: u64, ) -> bool { note_symbol_key_installed(sym_key); - gc_roots::note_symbol_property_root(obj_key, sym_key, value_bits); { let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); if guard.is_none() { @@ -1072,7 +1070,6 @@ pub(crate) static CLASS_STATIC_SYMBOLS_LATCH: crate::registry_latch::RegistryLat pub(crate) fn store_class_static_symbol_root(class_id: u32, sym_key: usize, value_bits: u64) { note_symbol_key_installed(sym_key); - gc_roots::note_class_static_symbol(class_id, sym_key, value_bits); CLASS_STATIC_SYMBOLS_LATCH.arm(); let symbol_id = unsafe { (*(sym_key as *const SymbolHeader)).id }; let created; diff --git a/crates/perry-runtime/src/symbol/accessors.rs b/crates/perry-runtime/src/symbol/accessors.rs index 9fb977e553..9ceeddbe3c 100644 --- a/crates/perry-runtime/src/symbol/accessors.rs +++ b/crates/perry-runtime/src/symbol/accessors.rs @@ -68,7 +68,6 @@ pub(crate) fn test_symbol_accessor_property_count() -> usize { #[cfg(test)] pub(crate) fn test_seed_symbol_accessor_property(obj_key: usize, sym_key: usize, get_bits: u64) { - super::gc_roots::note_symbol_accessor(obj_key, sym_key, get_bits, TAG_UNDEFINED); let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_ACCESSOR_PROPERTIES); guard.get_or_insert_with(HashMap::new).insert( (obj_key, sym_key), @@ -91,8 +90,6 @@ pub(crate) unsafe fn set_symbol_accessor_property( return; } crate::symbol::note_symbol_key_installed(sym_key); - super::gc_roots::note_symbol_property_root(obj_key, sym_key, crate::value::TAG_UNDEFINED); - super::gc_roots::note_symbol_accessor(obj_key, sym_key, get_bits, set_bits); { // `SYMBOL_PROPERTIES` is the only insertion-ordered record of symbol // property CREATION order, which `[[OwnPropertyKeys]]` must report @@ -222,29 +219,6 @@ pub(super) fn accessor_property_keys() -> Vec<(usize, usize)> { .unwrap_or_default() } -pub(super) fn accessor_property_count() -> usize { - let guard = crate::gc::lock_gc_root_registry(&SYMBOL_ACCESSOR_PROPERTIES); - guard.as_ref().map_or(0, HashMap::len) -} - -pub(super) fn relevant_accessor_property_keys() -> Vec<(usize, usize)> { - let guard = crate::gc::lock_gc_root_registry(&SYMBOL_ACCESSOR_PROPERTIES); - guard - .as_ref() - .map(|map| { - map.iter() - .filter_map(|(&(owner, sym_key), acc)| { - (crate::gc::young_log::addr_is_minor_collectible(owner) - || crate::gc::young_log::addr_is_minor_relevant(sym_key) - || crate::gc::young_log::bits_are_minor_relevant(acc.get) - || crate::gc::young_log::bits_are_minor_relevant(acc.set)) - .then_some((owner, sym_key)) - }) - .collect() - }) - .unwrap_or_default() -} - /// Step twin of `scan_symbol_accessor_roots_mut` for one snapshot key: /// strong-visits the get/set closures and rekeys owner/sym on a move. /// Cycle-based collections run ONLY the step scanner, so before this @@ -254,13 +228,13 @@ pub(super) fn scan_symbol_accessor_root_slot( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, owner: usize, sym_key: usize, -) -> Option<(usize, usize)> { +) { let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_ACCESSOR_PROPERTIES); let Some(map) = guard.as_mut() else { - return None; + return; }; let Some(acc) = map.get_mut(&(owner, sym_key)) else { - return None; + return; }; let mut new_owner = owner; let mut new_sym_key = sym_key; @@ -277,21 +251,6 @@ pub(super) fn scan_symbol_accessor_root_slot( map.insert((new_owner, new_sym_key), acc); } } - symbol_accessor_root_relevant_in(map, new_owner, new_sym_key) - .then_some((new_owner, new_sym_key)) -} - -fn symbol_accessor_root_relevant_in( - map: &HashMap<(usize, usize), SymbolAccessorDescriptor>, - owner: usize, - sym_key: usize, -) -> bool { - map.get(&(owner, sym_key)).is_some_and(|acc| { - crate::gc::young_log::addr_is_minor_collectible(owner) - || crate::gc::young_log::addr_is_minor_relevant(sym_key) - || crate::gc::young_log::bits_are_minor_relevant(acc.get) - || crate::gc::young_log::bits_are_minor_relevant(acc.set) - }) } pub(super) fn has_own_symbol_accessor(obj_key: usize, sym_key: usize) -> bool { diff --git a/crates/perry-runtime/src/symbol/gc_roots.rs b/crates/perry-runtime/src/symbol/gc_roots.rs index 7fec611384..8c05be3014 100644 --- a/crates/perry-runtime/src/symbol/gc_roots.rs +++ b/crates/perry-runtime/src/symbol/gc_roots.rs @@ -21,16 +21,11 @@ pub fn scan_symbol_side_table_roots(mark: &mut dyn FnMut(f64)) { } pub fn scan_symbol_side_table_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - if visitor.young_scope() { - scan_young_symbol_side_table_roots_mut(visitor); - return; - } scan_symbol_property_roots_mut(visitor); scan_symbol_property_attrs_mut(visitor); accessors::scan_symbol_accessor_roots_mut(visitor); scan_class_static_symbol_roots_mut(visitor); scan_symbol_pointer_metadata_roots_mut(visitor); - rebuild_symbol_young_log(); } fn scan_symbol_property_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { @@ -136,7 +131,7 @@ fn scan_symbol_pointer_metadata_roots_mut(visitor: &mut crate::gc::RuntimeRootVi } } -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Copy)] enum SymbolSideTableRootSlot { SymbolPropertyOwner { owner: usize }, SymbolPropertyEntry { owner: usize, sym_key: usize }, @@ -146,83 +141,15 @@ enum SymbolSideTableRootSlot { SymbolPointer { ptr: usize }, } -crate::perry_thread_local! { - static SYMBOL_SIDE_TABLE_YOUNG: RefCell> = - const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; - #[cfg(test)] - static TEST_SUPPRESS_SYMBOL_YOUNG_NOTE: std::cell::Cell = - const { std::cell::Cell::new(false) }; -} - -const SYMBOL_YOUNG_LOG_NAME: &str = "symbol.side_tables"; - -#[inline] -fn note_symbol_slot(slot: SymbolSideTableRootSlot) { - #[cfg(test)] - if TEST_SUPPRESS_SYMBOL_YOUNG_NOTE.with(std::cell::Cell::get) { - return; - } - SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().note(slot)); -} - -pub(super) fn note_symbol_property_root(owner: usize, sym_key: usize, value_bits: u64) { - if crate::gc::young_log::addr_is_minor_collectible(owner) { - note_symbol_slot(SymbolSideTableRootSlot::SymbolPropertyOwner { owner }); - } - if crate::gc::young_log::addr_is_minor_relevant(sym_key) - || crate::gc::young_log::bits_are_minor_relevant(value_bits) - { - note_symbol_slot(SymbolSideTableRootSlot::SymbolPropertyEntry { owner, sym_key }); - } -} - -pub(super) fn note_symbol_property_attrs(owner: usize, sym_key: usize) { - if crate::gc::young_log::addr_is_minor_collectible(owner) - || crate::gc::young_log::addr_is_minor_relevant(sym_key) - { - note_symbol_slot(SymbolSideTableRootSlot::SymbolPropertyAttrs { owner, sym_key }); - } -} - -pub(super) fn note_symbol_accessor(owner: usize, sym_key: usize, get_bits: u64, set_bits: u64) { - if crate::gc::young_log::addr_is_minor_collectible(owner) - || crate::gc::young_log::addr_is_minor_relevant(sym_key) - || crate::gc::young_log::bits_are_minor_relevant(get_bits) - || crate::gc::young_log::bits_are_minor_relevant(set_bits) - { - note_symbol_slot(SymbolSideTableRootSlot::SymbolAccessorProperty { owner, sym_key }); - } -} - -pub(super) fn note_class_static_symbol(class_id: u32, sym_key: usize, value_bits: u64) { - if crate::gc::young_log::addr_is_minor_relevant(sym_key) - || crate::gc::young_log::bits_are_minor_relevant(value_bits) - { - note_symbol_slot(SymbolSideTableRootSlot::ClassStaticSymbol { class_id, sym_key }); - } -} - -pub(super) fn note_symbol_pointer(ptr: usize) { - if crate::gc::young_log::addr_is_minor_collectible(ptr) { - note_symbol_slot(SymbolSideTableRootSlot::SymbolPointer { ptr }); - } -} - pub(crate) struct SymbolSideTableRootScanState { - slots: Option>, - kept: Vec, + slots: Vec, cursor: usize, - young: bool, - table_len: usize, } pub(crate) fn new_symbol_side_table_root_scan_state() -> Box { Box::new(SymbolSideTableRootScanState { - slots: None, - kept: Vec::new(), + slots: symbol_side_table_root_snapshot(), cursor: 0, - young: false, - table_len: 0, }) } @@ -234,49 +161,12 @@ pub(crate) fn scan_symbol_side_table_roots_mut_step( let state = state .downcast_mut::() .expect("symbol side-table root scanner state type"); - if state.slots.is_none() { - state.young = visitor.young_scope(); - if state.young { - state.table_len = symbol_side_table_root_len(); - #[cfg(any(debug_assertions, test))] - SYMBOL_SIDE_TABLE_YOUNG.with(|log| { - log.borrow() - .debug_assert_logged(SYMBOL_YOUNG_LOG_NAME, &relevant_symbol_slots()) - }); - state.slots = Some(SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().take_sorted())); - } else { - let authoritative = symbol_side_table_root_snapshot(); - state.table_len = authoritative.len(); - let _ = SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().take_sorted()); - state.slots = Some(authoritative); - } - } - let slots = state.slots.as_ref().expect("symbol slots initialized"); - while *remaining > 0 && state.cursor < slots.len() { - if let Some(slot) = scan_symbol_side_table_root_slot(visitor, slots[state.cursor]) { - state.kept.push(slot); - } + while *remaining > 0 && state.cursor < state.slots.len() { + scan_symbol_side_table_root_slot(visitor, state.slots[state.cursor]); state.cursor += 1; *remaining -= 1; } - let done = state.cursor >= slots.len(); - if done { - let logged = slots.len() as u64; - let kept = std::mem::take(&mut state.kept); - let kept_len = kept.len() as u64; - SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().extend(kept)); - crate::gc::young_log::note_walk( - SYMBOL_YOUNG_LOG_NAME, - crate::gc::young_log::YoungLogWalk { - partial: state.young, - logged, - visited: state.cursor as u64, - kept: kept_len, - table_len: state.table_len as u64, - }, - ); - } - done + state.cursor >= state.slots.len() } fn symbol_side_table_root_snapshot() -> Vec { @@ -328,163 +218,13 @@ fn symbol_side_table_root_snapshot() -> Vec { slots } -fn symbol_side_table_root_len() -> usize { - // Exact only for diagnostics: counting the property vectors is itself a - // whole-table walk, which the release minor must not pay merely to report - // how much work it skipped. - if !crate::gc::gc_diag_enabled() { - return 0; - } - let properties = { - let guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); - guard.as_ref().map_or(0, |map| { - map.len() + map.values().map(Vec::len).sum::() - }) - }; - let attrs = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTY_ATTRS) - .as_ref() - .map_or(0, |map| map.len()); - let class_statics = crate::gc::lock_gc_root_registry(&CLASS_STATIC_SYMBOLS) - .as_ref() - .map_or(0, |map| map.len()); - let pointers = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS) - .as_ref() - .map_or(0, |set| set.len()); - properties + attrs + accessors::accessor_property_count() + class_statics + pointers -} - -fn collect_relevant_symbol_slots() -> Vec { - let mut slots = Vec::new(); - { - let guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); - if let Some(map) = guard.as_ref() { - for (&owner, entries) in map { - if crate::gc::young_log::addr_is_minor_collectible(owner) { - slots.push(SymbolSideTableRootSlot::SymbolPropertyOwner { owner }); - } - for &(sym_key, value_bits) in entries { - if crate::gc::young_log::addr_is_minor_relevant(sym_key) - || crate::gc::young_log::bits_are_minor_relevant(value_bits) - { - slots.push(SymbolSideTableRootSlot::SymbolPropertyEntry { owner, sym_key }); - } - } - } - } - } - { - let guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTY_ATTRS); - if let Some(map) = guard.as_ref() { - slots.extend(map.keys().filter_map(|&(owner, sym_key)| { - (crate::gc::young_log::addr_is_minor_collectible(owner) - || crate::gc::young_log::addr_is_minor_relevant(sym_key)) - .then_some(SymbolSideTableRootSlot::SymbolPropertyAttrs { owner, sym_key }) - })); - } - } - slots.extend( - accessors::relevant_accessor_property_keys() - .into_iter() - .map( - |(owner, sym_key)| SymbolSideTableRootSlot::SymbolAccessorProperty { - owner, - sym_key, - }, - ), - ); - { - let guard = crate::gc::lock_gc_root_registry(&CLASS_STATIC_SYMBOLS); - if let Some(map) = guard.as_ref() { - slots.extend( - map.iter() - .filter_map(|(&(class_id, sym_key), &value_bits)| { - (crate::gc::young_log::addr_is_minor_relevant(sym_key) - || crate::gc::young_log::bits_are_minor_relevant(value_bits)) - .then_some(SymbolSideTableRootSlot::ClassStaticSymbol { class_id, sym_key }) - }), - ); - } - } - { - let guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); - if let Some(set) = guard.as_ref() { - slots.extend(set.iter().filter_map(|&ptr| { - crate::gc::young_log::addr_is_minor_collectible(ptr) - .then_some(SymbolSideTableRootSlot::SymbolPointer { ptr }) - })); - } - } - slots -} - -#[cfg(any(debug_assertions, test))] -fn relevant_symbol_slots() -> Vec { - collect_relevant_symbol_slots() -} - -fn rebuild_symbol_young_log() { - let table_len = symbol_side_table_root_len(); - let relevant = collect_relevant_symbol_slots(); - let _ = SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().take_sorted()); - let kept = relevant.len() as u64; - SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().extend(relevant)); - crate::gc::young_log::note_walk( - SYMBOL_YOUNG_LOG_NAME, - crate::gc::young_log::YoungLogWalk { - partial: false, - logged: table_len as u64, - visited: table_len as u64, - kept, - table_len: table_len as u64, - }, - ); -} - -fn scan_young_symbol_side_table_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - let table_len = symbol_side_table_root_len(); - #[cfg(any(debug_assertions, test))] - SYMBOL_SIDE_TABLE_YOUNG.with(|log| { - log.borrow() - .debug_assert_logged(SYMBOL_YOUNG_LOG_NAME, &relevant_symbol_slots()) - }); - let mut kept = SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().take_spare()); - let mut logged = 0_u64; - loop { - let batch = SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().take_sorted()); - if batch.is_empty() { - break; - } - logged += batch.len() as u64; - for slot in batch { - if let Some(slot) = scan_symbol_side_table_root_slot(visitor, slot) { - kept.push(slot); - } - } - } - let kept_len = kept.len() as u64; - SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().extend(kept)); - crate::gc::young_log::note_walk( - SYMBOL_YOUNG_LOG_NAME, - crate::gc::young_log::YoungLogWalk { - partial: true, - logged, - visited: logged, - kept: kept_len, - table_len: table_len as u64, - }, - ); -} - fn scan_symbol_side_table_root_slot( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, slot: SymbolSideTableRootSlot, -) -> Option { +) { match slot { SymbolSideTableRootSlot::SymbolPropertyOwner { owner } => { - rewrite_symbol_property_owner_if_forwarded(visitor, owner).and_then(|owner| { - crate::gc::young_log::addr_is_minor_collectible(owner) - .then_some(SymbolSideTableRootSlot::SymbolPropertyOwner { owner }) - }) + rewrite_symbol_property_owner_if_forwarded(visitor, owner); } SymbolSideTableRootSlot::SymbolPropertyEntry { owner, sym_key } => { // The preceding budget slice may already have rekeyed this @@ -495,7 +235,7 @@ fn scan_symbol_side_table_root_slot( visitor.visit_metadata_usize_slot(&mut healed_owner); let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); let Some(map) = guard.as_mut() else { - return None; + return; }; let lookup_owner = if map.contains_key(&healed_owner) { healed_owner @@ -506,43 +246,22 @@ fn scan_symbol_side_table_root_slot( .get_mut(&lookup_owner) .and_then(|entries| entries.iter_mut().find(|entry| entry.0 == sym_key)) else { - return None; + return; }; visitor.visit_usize_slot(entry_sym); visitor.visit_nanbox_u64_slot(value_bits); - (crate::gc::young_log::addr_is_minor_relevant(*entry_sym) - || crate::gc::young_log::bits_are_minor_relevant(*value_bits)) - .then_some(SymbolSideTableRootSlot::SymbolPropertyEntry { - owner: lookup_owner, - sym_key: *entry_sym, - }) } SymbolSideTableRootSlot::SymbolAccessorProperty { owner, sym_key } => { - accessors::scan_symbol_accessor_root_slot(visitor, owner, sym_key).map( - |(owner, sym_key)| SymbolSideTableRootSlot::SymbolAccessorProperty { - owner, - sym_key, - }, - ) + accessors::scan_symbol_accessor_root_slot(visitor, owner, sym_key); } SymbolSideTableRootSlot::SymbolPropertyAttrs { owner, sym_key } => { - rewrite_symbol_property_attrs_if_forwarded(visitor, owner, sym_key).and_then( - |(owner, sym_key)| { - (crate::gc::young_log::addr_is_minor_collectible(owner) - || crate::gc::young_log::addr_is_minor_relevant(sym_key)) - .then_some(SymbolSideTableRootSlot::SymbolPropertyAttrs { owner, sym_key }) - }, - ) + rewrite_symbol_property_attrs_if_forwarded(visitor, owner, sym_key); } SymbolSideTableRootSlot::ClassStaticSymbol { class_id, sym_key } => { - rewrite_class_static_symbol_entry_if_forwarded(visitor, class_id, sym_key) - .map(|sym_key| SymbolSideTableRootSlot::ClassStaticSymbol { class_id, sym_key }) + rewrite_class_static_symbol_entry_if_forwarded(visitor, class_id, sym_key); } SymbolSideTableRootSlot::SymbolPointer { ptr } => { - rewrite_symbol_pointer_metadata_if_forwarded(visitor, ptr).and_then(|ptr| { - crate::gc::young_log::addr_is_minor_collectible(ptr) - .then_some(SymbolSideTableRootSlot::SymbolPointer { ptr }) - }) + rewrite_symbol_pointer_metadata_if_forwarded(visitor, ptr); } } } @@ -550,41 +269,37 @@ fn scan_symbol_side_table_root_slot( fn rewrite_symbol_property_owner_if_forwarded( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, owner: usize, -) -> Option { +) { let mut new_owner = owner; - if visitor.visit_metadata_usize_slot(&mut new_owner) && new_owner != owner { - let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); - if let Some(map) = guard.as_mut() { - if let Some(entries) = map.remove(&owner) { - match map.entry(new_owner) { - std::collections::hash_map::Entry::Occupied(mut entry) => { - merge_symbol_property_entries(entry.get_mut(), entries); - } - std::collections::hash_map::Entry::Vacant(entry) => { - entry.insert(entries); - } + if !visitor.visit_metadata_usize_slot(&mut new_owner) || new_owner == owner { + return; + } + let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); + if let Some(map) = guard.as_mut() { + if let Some(entries) = map.remove(&owner) { + match map.entry(new_owner) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + merge_symbol_property_entries(entry.get_mut(), entries); + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(entries); } } } } - let guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES); - guard - .as_ref() - .is_some_and(|map| map.contains_key(&new_owner)) - .then_some(new_owner) } fn rewrite_symbol_property_attrs_if_forwarded( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, owner: usize, sym_key: usize, -) -> Option<(usize, usize)> { +) { let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTY_ATTRS); let Some(map) = guard.as_mut() else { - return None; + return; }; if !map.contains_key(&(owner, sym_key)) { - return None; + return; } let mut new_owner = owner; let mut new_sym_key = sym_key; @@ -595,21 +310,19 @@ fn rewrite_symbol_property_attrs_if_forwarded( map.insert((new_owner, new_sym_key), attrs); } } - map.contains_key(&(new_owner, new_sym_key)) - .then_some((new_owner, new_sym_key)) } fn rewrite_class_static_symbol_entry_if_forwarded( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, class_id: u32, sym_key: usize, -) -> Option { +) { let mut guard = crate::gc::lock_gc_root_registry(&CLASS_STATIC_SYMBOLS); let Some(map) = guard.as_mut() else { - return None; + return; }; let Some(value_bits) = map.get_mut(&(class_id, sym_key)) else { - return None; + return; }; let mut new_sym_key = sym_key; let moved = visitor.visit_usize_slot(&mut new_sym_key); @@ -619,37 +332,27 @@ fn rewrite_class_static_symbol_entry_if_forwarded( map.insert((class_id, new_sym_key), value_bits); } } - map.get(&(class_id, new_sym_key)).and_then(|value_bits| { - (crate::gc::young_log::addr_is_minor_relevant(new_sym_key) - || crate::gc::young_log::bits_are_minor_relevant(*value_bits)) - .then_some(new_sym_key) - }) } fn rewrite_symbol_pointer_metadata_if_forwarded( visitor: &mut crate::gc::RuntimeRootVisitor<'_>, ptr: usize, -) -> Option { +) { let mut new_ptr = ptr; - if visitor.visit_metadata_usize_slot(&mut new_ptr) && new_ptr != ptr { - let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); - if let Some(set) = guard.as_mut() { - set.remove(&ptr); - if new_ptr != 0 { - insert_symbol_pointer_in_set(set, new_ptr); - } + if !visitor.visit_metadata_usize_slot(&mut new_ptr) || new_ptr == ptr { + return; + } + let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); + if let Some(set) = guard.as_mut() { + set.remove(&ptr); + if new_ptr != 0 { + insert_symbol_pointer_in_set(set, new_ptr); } } - let guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); - guard - .as_ref() - .is_some_and(|set| set.contains(&new_ptr)) - .then_some(new_ptr) } #[cfg(test)] pub(crate) fn test_clear_symbol_side_table_roots() { - SYMBOL_SIDE_TABLE_YOUNG.with(|log| log.borrow_mut().clear()); *crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTIES) = None; *crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTY_ATTRS) = None; *crate::gc::lock_gc_root_registry(&CLASS_STATIC_SYMBOLS) = None; @@ -676,7 +379,6 @@ pub(crate) fn test_clear_symbol_side_table_roots() { } else { let mut set = new_ptr_hash_set(); for ptr in persistent { - note_symbol_pointer(ptr); insert_symbol_pointer_in_set(&mut set, ptr); } *guard = Some(set); @@ -723,7 +425,6 @@ pub(crate) fn test_seed_class_static_symbol_root(class_id: u32, sym_key: usize, // an unaligned sentinel. Seed only the root table they exercise; // production registration additionally reads SymbolHeader::id for // [[OwnPropertyKeys]] ordering and therefore requires a real Symbol. - note_class_static_symbol(class_id, sym_key, value_bits); CLASS_STATIC_SYMBOLS_LATCH.arm(); let mut guard = crate::gc::lock_gc_root_registry(&CLASS_STATIC_SYMBOLS); if guard.is_none() { @@ -773,31 +474,3 @@ pub(crate) fn test_symbol_pointer_root_contains(ptr: usize) -> bool { let guard = crate::gc::lock_gc_root_registry(&SYMBOL_POINTERS); guard.as_ref().is_some_and(|set| set.contains(&ptr)) } - -#[cfg(test)] -mod young_log_sabotage_tests { - use super::*; - - #[test] - fn symbol_log_rederivation_rejects_a_suppressed_property_writer() { - let _lock = crate::gc::global_side_table_test_lock(); - test_clear_symbol_side_table_roots(); - let owner = crate::object::js_object_alloc(0, 0) as usize; - let sym_bits = unsafe { crate::symbol::js_symbol_new_empty() }.to_bits(); - let sym_key = (sym_bits & POINTER_MASK) as usize; - TEST_SUPPRESS_SYMBOL_YOUNG_NOTE.with(|flag| flag.set(true)); - store_object_symbol_property_root(owner, sym_key, 7.0_f64.to_bits()); - TEST_SUPPRESS_SYMBOL_YOUNG_NOTE.with(|flag| flag.set(false)); - let missed = std::panic::catch_unwind(|| { - SYMBOL_SIDE_TABLE_YOUNG.with(|log| { - log.borrow() - .debug_assert_logged(SYMBOL_YOUNG_LOG_NAME, &relevant_symbol_slots()) - }); - }); - test_clear_symbol_side_table_roots(); - assert!( - missed.is_err(), - "sabotage: suppressing the property-store note must trip completeness" - ); - } -} diff --git a/crates/perry-runtime/src/symbol/properties.rs b/crates/perry-runtime/src/symbol/properties.rs index 49adaada16..ddcbcf4f32 100644 --- a/crates/perry-runtime/src/symbol/properties.rs +++ b/crates/perry-runtime/src/symbol/properties.rs @@ -90,7 +90,6 @@ pub(crate) fn set_symbol_property_attrs( return; } super::note_symbol_key_installed(sym_key); - super::gc_roots::note_symbol_property_attrs(owner, sym_key); let mut guard = crate::gc::lock_gc_root_registry(&SYMBOL_PROPERTY_ATTRS); if guard.is_none() { *guard = Some(crate::fast_hash::new_fast_key_hash_map()); diff --git a/scripts/gc_rekeyed_key_tables.json b/scripts/gc_rekeyed_key_tables.json index b31756956e..09687911b2 100644 --- a/scripts/gc_rekeyed_key_tables.json +++ b/scripts/gc_rekeyed_key_tables.json @@ -104,10 +104,10 @@ "why": "#8192: registered prune drops the whole cache entry when either weak half (prev_keys keys-array, key_ptr interned string) is dead; next_keys is a strong root and cannot be. #9754: the per-slot body shared by the full walk and the young-log walk." }, { - "site": "crates/perry-runtime/src/object/native_module/callable_exports.rs::scan_builtin_closure_metadata_roots_mut", + "site": "crates/perry-runtime/src/object/native_module/callable_exports/builtin_closure_metadata.rs::visit_owner", "table": "BUILTIN_CLOSURE_LENGTH / BUILTIN_CLOSURE_NON_CONSTRUCTABLE", "death": "dead_owner:prune_dead_builtin_closure_metadata_owners", - "why": "#8393: both tables are retained on the GC_TYPE_CLOSURE-narrowed predicate over their closure-address keys." + "why": "#8393: both tables are retained on the GC_TYPE_CLOSURE-narrowed predicate over their closure-address keys. The extracted visit_owner helper is the shared rekey path for the full and young-log walks." }, { "site": "crates/perry-runtime/src/object/shapes.rs::scan_shape_table_rekey_mut", diff --git a/scripts/thread_local_cold_allowlist.json b/scripts/thread_local_cold_allowlist.json index b27978e5db..e566b4aa29 100644 --- a/scripts/thread_local_cold_allowlist.json +++ b/scripts/thread_local_cold_allowlist.json @@ -1,6 +1,6 @@ { "_comment": "Files still declaring raw `thread_local!`. The count is the number of DECLARATIONS that survive into a shipping build \u2014 each one pays `_tlv_get_addr` per read on Darwin \u2014 and it is a ratchet, so adding a `static` to an already-listed file fails whether or not it opens a new block. New code should use `crate::perry_thread_local!` \u2014 see crates/perry-runtime/src/tls_hot.rs. Regenerate with scripts/check_thread_locals.py --update.", - "_hot_declarations": 405, + "_hot_declarations": 411, "files": { "crates/perry-runtime/src/agent.rs": 1, "crates/perry-runtime/src/arena/block.rs": 5, @@ -64,7 +64,8 @@ "crates/perry-runtime/src/node_submodules/test_once_unit_tests.rs": 2, "crates/perry-runtime/src/node_submodules/test_property.rs": 1, "crates/perry-runtime/src/node_submodules/trace_events.rs": 8, - "crates/perry-runtime/src/object/native_module/callable_exports.rs": 3, + "crates/perry-runtime/src/object/native_module/callable_exports.rs": 1, + "crates/perry-runtime/src/object/native_module/callable_exports/builtin_closure_metadata.rs": 2, "crates/perry-runtime/src/object/spill.rs": 1, "crates/perry-runtime/src/os/os_process_emitter.rs": 1, "crates/perry-runtime/src/os_process_streams.rs": 3, From 635fab2a7ad8e258f1d2e6a9b107f997f1583d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 15:24:16 +0200 Subject: [PATCH 22/27] perf(gc): prune per-object layout tables from a young-entry log on a minor A minor can only remove nursery owners, so walk the per-object layout tables' young-entry log rather than every standing key. Full collections retain the whole-table prune and rebuild the log from survivors. Replay 19a6cd201 on the #9957 phase-instrument tree. Its runtime hunks are unchanged; the young-log test file keeps both the newer fixed-cost scanner tests and the replayed layout-prune tests. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit dd279a8ba2de6b86a044d1074955e87fccf76d53) --- changelog.d/9841-layout-prune-young-log.md | 43 ++++ crates/perry-runtime/src/gc/dead_owner.rs | 2 +- crates/perry-runtime/src/gc/layout.rs | 16 +- crates/perry-runtime/src/gc/layout_tables.rs | 230 +++++++++++++++++- .../src/gc/tests/young_log_tests.rs | 131 ++++++++++ crates/perry-runtime/src/gc/young_log.rs | 7 +- 6 files changed, 417 insertions(+), 12 deletions(-) create mode 100644 changelog.d/9841-layout-prune-young-log.md diff --git a/changelog.d/9841-layout-prune-young-log.md b/changelog.d/9841-layout-prune-young-log.md new file mode 100644 index 0000000000..5804e50beb --- /dev/null +++ b/changelog.d/9841-layout-prune-young-log.md @@ -0,0 +1,43 @@ +**A minor's per-object layout death-prune now walks a young-entry log instead +of both tables** — on a compiled claude-code streamed reply it visited +**6,792,375 entries where at most 125,367 (1.85 %) could possibly have died, +and on 37 % of minors nothing could have died at all.** + +`prune_dead_per_object_layout_owners` asks "which owners died?" of every key +in `LAYOUT_SLOT_MASKS` + `TYPED_LAYOUTS`, tables sized by everything the +program ever created (~66k live keys on cc, from a history far larger). But +both of a minor's deadness predicates require the owner to be in the nursery: +`owner_is_dead_copied_minor_from_space` demands eden or the active survivor +half, and `PostTraceProbe::owner_is_dead` on a minor demands an in-arena, +untenured `HeapGeneration::Nursery` address. An owner that was old at the last +prune is still old, so the walk over it cannot remove anything. + +So the two maps get the young-entry log of #9754 (`gc/young_log.rs`): every +writer notes a key whose owner `layout_key_may_be_nursery` admits before the +entry becomes findable, a minor prunes from the log, and a survivor is +re-logged only while it is still young — a promoted owner leaves the log and +no later minor visits it again. A full prune keeps its whole-table walk (old +owners do die in a full trace) and rebuilds the log from the survivors it is +already classifying, at no extra pass. + +**Why this table pays where the scanners of #9754 did not.** Read back per +table on an unmodified binary, that PR's four converted tables are a net 0.78x +on cc and `closure.dynamic_props` is a 2.56x regression, because a scanner +keeps `addr_is_minor_relevant` — true for `Longlived` **by design**, since a +longlived object can point at a young one — and cc allocates its shape-key +arrays longlived, so those logs never drain (`kept/logged` median 1.000). A +prune's predicate is `layout_key_may_be_nursery`, which excludes `Longlived` +**and** `Old`; cc's tenuring promotes every survivor after one survival, so a +key leaves this log after one minor. Same mechanism, opposite sign, decided +entirely by which predicate the walk keeps on. The measured over-visit is 54x +at a 3300-character reply and 25x at 400, with `dead <= young_before` on +152/152 minor prunes — the empirical proof that the log's predicate is a sound +superset of what a minor can kill. + +Rule 2 of the design travels with it: under `debug_assertions` the young prune +re-derives the candidate set from the authoritative maps and panics on any +young key the log does not name, so deleting an arming site is a red test +rather than a dead owner's record surviving in silence. The in-borrow mask +mint in `layout_note_slot` — the dominant insert path on cc, and the one site +that published a young record without counting it — is armed for the first +time here. diff --git a/crates/perry-runtime/src/gc/dead_owner.rs b/crates/perry-runtime/src/gc/dead_owner.rs index 2eeff26bbe..126ce3e938 100644 --- a/crates/perry-runtime/src/gc/dead_owner.rs +++ b/crates/perry-runtime/src/gc/dead_owner.rs @@ -354,7 +354,7 @@ pub(super) const DEAD_KEY_PRUNES: &[DeadKeyPrune] = &[ table: "LAYOUT_SLOT_MASKS + TYPED_LAYOUTS", owner: DeadKeyOwner::Any, prune: crate::gc::layout_tables::prune_dead_per_object_layout_owners, - young_prune: None, + young_prune: Some(crate::gc::layout_tables::prune_dead_per_object_layout_owners_young), }, // Re-keyed by the per-object move hook, not by a metadata visitor. DeadKeyPrune { diff --git a/crates/perry-runtime/src/gc/layout.rs b/crates/perry-runtime/src/gc/layout.rs index 5ee4f87cb7..74ae34248f 100644 --- a/crates/perry-runtime/src/gc/layout.rs +++ b/crates/perry-runtime/src/gc/layout.rs @@ -954,11 +954,23 @@ pub(crate) fn layout_note_slot(parent_user: usize, slot_index: usize, value_bits } else { let mut mask = LayoutSlotMask::Inline(0); mask.set_slot(slot_index); + // The one insert site that holds its own `borrow_mut`, + // so it maintains the address filter, the young log + // and the young-record count inline too. The log lives + // in the hint, not in this map, so arming it here + // takes no second borrow — and it goes BEFORE the + // insert (`gc/young_log.rs` rule 1). Before #9841 this + // site published a young record without counting it; + // on cc it is the DOMINANT insert path (`TYPED_LAYOUTS` + // is empty there), so it is where a missing arm would + // do the most damage. + let young = super::layout_tables::arm_young_layout_key(parent_user); masks.insert(parent_user, mask); mark_per_object_layouts_nonempty(); - // The one insert site that holds its own `borrow_mut`, - // so it maintains the address filter inline too. super::layout_tables::layout_addr_filter_note(parent_user); + if young { + super::layout_tables::count_new_young_layout_record(); + } set_layout_state(header, GC_LAYOUT_SIDE_MASK); } } else { diff --git a/crates/perry-runtime/src/gc/layout_tables.rs b/crates/perry-runtime/src/gc/layout_tables.rs index 420c2b1df0..97da20152f 100644 --- a/crates/perry-runtime/src/gc/layout_tables.rs +++ b/crates/perry-runtime/src/gc/layout_tables.rs @@ -76,6 +76,16 @@ pub(in crate::gc) struct PerObjectLayoutHint { /// on every new nursery-keyed insert; made exact again by /// [`recount_young_layout_records`] after each collection's death prune. pub(in crate::gc) young_records: Cell, + /// #9754-style young-entry log for BOTH per-object maps + /// (`gc/young_log.rs`): the keys whose owner may still sit on a page a + /// minor can act on. A minor's death prune walks this instead of the + /// maps — an owner that was old at the last prune is still old, so only + /// a logged key can be found dead by a minor. + /// + /// It lives here, in the same hot slot as the flag and the filter, so a + /// writer arms it with the thread-local resolution it has already paid + /// for, and so nothing new is declared for `tls_hot::fill` to resolve. + pub(in crate::gc) young_keys: RefCell>, } impl PerObjectLayoutHint { @@ -85,10 +95,14 @@ impl PerObjectLayoutHint { sets: Cell::new(0), filter: std::cell::UnsafeCell::new([0u64; LAYOUT_ADDR_FILTER_WORDS]), young_records: Cell::new(0), + young_keys: RefCell::new(crate::gc::young_log::YoungLog::new()), } } } +/// The `[gc-young-log]` / `young_log::last_walk` row name for the two maps. +pub(in crate::gc) const LAYOUT_YOUNG_LOG_NAME: &str = "gc.layout_tables"; + impl Drop for PerObjectLayoutHint { fn drop(&mut self) { // The ownership bit and its teardown live in this ONE TLS value. The @@ -155,12 +169,28 @@ fn layout_key_may_be_nursery(addr: usize) -> bool { ) } -/// A NEW per-object record was keyed by `user_ptr`. +/// A per-object record is ABOUT to be keyed by `user_ptr`: if the owner sits +/// where a minor could kill it, log the key. Rule 1 of `gc/young_log.rs` — +/// note BEFORE the entry is findable. Returns that youngness so the caller can +/// bump the young-record count once it knows the insert was fresh, without a +/// second classification. #[inline] -fn note_new_layout_record(user_ptr: usize) { +pub(in crate::gc) fn arm_young_layout_key(user_ptr: usize) -> bool { if !layout_key_may_be_nursery(user_ptr) { - return; + return false; } + hot_per_object_layout_hint() + .young_keys + .borrow_mut() + .note(user_ptr); + true +} + +/// A NEW nursery-keyed record was published: keep the inline allocator's gate +/// ([`PERRY_YOUNG_LAYOUT_RECORDS`]) conservative until the next prune makes it +/// exact. +#[inline] +pub(in crate::gc) fn count_new_young_layout_record() { let hint = hot_per_object_layout_hint(); if let Some(next) = hint.young_records.get().checked_add(1) { hint.young_records.set(next); @@ -168,6 +198,33 @@ fn note_new_layout_record(user_ptr: usize) { } } +/// The flag proved BOTH maps empty, so every key the log still names is +/// stale. Dropping them here is what keeps the log bounded: a prune that +/// early-returns on the emptiness proof never drains it, so a workload that +/// repeatedly fills and empties the maps between collections would otherwise +/// accumulate one dead key per insert for ever. +#[cold] +fn drop_stale_young_layout_log() { + hot_per_object_layout_hint().young_keys.borrow_mut().clear(); +} + +/// A record is being re-keyed to `new_user` by the per-object move hook +/// (`transfer_per_object_*`), which runs during evacuation — i.e. BEFORE the +/// copied minor's prune, so the key this notes is one the prune will classify +/// in this very collection. +/// +/// Logged unconditionally: the destination is a to-space survivor (young), a +/// promoted address (old), or mid-evacuation not yet classifiable. Noting it +/// without asking is correct (the prune classifies once and an old key simply +/// drops) and keeps a page-map probe out of the evacuation loop. +#[inline] +fn arm_moved_layout_key(new_user: usize) { + hot_per_object_layout_hint() + .young_keys + .borrow_mut() + .note(new_user); +} + /// Publish this thread's young-record count and the delta to the process /// total. The count itself is derived by the death prune's single pass over /// the live keys (all cycle kinds), so promotion (a key moving to an old page) @@ -192,6 +249,7 @@ fn publish_young_layout_records(live: u32) { /// inline allocator's gate reads that instead of probing. pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn(usize) -> bool) { if !per_object_layouts_maybe_nonempty() { + drop_stale_young_layout_log(); return; } // ONE pass over each table, not three. The old shape visited every live @@ -217,6 +275,12 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn( layout_addr_filter_saturate(); } let mut young: u32 = 0; + // A full walk is authoritative, so it also REBUILDS the young log — from + // the survivors it is classifying anyway, at the cost of one `push` per + // young key and no extra pass (`young_log.rs`: "a full-scope scanner + // walks the whole table as before and REBUILDS the log from what it + // found"). + let mut kept = hint.young_keys.borrow_mut().take_spare(); let mut keep = |key: usize| { if is_dead_owner(key) { return false; @@ -233,6 +297,7 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn( } if layout_key_may_be_nursery(key) { young = young.saturating_add(1); + kept.push(key); } true }; @@ -248,6 +313,23 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn( typed.retain(|key, _| keep(*key)); had && typed.is_empty() }; + // A full walk is authoritative: rebuild the young log from the tables + // (same shape as the shape/descriptor full scanners). + { + let mut log = hint.young_keys.borrow_mut(); + let _ = log.take_sorted(); + log.extend(kept); + } + crate::gc::young_log::note_walk( + LAYOUT_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: false, + logged: occupancy as u64, + visited: occupancy as u64, + kept: u64::from(young), + table_len: occupancy as u64, + }, + ); publish_young_layout_records(young); // Runs last: when it finds both tables empty it disarms the flag, zeroes // the young count published above and clears the filter, which is the @@ -258,6 +340,134 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn( } } +/// [`prune_dead_per_object_layout_owners`] for a MINOR (`DEAD_KEY_PRUNES` +/// `young_prune`). +/// +/// # Why this is sound +/// +/// A minor's two deadness predicates both require the owner to be in the +/// nursery: `owner_is_dead_copied_minor_from_space` demands eden or the active +/// survivor half, and `PostTraceProbe::owner_is_dead` on a minor demands an +/// in-arena, untenured `HeapGeneration::Nursery` address. So the only keys a +/// minor can remove are the ones [`layout_key_may_be_nursery`] admits — which +/// is a strict SUPERSET of both (it also admits an unclassifiable address, and +/// classifies from the same page map). Every writer notes such a key before +/// the entry becomes findable, and every walk re-logs a survivor that is still +/// young, so the log names every candidate and the walk loses nothing. +/// +/// That predicate is the whole difference between this conversion and the +/// scanner conversions of #9754: a scanner keeps `addr_is_minor_relevant`, +/// which admits `Longlived` **by design** (a longlived object can point at a +/// young one), whereas a prune asks who DIED and therefore excludes +/// `Longlived` and `Old` both. +/// +/// A logged key that is in neither map is stale (moved away, removed) and +/// drops; a present key whose owner is dead is removed from both maps; a live +/// key is re-logged iff its owner is still young, so a promoted owner leaves +/// the log and no later minor visits it again. +/// +/// The address filter is NOT rebuilt here — the whole-table walk that rebuilt +/// it is exactly what this replaces. Its `false` is the only load-bearing +/// answer and a stale set bit is a false positive, so leaving bits behind is +/// safe; the amortised rebuild in [`layout_addr_filter_add`] and the full +/// prune keep it selective. +pub(in crate::gc) fn prune_dead_per_object_layout_owners_young( + is_dead_owner: &dyn Fn(usize) -> bool, +) { + if !per_object_layouts_maybe_nonempty() { + drop_stale_young_layout_log(); + return; + } + let hint = hot_per_object_layout_hint(); + let table_len = + (hot_layout_slot_masks().borrow().len() + hot_typed_layouts().borrow().len()) as u64; + // Rule 2 (`gc/young_log.rs`): re-derive the candidate set from the + // authoritative maps and refuse to run a partial walk that would miss one. + // A miss is a writer that published a young-keyed record without arming + // the log, which in release would silently keep a dead owner's record. + #[cfg(debug_assertions)] + { + let relevant: Vec = { + let masks = hot_layout_slot_masks().borrow(); + let typed = hot_typed_layouts().borrow(); + masks + .keys() + .chain(typed.keys()) + .copied() + .filter(|key| layout_key_may_be_nursery(*key)) + .collect() + }; + hint.young_keys + .borrow() + .debug_assert_logged(LAYOUT_YOUNG_LOG_NAME, &relevant); + } + let mut logged = 0u64; + let mut visited = 0u64; + // The record count is per MAP ENTRY, as the full prune counts it: a key + // present in both maps is two records and one log entry. + let mut young: u32 = 0; + let mut kept = hint.young_keys.borrow_mut().take_spare(); + let (masks_emptied, typed_emptied) = { + let mut masks = hot_layout_slot_masks().borrow_mut(); + let mut typed = hot_typed_layouts().borrow_mut(); + let had_masks = !masks.is_empty(); + let had_typed = !typed.is_empty(); + loop { + // Re-drained in a loop so a note made while this walk runs (the + // move hooks fire from inside a collection) is not lost. + let batch = hint.young_keys.borrow_mut().take_sorted(); + if batch.is_empty() { + break; + } + logged += batch.len() as u64; + for key in batch { + let in_masks = masks.contains_key(&key); + let in_typed = typed.contains_key(&key); + if !in_masks && !in_typed { + continue; + } + visited += 1; + if is_dead_owner(key) { + if in_masks { + masks.remove(&key); + } + if in_typed { + typed.remove(&key); + } + continue; + } + if layout_key_may_be_nursery(key) { + young = young + .saturating_add(u32::from(in_masks)) + .saturating_add(u32::from(in_typed)); + kept.push(key); + } + } + } + (had_masks && masks.is_empty(), had_typed && typed.is_empty()) + }; + let kept_len = kept.len() as u64; + hint.young_keys.borrow_mut().extend(kept); + crate::gc::young_log::note_walk( + LAYOUT_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: true, + logged, + visited, + kept: kept_len, + table_len, + }, + ); + publish_young_layout_records(young); + // Runs last, as in the full prune: with both maps empty it disarms the + // flag, zeroes the count published above and clears the filter. + refresh_per_object_layouts_flag(masks_emptied || typed_emptied); + if crate::hot_diag::layout_on() { + // `rebuilt_filter = false`: a young prune never rebuilds it. + layout_diag_note_prune(false); + } +} + /// `PERRY_LAYOUT_DIAG`'s per-prune sample. Out of line and behind /// [`crate::hot_diag::layout_on`] so an unarmed build pays one relaxed load. #[cold] @@ -831,12 +1041,14 @@ pub(in crate::gc) fn refresh_per_object_layouts_flag(touched_map_emptied: bool) pub(in crate::gc) fn typed_layouts_insert(user_ptr: usize, descriptor: TypedLayoutDescriptor) { mark_per_object_layouts_nonempty(); layout_addr_filter_add(user_ptr); + // Armed BEFORE the insert makes the entry findable (young-log rule 1). + let young = arm_young_layout_key(user_ptr); let fresh = hot_typed_layouts() .borrow_mut() .insert(user_ptr, descriptor) .is_none(); - if fresh { - note_new_layout_record(user_ptr); + if fresh && young { + count_new_young_layout_record(); } } @@ -845,12 +1057,14 @@ pub(in crate::gc) fn typed_layouts_insert(user_ptr: usize, descriptor: TypedLayo pub(in crate::gc) fn slot_masks_insert(user_ptr: usize, mask: LayoutSlotMask) { mark_per_object_layouts_nonempty(); layout_addr_filter_add(user_ptr); + // Armed BEFORE the insert makes the entry findable (young-log rule 1). + let young = arm_young_layout_key(user_ptr); let fresh = hot_layout_slot_masks() .borrow_mut() .insert(user_ptr, mask) .is_none(); - if fresh { - note_new_layout_record(user_ptr); + if fresh && young { + count_new_young_layout_record(); } } @@ -936,6 +1150,7 @@ pub(in crate::gc) fn transfer_per_object_descriptor(old_user: usize, new_user: u typed.remove(&new_user); match typed.remove(&old_user) { Some(layout) => { + arm_moved_layout_key(new_user); typed.insert(new_user, layout); drop(typed); layout_addr_filter_add(new_user); @@ -956,6 +1171,7 @@ pub(in crate::gc) fn transfer_per_object_slot_mask(old_user: usize, new_user: us let mut masks = hot_layout_slot_masks().borrow_mut(); masks.remove(&new_user); if let Some(mask) = masks.remove(&old_user) { + arm_moved_layout_key(new_user); masks.insert(new_user, mask); drop(masks); layout_addr_filter_add(new_user); diff --git a/crates/perry-runtime/src/gc/tests/young_log_tests.rs b/crates/perry-runtime/src/gc/tests/young_log_tests.rs index 53eaa899f3..27ed7c6afe 100644 --- a/crates/perry-runtime/src/gc/tests/young_log_tests.rs +++ b/crates/perry-runtime/src/gc/tests/young_log_tests.rs @@ -844,3 +844,134 @@ fn promoted_box_root_leaves_log_and_is_found_by_full_walk() { ); assert!(!walk("box.roots").partial); } + +// -------------------------------------------------- per-object layout tables +// +// #9841: the DEATH PRUNE of `LAYOUT_SLOT_MASKS + TYPED_LAYOUTS`, not a root +// scanner. Its predicate is `layout_key_may_be_nursery`, which excludes +// `Longlived` AND `Old` — strictly stronger than the scanners' +// `addr_is_minor_relevant` — so an old-keyed record is not merely cheap to +// visit, it is provably impossible for a minor to remove. + +use crate::gc::layout_tables::{test_per_object_layout_present, LAYOUT_YOUNG_LOG_NAME}; + +/// A nursery object whose header says POINTER_FREE and which then takes a +/// pointer store — the mutator path that mints a mask from inside +/// `layout_note_slot`'s own `borrow_mut` (WRITER 3). On cc that is the +/// dominant insert path: `TYPED_LAYOUTS` is empty there and every one of the +/// ~66k live keys is a `LAYOUT_SLOT_MASKS` entry. +fn young_masked_object() -> usize { + let obj = crate::object::js_object_alloc(0, 8); + crate::object::js_object_set_field(obj, 0, crate::value::JSValue::number(1.0)); + crate::object::js_object_set_field(obj, 1, crate::value::JSValue::number(2.0)); + crate::gc::layout_clear_for_ptr(obj as usize); + unsafe { crate::gc::layout_init_pointer_free(obj as *mut u8) }; + let child = crate::string::js_string_from_bytes(b"late-pointer".as_ptr(), 12); + crate::object::js_object_set_field(obj, 1, crate::value::JSValue::string_ptr(child)); + assert!( + test_per_object_layout_present(obj as usize), + "premise: the in-place mask mint published a per-object record" + ); + obj as usize +} + +/// WRITER 3's arming site. Delete `arm_young_layout_key` from +/// `gc/layout.rs`'s in-borrow mint and this goes red: under +/// `debug_assertions` on the log-completeness re-derivation, and in release +/// on the record the young prune can no longer see. +#[test] +fn dead_young_masked_owner_is_pruned_through_the_layout_log() { + let _guard = CopyingNurseryTestGuard::new(1); + // One rooted young object so the minor has real work; the owner is not it. + js_shadow_slot_set(0, string_bits(young_leaf())); + + let dead = young_masked_object(); + + let _ = gc_collect_minor(); + + assert!( + !test_per_object_layout_present(dead), + "the dead young owner's per-object layout record must be pruned from the log" + ); + let row = walk(LAYOUT_YOUNG_LOG_NAME); + assert!( + row.partial, + "a copying minor must take the young-scoped prune: {row:?}" + ); + assert!( + row.visited >= 1, + "the logged key must have been visited: {row:?}" + ); +} + +/// WRITER 4's arming site (`transfer_per_object_slot_mask`, which runs during +/// evacuation and therefore BEFORE this collection's prune). Delete its +/// `arm_moved_layout_key` and the re-derivation panics here on the to-space +/// key. +#[test] +fn surviving_young_masked_owner_is_rekeyed_and_stays_logged() { + let _guard = CopyingNurseryTestGuard::new(1); + + let obj = young_masked_object(); + js_shadow_slot_set(0, ptr_bits(obj)); + + let _ = gc_collect_minor(); + + let after = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!(after, obj, "the rooted owner must have been evacuated"); + assert!( + test_per_object_layout_present(after), + "the mask must follow its owner to the new address" + ); + assert!( + !test_per_object_layout_present(obj), + "the stale from-space key must be gone" + ); + let row = walk(LAYOUT_YOUNG_LOG_NAME); + assert!(row.partial, "{row:?}"); + assert!( + row.visited >= 1, + "the move hook's key must have been logged and visited: {row:?}" + ); + if crate::arena::pointer_in_nursery(after) { + assert!( + row.kept >= 1, + "a survivor still in the nursery must stay logged: {row:?}" + ); + } +} + +/// Rule 3: the skip has to be observable, or a latch that never fires looks +/// landed. An OLD-keyed record cannot be found dead by any minor, so the +/// young prune must not visit it at all. +#[test] +fn old_layout_records_are_skipped_by_a_minor() { + let _guard = CopyingNurseryTestGuard::new(0); + + // Drain whatever this thread's earlier tests left young, so `visited` + // below is about the record installed after it. + let _ = gc_collect_minor(); + + let (owner, _) = unsafe { alloc_old_test_object(2) }; + crate::gc::layout_tables::slot_masks_insert( + owner as usize, + crate::gc::layout::LayoutSlotMask::from_words(&[1]), + ); + + let _ = gc_collect_minor(); + + assert!( + test_per_object_layout_present(owner as usize), + "an old owner's record must survive a minor" + ); + let row = walk(LAYOUT_YOUNG_LOG_NAME); + assert!(row.partial, "{row:?}"); + assert!(row.table_len >= 1, "{row:?}"); + assert_eq!( + row.visited, 0, + "an old-keyed record is not a candidate for any minor and must not be \ + visited: {row:?}" + ); + + crate::gc::layout_clear_for_ptr(owner as usize); +} diff --git a/crates/perry-runtime/src/gc/young_log.rs b/crates/perry-runtime/src/gc/young_log.rs index d83d9bb36c..bfc482c07c 100644 --- a/crates/perry-runtime/src/gc/young_log.rs +++ b/crates/perry-runtime/src/gc/young_log.rs @@ -145,8 +145,11 @@ impl YoungLog { } } - /// Test-only: the table resets (`test_clear_*`) clear their log with them. - #[cfg(test)] + /// Drop every logged key, keeping both buffers' capacity. For a caller + /// that has just PROVED its table empty: every key in the log is then + /// stale, and a walk that early-returns on that proof would otherwise + /// carry them forward for ever (the table resets `test_clear_*` use it + /// for the same reason). pub(crate) fn clear(&mut self) { self.keys.clear(); self.spare.clear(); From 8a260356fdc54bf70e7725a4b5f1b71f6173dcc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 15:59:48 +0200 Subject: [PATCH 23/27] diag(gc): histogram the surviving per-object layout-mask residue Under PERRY_LAYOUT_DIAG, time each existing death-prune walk and emit marginal histograms for surviving mask owners, slot counts, pointer share, and heap space. Price the standing residue as per-key prune nanoseconds and the maximum tag checks masks can save in one full trace. Keep insert provenance in diagnostic-only counters so LayoutSlotMask and the unarmed trace/store representation do not change. Cover kind/bucket routing and prove the unarmed histogram loop stays dark with sabotage-capable tests. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit 97b550085869f108c0789ec07ac620ab9de7f295) --- crates/perry-runtime/src/gc/layout.rs | 47 ++--- crates/perry-runtime/src/gc/layout_tables.rs | 163 +++++++++++++++++- .../src/gc/tests/layout_residue_histogram.rs | 99 +++++++++++ crates/perry-runtime/src/gc/tests/mod.rs | 1 + crates/perry-runtime/src/hot_diag.rs | 136 ++++++++++++++- 5 files changed, 409 insertions(+), 37 deletions(-) create mode 100644 crates/perry-runtime/src/gc/tests/layout_residue_histogram.rs diff --git a/crates/perry-runtime/src/gc/layout.rs b/crates/perry-runtime/src/gc/layout.rs index 74ae34248f..2882f4f75e 100644 --- a/crates/perry-runtime/src/gc/layout.rs +++ b/crates/perry-runtime/src/gc/layout.rs @@ -1,28 +1,23 @@ -//! Per-object pointer-slot layout: the `GcHeader._reserved` layout states, -//! store-time descriptor maintenance (`layout_note_slot`), rebuild/transfer -//! across copying GC, and the child-slot enumeration the collector walks. -//! The slot-mask representation lives in `layout/slot_mask.rs`; the -//! typed-shape descriptor *installation* protocol (`js_gc_init_typed_shape_layout` -//! / `js_gc_declare_typed_shape_layout`) lives in `layout/typed_shape.rs`. +//! Per-object pointer-slot states, store maintenance, copying-GC transfer and +//! child-slot enumeration. Mask storage is in `layout/slot_mask.rs`; typed +//! descriptor installation is in `layout/typed_shape.rs`. use super::hot_tls::{hot_layout_slot_masks, hot_shape_layouts}; use super::layout_tables::{ - layout_forget_object, mark_per_object_layouts_nonempty, per_object_slot_mask, - refresh_per_object_layouts_flag, slot_masks_insert, slot_masks_remove, + layout_forget_object, layout_note_store_mask_insert, mark_per_object_layouts_nonempty, + per_object_slot_mask, refresh_per_object_layouts_flag, slot_masks_insert, + slot_masks_insert_birth, slot_masks_insert_rebuild, slot_masks_remove, transfer_per_object_descriptor, transfer_per_object_slot_mask, typed_layouts_insert, typed_layouts_remove, with_per_object_descriptor, }; use super::*; - -// Copied-nursery survival age stored in otherwise-unused low -// GcHeader._reserved bits. Bits 0..2 remain object freeze/seal flags -// and bits 14..15 remain layout state. +// Copied-nursery survival age in otherwise-unused low `_reserved` bits; +// bits 0..2 remain object flags and bits 14..15 remain layout state. pub(super) const GC_COPY_SURVIVAL_AGE_SHIFT: usize = 3; pub(super) const GC_COPY_SURVIVAL_AGE_MASK: u16 = 0x0038; pub(super) const GC_COPY_PROMOTION_SURVIVALS: u8 = 4; -// Pointer-slot layout state stored in the high bits of GcHeader._reserved. -// Low bits remain object freeze/seal/preventExtensions flags. +// Pointer-slot layout state in high `_reserved` bits; low bits remain object flags. pub const GC_LAYOUT_STATE_MASK: u16 = 0xC000; pub(super) const GC_LAYOUT_UNKNOWN: u16 = 0x0000; /// No payload slot holds a pointer, so `heap_payload_slot_selection` skips the @@ -42,18 +37,11 @@ pub(super) const GC_LAYOUT_UNKNOWN: u16 = 0x0000; /// probe read its records only after the last GC. Under `PERRY_JSON_TAPE=0` the /// same sabotage SIGSEGVs. So: /// -/// - "clean at rate 1 + from-space protect" is evidence only once you have -/// shown the misdeclared object EXISTED during a collection; -/// - `PERRY_GC_FROMSPACE_SCAN=1` is the instrument to prefer — its -/// whole-payload word scan consults no layout state, and it reported the -/// stranded children at exactly `dangling=8000 owners=4000`; -/// - `PERRY_GC_VERIFY_EVACUATION` is blind here by construction: it walks the -/// same enumeration the rewrite pass walks, which is to say it asks this -/// state which slots exist. -/// -/// The workload-free detectors are the child-slot enumerator and relocation -/// across a copying minor; worked example, sabotage-verified in both -/// directions: `gc/tests/copying/deferred_finalize_7635.rs`. +/// Therefore first prove the object existed during collection; prefer +/// `PERRY_GC_FROMSPACE_SCAN=1`, whose whole-payload scan ignores layout state. +/// `PERRY_GC_VERIFY_EVACUATION` is blind because it trusts this enumeration. +/// Workload-free coverage lives in the child-slot and copying-relocation tests +/// in `gc/tests/copying/deferred_finalize_7635.rs`. pub const GC_LAYOUT_POINTER_FREE: u16 = 0x4000; pub(crate) const GC_LAYOUT_SIDE_MASK: u16 = 0x8000; // A side-layout payload whose entire live prefix contains pointers. Bit 13 is @@ -966,6 +954,7 @@ pub(crate) fn layout_note_slot(parent_user: usize, slot_index: usize, value_bits // do the most damage. let young = super::layout_tables::arm_young_layout_key(parent_user); masks.insert(parent_user, mask); + layout_note_store_mask_insert(); mark_per_object_layouts_nonempty(); super::layout_tables::layout_addr_filter_note(parent_user); if young { @@ -1173,7 +1162,7 @@ pub(super) unsafe fn layout_rebuild_from_slots_with_policy( slot_masks_remove(user_ptr as usize); } else { set_layout_state(header, GC_LAYOUT_SIDE_MASK); - slot_masks_insert(user_ptr as usize, mask); + slot_masks_insert_rebuild(user_ptr as usize, mask); } } @@ -1228,7 +1217,7 @@ pub(crate) unsafe fn layout_init_from_slots( set_layout_state(header, GC_LAYOUT_UNKNOWN); } else { set_layout_state(header, GC_LAYOUT_SIDE_MASK); - slot_masks_insert(user_ptr as usize, LayoutSlotMask::Inline(bits)); + slot_masks_insert_birth(user_ptr as usize, LayoutSlotMask::Inline(bits)); } return any_pointer; } @@ -1247,7 +1236,7 @@ pub(crate) unsafe fn layout_init_from_slots( set_layout_state(header, GC_LAYOUT_UNKNOWN); } else { set_layout_state(header, GC_LAYOUT_SIDE_MASK); - slot_masks_insert(user_ptr as usize, mask); + slot_masks_insert_birth(user_ptr as usize, mask); } any_pointer } diff --git a/crates/perry-runtime/src/gc/layout_tables.rs b/crates/perry-runtime/src/gc/layout_tables.rs index 97da20152f..76a9f298b0 100644 --- a/crates/perry-runtime/src/gc/layout_tables.rs +++ b/crates/perry-runtime/src/gc/layout_tables.rs @@ -31,7 +31,9 @@ use super::hot_tls::{hot_layout_slot_masks, hot_per_object_layout_hint, hot_typed_layouts}; use super::layout::{LayoutSlotMask, TypedLayoutDescriptor}; -use super::types::{GcHeader, GC_FLAG_ARENA, GC_HEADER_SIZE, GC_TYPE_ARRAY, GC_TYPE_OBJECT}; +use super::types::{ + GcHeader, GC_FLAG_ARENA, GC_HEADER_SIZE, GC_TYPE_ARRAY, GC_TYPE_CLOSURE, GC_TYPE_OBJECT, +}; use std::cell::{Cell, RefCell}; thread_local! { @@ -252,6 +254,9 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn( drop_stale_young_layout_log(); return; } + // Exactly one arm test per non-empty prune. Everything below it — the + // timer and the residue-wide histogram — is absent when the sink is off. + let layout_diag = crate::hot_diag::layout_on(); // ONE pass over each table, not three. The old shape visited every live // key three times per collection — `retain`, then // `layout_addr_filter_rebuild` (which first collected them all into a @@ -301,6 +306,7 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn( } true }; + let prune_walk_started = layout_diag.then(std::time::Instant::now); let masks_emptied = { let mut masks = hot_layout_slot_masks().borrow_mut(); let had = !masks.is_empty(); @@ -313,6 +319,9 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn( typed.retain(|key, _| keep(*key)); had && typed.is_empty() }; + let prune_walk_us = prune_walk_started.map_or(0, |started| { + started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64 + }); // A full walk is authoritative: rebuild the young log from the tables // (same shape as the shape/descriptor full scanners). { @@ -335,8 +344,8 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn( // the young count published above and clears the filter, which is the // correct end state whichever branch the pass took. refresh_per_object_layouts_flag(masks_emptied || typed_emptied); - if crate::hot_diag::layout_on() { - layout_diag_note_prune(rebuild_filter); + if layout_diag { + layout_diag_note_prune(rebuild_filter, prune_walk_us); } } @@ -378,6 +387,8 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners_young( drop_stale_young_layout_log(); return; } + // Exactly one arm test per non-empty prune; see the full-prune twin. + let layout_diag = crate::hot_diag::layout_on(); let hint = hot_per_object_layout_hint(); let table_len = (hot_layout_slot_masks().borrow().len() + hot_typed_layouts().borrow().len()) as u64; @@ -407,6 +418,7 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners_young( // present in both maps is two records and one log entry. let mut young: u32 = 0; let mut kept = hint.young_keys.borrow_mut().take_spare(); + let prune_walk_started = layout_diag.then(std::time::Instant::now); let (masks_emptied, typed_emptied) = { let mut masks = hot_layout_slot_masks().borrow_mut(); let mut typed = hot_typed_layouts().borrow_mut(); @@ -446,6 +458,9 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners_young( } (had_masks && masks.is_empty(), had_typed && typed.is_empty()) }; + let prune_walk_us = prune_walk_started.map_or(0, |started| { + started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64 + }); let kept_len = kept.len() as u64; hint.young_keys.borrow_mut().extend(kept); crate::gc::young_log::note_walk( @@ -462,20 +477,21 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners_young( // Runs last, as in the full prune: with both maps empty it disarms the // flag, zeroes the count published above and clears the filter. refresh_per_object_layouts_flag(masks_emptied || typed_emptied); - if crate::hot_diag::layout_on() { + if layout_diag { // `rebuilt_filter = false`: a young prune never rebuilds it. - layout_diag_note_prune(false); + layout_diag_note_prune(false, prune_walk_us); } } /// `PERRY_LAYOUT_DIAG`'s per-prune sample. Out of line and behind /// [`crate::hot_diag::layout_on`] so an unarmed build pays one relaxed load. #[cold] -fn layout_diag_note_prune(rebuilt_filter: bool) { +fn layout_diag_note_prune(rebuilt_filter: bool, prune_walk_us: u64) { let (typed_len, masks_len) = ( hot_typed_layouts().borrow().len(), hot_layout_slot_masks().borrow().len(), ); + let residue = layout_residue_histogram(prune_walk_us); let hint = hot_per_object_layout_hint(); // SAFETY: as in the pass above — this thread's own filter, no other // reference live. @@ -492,9 +508,117 @@ fn layout_diag_note_prune(rebuilt_filter: bool) { LAYOUT_ADDR_FILTER_BITS, rebuilt_filter, layout_addr_filter_saturating_occupancy(), + residue, ); } +/// Walk the surviving mask table once for `PERRY_LAYOUT_DIAG` only. +/// +/// The logical slot bound comes from the same owner metadata the tracer uses: +/// array length, shape-derived object live slots, or real closure captures. +/// A mask cannot legitimately belong to any other GC kind, but `other` keeps +/// the diagnostic total honest if a stale/corrupt entry is ever observed. +#[cold] +fn layout_residue_histogram(prune_walk_us: u64) -> crate::hot_diag::LayoutResidueHistogram { + let mut out = crate::hot_diag::LayoutResidueHistogram { + prune_walk_us, + ..Default::default() + }; + let masks = hot_layout_slot_masks().borrow(); + out.keys = masks.len() as u64; + for (&owner, mask) in masks.iter() { + #[cfg(test)] + LAYOUT_RESIDUE_HISTOGRAM_ENTRIES.with(|n| n.set(n.get().saturating_add(1))); + + let Some(header) = (unsafe { crate::value::addr_class::try_read_tracked_gc_header(owner) }) + else { + out.other += 1; + out.slots[0] += 1; + out.pointer_share[0] += 1; + out.space[2] += 1; + continue; + }; + // SAFETY: `try_read_tracked_gc_header` proved this exact owner belongs + // to either an arena allocation or the tracked malloc registry. + let header = unsafe { header.as_ref() }; + let slot_count = unsafe { + match header.obj_type { + GC_TYPE_CLOSURE => { + out.closure += 1; + let closure = owner as *const crate::closure::ClosureHeader; + crate::closure::real_capture_count((*closure).capture_count) as usize + } + GC_TYPE_OBJECT => { + out.object += 1; + crate::object::object_live_slot_count( + owner as *const crate::object::ObjectHeader, + ) as usize + } + GC_TYPE_ARRAY => { + out.array += 1; + let array = owner as *const crate::array::ArrayHeader; + ((*array).length as usize).min((*array).capacity as usize) + } + _ => { + out.other += 1; + (header.size as usize).saturating_sub(GC_HEADER_SIZE) / 8 + } + } + }; + + let slot_bucket = match slot_count { + 0..=7 => 0, + 8..=15 => 1, + 16..=31 => 2, + 32..=63 => 3, + 64..=255 => 4, + _ => 5, + }; + out.slots[slot_bucket] += 1; + + let pointer_slots = mask.count_slots(slot_count); + let share_bucket = if pointer_slots.saturating_mul(4) <= slot_count { + 0 + } else if pointer_slots.saturating_mul(2) <= slot_count { + 1 + } else if pointer_slots.saturating_mul(4) <= slot_count.saturating_mul(3) { + 2 + } else { + 3 + }; + out.pointer_share[share_bucket] += 1; + out.est_tag_checks_saved_per_trace = out + .est_tag_checks_saved_per_trace + .saturating_add(slot_count.saturating_sub(pointer_slots) as u64); + + if header.gc_flags & GC_FLAG_ARENA == 0 { + out.space[2] += 1; + } else if crate::arena::classify_heap_space(owner).is_nursery() { + out.space[0] += 1; + } else { + // Old, Longlived and the transient PromotedYoung classification + // are all old-page residents for this three-way price split. + out.space[1] += 1; + } + } + out +} + +#[cfg(test)] +crate::perry_thread_local! { + static LAYOUT_RESIDUE_HISTOGRAM_ENTRIES: Cell = const { Cell::new(0) }; +} + +#[cfg(test)] +pub(in crate::gc) fn test_reset_layout_residue_histogram_entries() { + LAYOUT_RESIDUE_HISTOGRAM_ENTRIES.with(|n| n.set(0)); +} + +#[cfg(test)] +pub(in crate::gc) fn test_layout_residue_histogram_entries() -> usize { + LAYOUT_RESIDUE_HISTOGRAM_ENTRIES.with(Cell::get) +} + #[cfg(test)] pub(in crate::gc) fn test_per_object_layout_present(user_ptr: usize) -> bool { hot_layout_slot_masks().borrow().contains_key(&user_ptr) @@ -1054,7 +1178,7 @@ pub(in crate::gc) fn typed_layouts_insert(user_ptr: usize, descriptor: TypedLayo /// The one way to add a per-object pointer mask. #[inline] -pub(in crate::gc) fn slot_masks_insert(user_ptr: usize, mask: LayoutSlotMask) { +pub(in crate::gc) fn slot_masks_insert(user_ptr: usize, mask: LayoutSlotMask) -> bool { mark_per_object_layouts_nonempty(); layout_addr_filter_add(user_ptr); // Armed BEFORE the insert makes the entry findable (young-log rule 1). @@ -1066,6 +1190,31 @@ pub(in crate::gc) fn slot_masks_insert(user_ptr: usize, mask: LayoutSlotMask) { if fresh && young { count_new_young_layout_record(); } + fresh +} + +/// Insert-site wrappers for the diagnostic counter. Keeping them here avoids +/// carrying provenance in `LayoutSlotMask`, whose size and hot-path shape must +/// not change for an optional instrument. +#[inline] +pub(in crate::gc) fn slot_masks_insert_birth(user_ptr: usize, mask: LayoutSlotMask) { + if slot_masks_insert(user_ptr, mask) && crate::hot_diag::layout_on() { + crate::hot_diag::layout_note_mask_insert(crate::hot_diag::LayoutMaskInsertSite::Birth); + } +} + +#[inline] +pub(in crate::gc) fn slot_masks_insert_rebuild(user_ptr: usize, mask: LayoutSlotMask) { + if slot_masks_insert(user_ptr, mask) && crate::hot_diag::layout_on() { + crate::hot_diag::layout_note_mask_insert(crate::hot_diag::LayoutMaskInsertSite::Rebuild); + } +} + +#[inline] +pub(in crate::gc) fn layout_note_store_mask_insert() { + if crate::hot_diag::layout_on() { + crate::hot_diag::layout_note_mask_insert(crate::hot_diag::LayoutMaskInsertSite::Store); + } } /// Drop `user_ptr`'s per-object typed descriptor (only). diff --git a/crates/perry-runtime/src/gc/tests/layout_residue_histogram.rs b/crates/perry-runtime/src/gc/tests/layout_residue_histogram.rs new file mode 100644 index 0000000000..d7c4c66ae3 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/layout_residue_histogram.rs @@ -0,0 +1,99 @@ +use super::support::*; + +fn pointer_bits() -> u64 { + let child = crate::string::js_string_from_bytes(b"residue-child".as_ptr(), 13); + string_bits(child as usize) +} + +fn closure_with_captures(slot_count: usize, pointer_slots: usize) -> usize { + let pointer = pointer_bits(); + let mut captures = vec![1.0f64.to_bits(); slot_count]; + captures[..pointer_slots].fill(pointer); + let closure = crate::closure::js_closure_alloc(std::ptr::null(), slot_count as u32); + unsafe { + let slots = crate::closure::closure_capture_slots_mut(closure); + std::ptr::copy_nonoverlapping(captures.as_ptr(), slots, slot_count); + crate::gc::layout_init_from_slots(closure as *mut u8, slots, slot_count); + } + closure as usize +} + +/// The requested marginal-histogram fixture. Swapping the `8..=15` and +/// `16..=31` bounds makes the 20-capture assertion fail. +#[test] +fn layout_residue_histogram_counts_by_kind_and_bucket() { + let _gc = CopyingNurseryTestGuard::new(0); + let _diag = crate::hot_diag::LayoutDiagTestGuard::force(true); + + let closure5 = closure_with_captures(5, 1); + let closure20 = closure_with_captures(20, 10); + + let object70 = crate::object::js_object_alloc(0, 70); + let pointer = pointer_bits(); + unsafe { + let fields = (object70 as *mut u8).add(std::mem::size_of::()) + as *mut u64; + for slot in 0..70 { + fields.add(slot).write(if slot < 60 { + pointer + } else { + (slot as f64).to_bits() + }); + } + crate::object::rebuild_object_field_layout(object70, 70); + } + + crate::gc::layout_tables::test_reset_layout_residue_histogram_entries(); + crate::gc::layout_tables::prune_dead_per_object_layout_owners(&|_| false); + + let residue = crate::hot_diag::LayoutDiagTestGuard::residue(); + assert_eq!(residue.keys, 3, "{residue:?}"); + assert_eq!(residue.closure, 2, "{residue:?}"); + assert_eq!(residue.object, 1, "{residue:?}"); + assert_eq!(residue.array, 0, "{residue:?}"); + assert_eq!(residue.other, 0, "{residue:?}"); + assert_eq!(residue.slots, [1, 0, 1, 0, 1, 0], "{residue:?}"); + assert_eq!(residue.pointer_share, [1, 1, 0, 1], "{residue:?}"); + assert_eq!(residue.space, [3, 0, 0], "{residue:?}"); + assert_eq!(residue.est_tag_checks_saved_per_trace, 24, "{residue:?}"); + assert_eq!( + crate::gc::layout_tables::test_layout_residue_histogram_entries(), + 3 + ); + + let output = crate::hot_diag::LayoutDiagTestGuard::output(); + assert!(output.contains("[layout-diag] residue keys=3 closure=2 object=1 array=0 other=0")); + assert!(output.contains("slots{4-7=1 8-15=0 16-31=1 32-63=0 64-255=1 256+=0}")); + assert!(output.contains("ptr_share{q1=1 q2=1 q3=0 q4=1}")); + assert!(output.contains("space{nursery=3 old=0 malloc=0}")); + assert!(output.contains("inserts_since{birth=2 rebuild=1 store=0}")); + assert!(output.contains("[layout-diag] price per_key_prune_ns=")); + assert!(output.contains("est_tag_checks_saved_per_trace=24")); + + for owner in [closure5, closure20, object70 as usize] { + crate::gc::layout_clear_for_ptr(owner); + } +} + +/// Dropping the single `layout_on()` gate in either prune makes the test-only +/// entry counter non-zero, even though the output sink remains unarmed. +#[test] +fn layout_residue_histogram_is_silent_when_unarmed() { + let _gc = CopyingNurseryTestGuard::new(0); + let _diag = crate::hot_diag::LayoutDiagTestGuard::force(false); + let closure = closure_with_captures(5, 1); + + crate::gc::layout_tables::test_reset_layout_residue_histogram_entries(); + crate::gc::layout_tables::prune_dead_per_object_layout_owners(&|_| false); + + assert!( + crate::hot_diag::LayoutDiagTestGuard::output().is_empty(), + "an unarmed prune must emit no residue line" + ); + assert_eq!( + crate::gc::layout_tables::test_layout_residue_histogram_entries(), + 0, + "the histogram entry loop must not run while the sink is off" + ); + crate::gc::layout_clear_for_ptr(closure); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index a698870282..67832ef5ba 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -37,6 +37,7 @@ mod incremental_sweep_reclaim; mod inline_generation_gate_contract; mod inline_pointer_bearing_contract; mod layout_pointer_free_hazard; +mod layout_residue_histogram; mod layout_trace; mod lazy_intrinsic_towers; mod lazy_tape_side_alloc; diff --git a/crates/perry-runtime/src/hot_diag.rs b/crates/perry-runtime/src/hot_diag.rs index c68c4bb78a..934e9df0bc 100644 --- a/crates/perry-runtime/src/hot_diag.rs +++ b/crates/perry-runtime/src/hot_diag.rs @@ -503,6 +503,14 @@ fn ic_sink() -> &'static Option { static LAYOUT_SINK: OnceLock> = OnceLock::new(); static LAYOUT_ON: AtomicBool = AtomicBool::new(false); +#[cfg(test)] +thread_local! { + /// Per-test override/capture: mutating the process environment cannot + /// safely arm one libtest thread without affecting its neighbours. + static LAYOUT_TEST_ARMED: std::cell::Cell> = const { std::cell::Cell::new(None) }; + static LAYOUT_TEST_OUTPUT: RefCell = const { RefCell::new(String::new()) }; +} + fn layout_sink() -> &'static Option { LAYOUT_SINK.get_or_init(|| { let sink = sink_from_env("PERRY_LAYOUT_DIAG"); @@ -514,12 +522,43 @@ fn layout_sink() -> &'static Option { /// Is the per-object-layout occupancy instrument armed? #[inline] pub fn layout_on() -> bool { + #[cfg(test)] + if let Some(armed) = LAYOUT_TEST_ARMED.with(std::cell::Cell::get) { + return armed; + } if LAYOUT_SINK.get().is_none() { layout_sink(); } LAYOUT_ON.load(Ordering::Relaxed) } +/// Which of the three dynamically learned mask paths inserted a new key. +/// +/// Kept as a diagnostic counter rather than a field on `LayoutSlotMask`: the +/// latter is on the trace/store path even when diagnostics are off, and +/// changing its representation would violate this instrument's no-op contract. +#[derive(Clone, Copy)] +pub(crate) enum LayoutMaskInsertSite { + Birth = 0, + Rebuild = 1, + Store = 2, +} + +/// Marginal histograms of the surviving `LAYOUT_SLOT_MASKS` residue. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct LayoutResidueHistogram { + pub(crate) keys: u64, + pub(crate) closure: u64, + pub(crate) object: u64, + pub(crate) array: u64, + pub(crate) other: u64, + pub(crate) slots: [u64; 6], + pub(crate) pointer_share: [u64; 4], + pub(crate) space: [u64; 3], + pub(crate) est_tag_checks_saved_per_trace: u64, + pub(crate) prune_walk_us: u64, +} + /// One collection's view of the per-object layout tables and the 4096-bit /// address filter that is supposed to keep evacuation off them. /// @@ -553,22 +592,36 @@ pub struct LayoutDiag { outgrown: u64, /// Keys visited by prunes that DID rebuild — the walk that is still paid. rebuilt_keys: u64, + residue: LayoutResidueHistogram, + inserts_since: [u64; 3], } crate::perry_thread_local! { static LAYOUT_DIAG: RefCell = RefCell::new(LayoutDiag::default()); } +/// Record one newly inserted per-object pointer mask. The caller has already +/// tested [`layout_on`], so an unarmed run never resolves this counter's TLS. +#[inline] +pub(crate) fn layout_note_mask_insert(site: LayoutMaskInsertSite) { + LAYOUT_DIAG.with(|d| { + let mut d = d.borrow_mut(); + let counter = &mut d.inserts_since[site as usize]; + *counter = counter.saturating_add(1); + }); +} + /// Record one death-prune's occupancy. `rebuilt_filter` says whether this /// prune rebuilt the address filter from its survivors, or found the tables /// too full for a 4,096-bit sketch to discriminate and saturated it instead. -pub fn layout_note_prune( +pub(crate) fn layout_note_prune( typed_len: usize, masks_len: usize, filter_bits_set: usize, filter_bits_total: usize, rebuilt_filter: bool, useful_keys: usize, + residue: LayoutResidueHistogram, ) { LAYOUT_DIAG.with(|d| { let mut d = d.borrow_mut(); @@ -581,6 +634,7 @@ pub fn layout_note_prune( d.filter_bits_total = filter_bits_total; d.filter_bits_set_max = d.filter_bits_set_max.max(filter_bits_set); d.useful_keys = useful_keys; + d.residue = residue; if rebuilt_filter { d.rebuilt += 1; d.rebuilt_keys += (typed_len + masks_len) as u64; @@ -588,6 +642,12 @@ pub fn layout_note_prune( d.outgrown += 1; } let text = d.render(); + d.inserts_since = [0; 3]; + #[cfg(test)] + if LAYOUT_TEST_ARMED.with(std::cell::Cell::get) == Some(true) { + LAYOUT_TEST_OUTPUT.with(|out| out.borrow_mut().push_str(&text)); + return; + } if let Some(sink) = layout_sink() { write_sink(sink, &text); } @@ -640,10 +700,84 @@ impl LayoutDiag { " filter rebuilds={} over {} keys walked; outgrown-and-skipped={}", self.rebuilt, self.rebuilt_keys, self.outgrown ); + let r = self.residue; + let _ = writeln!( + out, + "[layout-diag] residue keys={} closure={} object={} array={} other={} \ + slots{{4-7={} 8-15={} 16-31={} 32-63={} 64-255={} 256+={}}} \ + ptr_share{{q1={} q2={} q3={} q4={}}} \ + space{{nursery={} old={} malloc={}}} \ + inserts_since{{birth={} rebuild={} store={}}} prune_walk_us={}", + r.keys, + r.closure, + r.object, + r.array, + r.other, + r.slots[0], + r.slots[1], + r.slots[2], + r.slots[3], + r.slots[4], + r.slots[5], + r.pointer_share[0], + r.pointer_share[1], + r.pointer_share[2], + r.pointer_share[3], + r.space[0], + r.space[1], + r.space[2], + self.inserts_since[0], + self.inserts_since[1], + self.inserts_since[2], + r.prune_walk_us, + ); + let per_key_prune_ns = if r.keys == 0 { + 0 + } else { + r.prune_walk_us.saturating_mul(1_000) / r.keys + }; + let _ = writeln!( + out, + "[layout-diag] price per_key_prune_ns={} est_tag_checks_saved_per_trace={}", + per_key_prune_ns, r.est_tag_checks_saved_per_trace + ); out } } +/// Test-only per-thread sink override, matching `GcDiagTestGuard`'s shape. +#[cfg(test)] +pub(crate) struct LayoutDiagTestGuard { + previous: Option, +} + +#[cfg(test)] +impl LayoutDiagTestGuard { + pub(crate) fn force(armed: bool) -> Self { + let previous = LAYOUT_TEST_ARMED.with(|value| value.replace(Some(armed))); + LAYOUT_TEST_OUTPUT.with(|out| out.borrow_mut().clear()); + LAYOUT_DIAG.with(|diag| *diag.borrow_mut() = LayoutDiag::default()); + Self { previous } + } + + pub(crate) fn output() -> String { + LAYOUT_TEST_OUTPUT.with(|out| out.borrow().clone()) + } + + pub(crate) fn residue() -> LayoutResidueHistogram { + LAYOUT_DIAG.with(|diag| diag.borrow().residue) + } +} + +#[cfg(test)] +impl Drop for LayoutDiagTestGuard { + fn drop(&mut self) { + LAYOUT_TEST_ARMED.with(|value| value.set(self.previous)); + LAYOUT_TEST_OUTPUT.with(|out| out.borrow_mut().clear()); + LAYOUT_DIAG.with(|diag| *diag.borrow_mut() = LayoutDiag::default()); + } +} + /// Is the IC-miss instrument armed? One relaxed load once initialised. #[inline] pub fn ic_on() -> bool { From 605e94ab66bfdec4bb627d862498b2a36c3818cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 16:07:13 +0200 Subject: [PATCH 24/27] docs(perf): report layout-mask residue histogram Record the replay evidence, field derivations, sabotage-capable tests, follow-up rule candidates, and exact Stage LP measurement request. Preserve the binding disk-floor result for the release gates that could not start. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit 227811137de9ec662c600016542d6a091e5c77cc) --- .../codex/REPORT_layout_residue.md | 207 ++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 cc-perf-campaign/codex/REPORT_layout_residue.md diff --git a/cc-perf-campaign/codex/REPORT_layout_residue.md b/cc-perf-campaign/codex/REPORT_layout_residue.md new file mode 100644 index 0000000000..6a2d42f90b --- /dev/null +++ b/cc-perf-campaign/codex/REPORT_layout_residue.md @@ -0,0 +1,207 @@ +# Per-object layout-mask residue histogram + +Runtime implementation SHA: `97b550085869f108c0789ec07ac620ab9de7f295` + +Young-log replay SHA: `dd279a8ba2de6b86a044d1074955e87fccf76d53` + +Branch: `perf/layout-residue-histogram`, based on +`fork/perf/minor-phases-and-logs` at +`151680fa0f6ffbf6ebb7d8166e2424fc0d031b49`. + +## Part 0: #9895 replay + +`git cherry 151680fa0 0a427a39f` prints `- 390796a11`: +`390796a11`'s one-pass layout prune and saturating filter are already in the +base. It prints `+ 19a6cd201`, so that commit alone was replayed. The only +conflict was `gc/tests/young_log_tests.rs`; the #9957 fixed-cost scanner tests +and #9895's layout-prune tests were kept as adjacent blocks. The runtime hunks +were not edited. + +`git range-diff 19a6cd201^! dd279a8ba^!` shows only commit metadata/message and +the expected `young_log_tests.rs` insertion context. No runtime/code hunk +differs. The replayed tests are present: + +- `dead_young_masked_owner_is_pruned_through_the_layout_log` +- `surviving_young_masked_owner_is_rekeyed_and_stays_logged` +- `old_layout_records_are_skipped_by_a_minor` + +The superseded `41a8af7da`, `bdd1fc003`, and `0a427a39f` were not replayed. +Consequently their full-walk test names/guard are intentionally absent: +`young_closure_prop_value_is_traced_and_moved_by_a_minor`, +`young_value_under_an_old_closure_owner_is_traced_by_a_minor`, +`old_closure_entries_survive_a_minor_full_walk`, and +`dropping_a_logged_closure_owner_trips_the_prune_rule2_check`. The original +young-walk tests remain because #9957 measured `closure.dynamic_props` winning +3.75 -> 3.06 ms and the base retains that log. + +## Diagnostic fields and derivation + +The instrument calls `layout_on()` exactly once in each non-empty full or +young death prune. When it is false, neither an `Instant` nor the surviving +mask pass is created. When true, `prune_walk_us` times only the existing prune +entry loop: the two `retain` walks for a full prune, or the drained young-key +loop for a minor. The diagnostic histogram pass runs afterwards and is not +included in that price. + +The first new line is: + +```text +[layout-diag] residue keys= closure= object= array= other= slots{4-7= 8-15= 16-31= 32-63= 64-255= 256+=} ptr_share{q1= q2= q3= q4=} space{nursery= old= malloc=} inserts_since{birth= rebuild= store=} prune_walk_us= +``` + +- `keys` is the surviving `LAYOUT_SLOT_MASKS.len()` after the death prune. +- `closure`, `object`, `array`, and `other` come from the tracked owner's + `GcHeader.obj_type`. `other` is defensive: legitimate mask owners are the + three layout-bearing kinds. +- Logical slots use the same owner facts as tracing: masked closure + `real_capture_count`, shape-derived `object_live_slot_count`, and array + `min(length, capacity)`. Other types fall back to GC payload words. The six + indices are `<=7`, 8-15, 16-31, 32-63, 64-255, and 256+. Under the default + floor a legitimate first-bucket mask is 4-7; the `<=7` implementation keeps + totals exhaustive if an override or corrupt residue produces a smaller one. +- Pointer slots are `LayoutSlotMask::count_slots(logical_slots)`. Quartiles + are non-overlapping: q1 `<=25%`, q2 `>25% and <=50%`, q3 `>50% and <=75%`, + q4 `>75%`. +- `nursery` is an arena owner in Eden or either survivor half. `malloc` is an + owner without `GC_FLAG_ARENA`. Remaining arena spaces (`Old`, `Longlived`, + and transient `PromotedYoung`) are `old` for this three-way price split. +- `birth`, `rebuild`, and `store` count fresh HashMap keys inserted by + `layout_init_from_slots`, `layout_rebuild_from_slots` (including the exact + rebuild wrapper), and `layout_note_slot`. Updates to an existing mask and + GC rekeys do not count. The three counters reset after every emitted prune. + Provenance is deliberately not stored on each `LayoutSlotMask`: doing so + would change a representation and trace/store path that exists when the + diagnostic is off. The separate counters answer insertion traffic, not the + historical origin of each survivor. +- `prune_walk_us` is `Instant::elapsed().as_micros()` around the existing loop, + saturated to `u64`. + +The second new line is: + +```text +[layout-diag] price per_key_prune_ns= est_tag_checks_saved_per_trace= +``` + +`per_key_prune_ns` is integer `prune_walk_us * 1000 / keys` (zero for no mask +keys). `est_tag_checks_saved_per_trace` is the saturated sum of +`logical_slots - pointer_slots` over every surviving mask: the maximum number +of exact tag checks the whole residue can avoid in one full trace. It is a +benefit ceiling, not a claim that every object is traced every cycle. + +This histogram can decide which owner kind and logical-slot bucket dominates, +how sparse the masks are, where their owners live, and whether the 64+ buckets +where a mask can plausibly earn program-global upkeep are a rounding error. +It does not change a threshold or selection rule. + +## Tests and sabotage evidence + +- `layout_residue_histogram_counts_by_kind_and_bucket` arms a per-thread test + sink, creates 5- and 20-capture closures with pointer captures, rebuilds one + 70-slot object, prunes with all owners surviving, and asserts kind, slot, + quartile, space, insert-site, saved-check, and output fields. It passed in + the focused release `layout` run (137 passed, 0 failed). Sabotage swapped + the 8-15 and 16-31 destinations; the + test failed with slots `[1, 1, 0, 0, 1, 0]` against expected + `[1, 0, 1, 0, 1, 0]`. +- `layout_residue_histogram_is_silent_when_unarmed` forces the per-thread sink + off, performs a prune, asserts no residue output, and reads a test-only + histogram-entry counter. It passed in the same focused release run. + Sabotage removed the full-prune `if layout_diag` guard; it failed with 1 + entry visited against expected 0. + +## Follow-up rule candidates + +### (a) Young-entry log for the prune + +This is orthogonal to deciding which masks deserve to exist, and Part 0 has +already stacked it here. `PerObjectLayoutHint.young_keys` is armed before every +new/moved young record becomes findable. A minor drains only those candidates, +drops stale/dead/promoted keys, and a full prune rebuilds the log from its +authoritative table walk. It changes repeated minor pruning from O(standing +keys) toward O(young churn), while the histogram describes the residue and is +armed-only. It cannot remove the full-trace walk or the mask's insert/store/ +death costs, so a high floor or shared representation can still win on top. + +### (b) Measured break-even floor + +A follow-up can replace the corpus default of four with a floor derived from +`per_key_prune_ns`, expected prunes during an owner's lifetime, and a separately +measured tag-check nanosecond cost. The mask's maximum per-trace return is +already printed as `slots - pointer_slots`; its standing prune cost is printed +per key. The relevant funnels are centralized: bulk birth and rebuild compare +against `layout_mask_min_slots`, while store-time creation goes through +`layout_prefers_scan_over_mask` (with the existing object-specific default of +eight). The missing input is tag-check cost and trace frequency by lifetime; +without those, converting the current price line directly into a slot number +would mix one prune with one full trace and repeat the campaign's wrong-ratio +failure mode. + +### (c) Closure masks keyed by function + +This is structurally possible when every instance agrees. `ClosureHeader` +provides a stable native `func_ptr` and `real_capture_count`, and +`layout_init_from_slots` observes the complete birth mask. A function-keyed +descriptor can store `(slot_count, mask)`, reuse it for agreeing instances, +and poison the function on the first differing birth or later capture store, +matching `SHAPE_LAYOUTS`' `Some/None` ambiguity pattern. Poison must make every +instance fall back to conservative tag scanning (safe even for earlier +instances that omitted per-object masks), while a diverging stored instance +can retain its exact per-object mask. The present header has only +`GC_LAYOUT_SIDE_MASK`, not a function-shared state, so mask resolution and +`layout_note_slot` would need an explicit shared lookup/fallback protocol. +Function entries then need no death prune because code pointers are stable and +the table is O(functions), but agreement/poison tests must cover post-birth +stores and differing capture counts before this becomes a rule. + +## Validation + +- `cargo test -p perry-runtime --release --lib -j4 -- --test-threads=1 layout`: + PASS before the two sabotage checks: 137 passed, 0 failed, 3,143 filtered. + Both sabotages were then reverted. A final rerun against the restored source + could not start because the mandatory free-space check fell below 12 GB. +- Full `cargo test -p perry-runtime --release --lib -j4 -- --test-threads=1`: + NOT RUN: blocked by the same free-space floor. +- `cargo build --release -p perry-runtime --features wasm-host -j4`: NOT RUN: + blocked by the same free-space floor. +- `rustfmt --check`: PASS on every changed Rust file. +- `git diff --check`: PASS. +- `scripts/check_file_size.sh`: PASS; no Rust file exceeds 2,000 lines + (`gc/layout.rs` is 1,999). +- No local cc run, as requested. + +Every Cargo attempt checked `df -g /` immediately beforehand. The final gate +rerun is currently blocked because the latest check reports 10 GB available, +below the binding 12 GB floor; no below-floor Cargo command was started. + +## Stage LP: exact perrymaster request and falsifiers + +Relink the #9957 tree plus `97b550085869f108c0789ec07ac620ab9de7f295` +(young-log prune plus residue histogram) on m6mp's object cache. Run two +graceful four-turn 3300-character candidate repetitions against `app-m6mp` +with `PERRY_GC_DIAG=1`; do not enable layout diagnostics for the CPU A/B, +because its deliberate whole-residue histogram is measurement overhead. Then +run one additional candidate capture with +`PERRY_GC_DIAG=1 PERRY_LAYOUT_DIAG=stderr`. + +The falsifiers are: + +- `dead_owner_side_table_pruning` for `LAYOUT_SLOT_MASKS + TYPED_LAYOUTS` + falls from 8.7-9.3 ms to at most 1 ms per steady minor, following the + +3...+10 young-key churn rather than 155-161k standing keys. +- Total steady copied-minor pause falls from about 46 ms to about 38 ms. +- Four-turn CPU improves by 1-2%; report both repetitions, not only a minimum. +- Peak and settled RSS remain within +3%. + +For the +29 MB settled result #9895 observed on main, one 160k-key log retains +two `Vec` buffers. At a 262,144-entry capacity that is about 4 MiB, so a +single table-owning thread cannot explain +29 MB; that delta would require +about seven similarly grown thread-local logs or allocator-retained secondary +effects. Stage LP must either report enough per-thread log capacity to account +for it or show the settled delta gone. Do not label +29 MB “the log” without +that reconciliation. + +From the layout-diagnostic run, preserve the residue lines for minors 5, 10, +13 (turn-2 maximum), 17, and 25. Report medians of both price fields over the +steady minors, alongside the phase values. Those rows decide whether closure/ +object/array and the 4-7/8-15/16-31/32-63/64+ populations justify a threshold +or function-keyed follow-up. From bf01eaa279352514da79f7ebff43f114133700f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 16:09:37 +0200 Subject: [PATCH 25/27] docs(perf): record parked-lane artifact cleanup Document the required target cleanup after the final Cargo gates were blocked by the campaign's 12 GB free-space floor. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit a4a6978a8606922bd41b41839bcae5e7537ed242) --- cc-perf-campaign/codex/REPORT_layout_residue.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cc-perf-campaign/codex/REPORT_layout_residue.md b/cc-perf-campaign/codex/REPORT_layout_residue.md index 6a2d42f90b..c670db000e 100644 --- a/cc-perf-campaign/codex/REPORT_layout_residue.md +++ b/cc-perf-campaign/codex/REPORT_layout_residue.md @@ -170,8 +170,13 @@ stores and differing capture counts before this becomes a rule. - No local cc run, as requested. Every Cargo attempt checked `df -g /` immediately beforehand. The final gate -rerun is currently blocked because the latest check reports 10 GB available, +rerun is currently blocked because the latest check reports 11 GB available, below the binding 12 GB floor; no below-floor Cargo command was started. +Per the campaign's parked-lane rule, this worktree's disposable `target/` was +then removed. The focused run did compile and execute its release test binary, +but that cleanup means there is no retained artifact mtime to present as final +build proof; the focused result is evidence for the named tests, not a +substitute for the blocked final gates. ## Stage LP: exact perrymaster request and falsifiers From 120a816173ee3261af281f869dfcb09af29cc90f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 7 Sep 2026 16:10:50 +0200 Subject: [PATCH 26/27] docs(perf): qualify replay test evidence Distinguish the two replay tests covered by the focused layout filter from the remaining named test whose full-suite gate was blocked by disk capacity. Claude-Session: https://claude.ai/code/session_011dhBmdn4vGgNibjo3oZqTo (cherry picked from commit 324166e4f6f2d65c6989d51e70fcffffe821bb17) --- cc-perf-campaign/codex/REPORT_layout_residue.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cc-perf-campaign/codex/REPORT_layout_residue.md b/cc-perf-campaign/codex/REPORT_layout_residue.md index c670db000e..69611e80ac 100644 --- a/cc-perf-campaign/codex/REPORT_layout_residue.md +++ b/cc-perf-campaign/codex/REPORT_layout_residue.md @@ -25,6 +25,11 @@ differs. The replayed tests are present: - `surviving_young_masked_owner_is_rekeyed_and_stays_logged` - `old_layout_records_are_skipped_by_a_minor` +The focused `layout` filter ran the first and third green. The middle name +does not contain `layout`; its required full-suite execution is one of the +gates the 12 GB disk floor prevented, so it is present but not claimed green +on this machine. + The superseded `41a8af7da`, `bdd1fc003`, and `0a427a39f` were not replayed. Consequently their full-walk test names/guard are intentionally absent: `young_closure_prop_value_is_traced_and_moved_by_a_minor`, From d14b601a8806f7971437e0b9c32f74143e27751e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 8 Sep 2026 00:53:41 +0200 Subject: [PATCH 27/27] fix(train): five gate failures from the #9976/#9977 stacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - copying.rs reached 2066 lines. The remembered-set scan and pinned-young preflight move to copying/remembered_scan.rs. Their `pub(super)` meant `gc` in the parent and means `copying` in the child, so both are widened to `pub(in crate::gc)` to keep the original reach. - regex_census.rs: `rows` is extended only under `regex-engine`, so the binding is unused-mut without that feature and REQUIRES mut with it. Scoped the allow to the feature-off build rather than dropping `mut`, which breaks the feature-on build. (My first attempt dropped it.) - shapes.rs: #9976 deliberately removed `family_push_back`'s production rekey caller — the scanner-internal rekey note explains why re-entering the writer funnel mid-walk is wrong — leaving shapes_test_support as its only consumer. Gated `#[cfg(test)]` to match. - Five new holders classified: BOX_YOUNG_ROOTS is covered_elsewhere, since every address in that minor remembered set is also in the box registry that scan_box_roots_mut walks; the four test seams and the histogram counter are not_a_gc_pointer. - PASS1_MARKED re-audited: gc/mod.rs gains two module declarations and one reg_scanner! registration, gc/census.rs widens side_tables() and adds census rows plus a test. A scanner registration adds a root SOURCE and runs nowhere between the census boundaries; census reporting runs from the diagnostic dump, not inside a cycle. LAYOUT_DIAG's entry is deleted: the holder became covered, which the gate calls the receipt. --- crates/perry-runtime/src/gc/copying.rs | 106 +---------------- .../src/gc/copying/remembered_scan.rs | 111 ++++++++++++++++++ crates/perry-runtime/src/gc/regex_census.rs | 4 + crates/perry-runtime/src/object/shapes.rs | 4 + scripts/gc_runtime_root_holders.json | 41 +++++-- 5 files changed, 154 insertions(+), 112 deletions(-) create mode 100644 crates/perry-runtime/src/gc/copying/remembered_scan.rs diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index e6da9db671..d4b82fe7fd 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -831,110 +831,8 @@ pub(super) fn last_untraced_decline_reason() -> &'static str { UNTRACED_DECLINE_REASON.with(std::cell::Cell::get) } -pub(super) fn scan_remembered_dirty_slots_copying( - snapshot: &RememberedDirtySnapshot, - mut covered: Option<&mut crate::fast_hash::PtrHashSet>, - mut visit: impl FnMut(*mut u64, *mut GcHeader, bool, &mut RememberedSetTraceStats), -) -> RememberedSetTraceStats { - let mut stats = RememberedSetTraceStats { - entries_scanned: snapshot.dirty_old_pages.len() - + snapshot.external_dirty_entries.len() - + snapshot.fallback_headers.len(), - dirty_pages_before: snapshot.dirty_pages.len(), - dirty_pages_scanned: snapshot.dirty_pages.len(), - ..RememberedSetTraceStats::default() - }; - let mut seen_headers = crate::fast_hash::new_ptr_hash_set(); - - let mut scan_header = |header: *mut GcHeader, stats: &mut RememberedSetTraceStats| unsafe { - if header.is_null() || !seen_headers.insert(header as usize) { - return; - } - let arena_parent = plausible_gc_header(header, true); - let malloc_parent = !arena_parent && plausible_gc_header(header, false); - if !arena_parent && !malloc_parent { - return; - } - let user = (header as *mut u8).add(GC_HEADER_SIZE) as usize; - if arena_parent - && !matches!( - crate::arena::classify_heap_generation(user), - crate::arena::HeapGeneration::Old - ) - { - return; - } - stats.old_objects_considered += 1; - stats.valid_roots += 1; - stats.dirty_objects_scanned += 1; - let mut changed = false; - let mut visit_slot = |slot: *mut u64, stats: &mut RememberedSetTraceStats| { - let external = !matches!( - crate::arena::classify_heap_generation(slot as usize), - crate::arena::HeapGeneration::Old - ); - let before = *slot; - visit(slot, header, external, stats); - changed |= *slot != before; - }; - let complete = - scan_dirty_object_slots(header, &snapshot.dirty_pages, stats, &mut visit_slot); - if complete { - if let Some(covered) = covered.as_deref_mut() { - covered.insert(header as usize); - } - } - if changed { - run_gc_rewrite_hook((*header).obj_type, user); - } - }; - - if !snapshot.dirty_old_pages.is_empty() { - crate::arena::old_arena_walk_objects_on_pages(&snapshot.dirty_old_pages, |header| { - scan_header(header as *mut GcHeader, &mut stats); - }); - } - for &(_, header_addr) in &snapshot.external_dirty_entries { - scan_header(header_addr as *mut GcHeader, &mut stats); - } - for header_addr in snapshot.fallback_headers.iter().copied() { - scan_header(header_addr as *mut GcHeader, &mut stats); - } - - stats.dirty_pages_after = remembered_dirty_page_count(); - stats -} - -/// The young-pin latch was clear, the preflight was skipped on that proof, and -/// the copier then met bytes that describe a pinned young object anyway. -/// This is the instant relocation would become unsafe, but it does not by -/// itself identify the violated invariant. In #7990 the header was internally -/// impossible (`GC_TYPE_MAP | GC_FLAG_INTERNED`), and the fault disappeared -/// when comparison operands were rooted; the pin latch itself was complete. -/// -/// There is no recovery: leaving the object in from-space strands the -/// referring slot on memory `copying_reset_from_spaces_and_flip` is about to -/// retire, and moving it invalidates a raw address nothing will rewrite. Abort -/// loudly at the faulting site instead of corrupting the heap silently. -#[cold] -#[inline(never)] -unsafe fn pinned_young_move_under_skipped_preflight(header: *mut GcHeader) -> ! { - // #7990: the report is built in `gc/pin.rs` from the header's own flags, - // because those flags are the only evidence that distinguishes an - // incomplete pin latch from a dangling pointer into recycled memory — and - // this message used to assert the former as fact while `gc_pin_sites.py`, - // the tool it told the reader to run, answered OK. - eprintln!( - "{}", - super::pin::pinned_young_move_report( - header as usize, - (*header).obj_type, - (*header).size, - (*header).gc_flags, - ) - ); - std::process::abort() -} +mod remembered_scan; +pub(super) use remembered_scan::*; pub(super) struct CopiedMinorEligibility { pub(super) eligible: bool, diff --git a/crates/perry-runtime/src/gc/copying/remembered_scan.rs b/crates/perry-runtime/src/gc/copying/remembered_scan.rs new file mode 100644 index 0000000000..026a2b7776 --- /dev/null +++ b/crates/perry-runtime/src/gc/copying/remembered_scan.rs @@ -0,0 +1,111 @@ +//! Remembered-set scanning and the pinned-young preflight for the copying +//! minor, split out of `copying.rs` for the 2000-line file cap. +//! +//! A child module, so `use super::*` reaches the parent's private items. + +use super::*; + +pub(in crate::gc) fn scan_remembered_dirty_slots_copying( + snapshot: &RememberedDirtySnapshot, + mut covered: Option<&mut crate::fast_hash::PtrHashSet>, + mut visit: impl FnMut(*mut u64, *mut GcHeader, bool, &mut RememberedSetTraceStats), +) -> RememberedSetTraceStats { + let mut stats = RememberedSetTraceStats { + entries_scanned: snapshot.dirty_old_pages.len() + + snapshot.external_dirty_entries.len() + + snapshot.fallback_headers.len(), + dirty_pages_before: snapshot.dirty_pages.len(), + dirty_pages_scanned: snapshot.dirty_pages.len(), + ..RememberedSetTraceStats::default() + }; + let mut seen_headers = crate::fast_hash::new_ptr_hash_set(); + + let mut scan_header = |header: *mut GcHeader, stats: &mut RememberedSetTraceStats| unsafe { + if header.is_null() || !seen_headers.insert(header as usize) { + return; + } + let arena_parent = plausible_gc_header(header, true); + let malloc_parent = !arena_parent && plausible_gc_header(header, false); + if !arena_parent && !malloc_parent { + return; + } + let user = (header as *mut u8).add(GC_HEADER_SIZE) as usize; + if arena_parent + && !matches!( + crate::arena::classify_heap_generation(user), + crate::arena::HeapGeneration::Old + ) + { + return; + } + stats.old_objects_considered += 1; + stats.valid_roots += 1; + stats.dirty_objects_scanned += 1; + let mut changed = false; + let mut visit_slot = |slot: *mut u64, stats: &mut RememberedSetTraceStats| { + let external = !matches!( + crate::arena::classify_heap_generation(slot as usize), + crate::arena::HeapGeneration::Old + ); + let before = *slot; + visit(slot, header, external, stats); + changed |= *slot != before; + }; + let complete = + scan_dirty_object_slots(header, &snapshot.dirty_pages, stats, &mut visit_slot); + if complete { + if let Some(covered) = covered.as_deref_mut() { + covered.insert(header as usize); + } + } + if changed { + run_gc_rewrite_hook((*header).obj_type, user); + } + }; + + if !snapshot.dirty_old_pages.is_empty() { + crate::arena::old_arena_walk_objects_on_pages(&snapshot.dirty_old_pages, |header| { + scan_header(header as *mut GcHeader, &mut stats); + }); + } + for &(_, header_addr) in &snapshot.external_dirty_entries { + scan_header(header_addr as *mut GcHeader, &mut stats); + } + for header_addr in snapshot.fallback_headers.iter().copied() { + scan_header(header_addr as *mut GcHeader, &mut stats); + } + + stats.dirty_pages_after = remembered_dirty_page_count(); + stats +} + +/// The young-pin latch was clear, the preflight was skipped on that proof, and +/// the copier then met bytes that describe a pinned young object anyway. +/// This is the instant relocation would become unsafe, but it does not by +/// itself identify the violated invariant. In #7990 the header was internally +/// impossible (`GC_TYPE_MAP | GC_FLAG_INTERNED`), and the fault disappeared +/// when comparison operands were rooted; the pin latch itself was complete. +/// +/// There is no recovery: leaving the object in from-space strands the +/// referring slot on memory `copying_reset_from_spaces_and_flip` is about to +/// retire, and moving it invalidates a raw address nothing will rewrite. Abort +/// loudly at the faulting site instead of corrupting the heap silently. +#[cold] +#[inline(never)] +pub(in crate::gc) unsafe fn pinned_young_move_under_skipped_preflight(header: *mut GcHeader) -> ! { + // #7990: the report is built in `gc/pin.rs` from the header's own flags, + // because those flags are the only evidence that distinguishes an + // incomplete pin latch from a dangling pointer into recycled memory — and + // this message used to assert the former as fact while `gc_pin_sites.py`, + // the tool it told the reader to run, answered OK. + eprintln!( + "{}", + super::pin::pinned_young_move_report( + header as usize, + (*header).obj_type, + (*header).size, + (*header).gc_flags, + ) + ); + std::process::abort() +} diff --git a/crates/perry-runtime/src/gc/regex_census.rs b/crates/perry-runtime/src/gc/regex_census.rs index ac6647b72c..3d49ae8f1e 100644 --- a/crates/perry-runtime/src/gc/regex_census.rs +++ b/crates/perry-runtime/src/gc/regex_census.rs @@ -9,6 +9,10 @@ pub(super) fn side_table_document_from(mut ordinary: Vec) -> serde // Replace the legacy RegExp tuples with the rich, reconciled rows. ordinary.retain(|(table, _, _)| !table.starts_with("regex.")); let non_regex_total = ordinary.iter().map(|(_, _, bytes)| *bytes).sum::(); + // `rows` is extended only under `regex-engine`; without that feature the + // binding is never mutated, so scope the allow to that configuration rather + // than dropping `mut` (which breaks the feature-on build). + #[cfg_attr(not(feature = "regex-engine"), allow(unused_mut))] let mut rows = ordinary .drain(..) .map(|(table, entries, bytes)| { diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 51724f362b..bab64b561b 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -313,6 +313,10 @@ impl ShapeTableInner { } #[inline] + // #9976 removed the production rekey caller deliberately (see the + // scanner-internal rekey note below); `shapes_test_support` is the only + // remaining consumer, and it is `#[cfg(test)]`. + #[cfg(test)] fn family_push_back(&mut self, keys: u64, id: u32) { self.note_young_keys(keys); self.families.entry(keys).or_default().push_back(id); diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 468bb5ef33..46329aa2af 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -172,6 +172,13 @@ "verdict": "test_only", "why": "#[cfg(test)] AtomicUsize one-shot flag that asks resolve_async_resource_handle to force a collection; stores only 0 or 1 and is absent from shipped binaries." }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "BOX_YOUNG_ROOTS", + "verdict": "covered_elsewhere", + "why": "#9976: the minor remembered set for box roots \u2014 a YoungLog of box addresses whose payload may matter to a minor. Every address in it is also in the box REGISTRY, which the module's own doc calls the authoritative full/major root set and which `scan_box_roots_mut` walks. The log is an accelerator over that set, not an independent holder: an address dropped from it is still reached through the registry.", + "scanner": "box::scan_box_roots_mut (crates/perry-runtime/src/box.rs), registered by reg_scanner! in crates/perry-runtime/src/gc/mod.rs" + }, { "file": "crates/perry-runtime/src/buffer/header.rs", "name": "BUFFER_ADDR_RANGE", @@ -224,6 +231,12 @@ "verdict": "not_a_gc_pointer", "why": "Cache miss counter (telemetry). Holds no address." }, + { + "file": "crates/perry-runtime/src/closure/dynamic_props.rs", + "name": "TEST_SUPPRESS_CLOSURE_YOUNG_NOTE", + "verdict": "not_a_gc_pointer", + "why": "#9976: a `Cell` test seam that suppresses the closure young-log note so a test can exercise the full-walk fallback. A FLAG; there is no slot for the collector." + }, { "file": "crates/perry-runtime/src/closure/registry.rs", "name": "CLOSURE_BODY_REGISTRY", @@ -276,7 +289,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module \u2014 all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -293,7 +306,7 @@ "sources": { "crates/perry-runtime/src/gc/census.rs": "0fd14b011bdbe0ae561e105d6d1acf0e9cc03e8a86ddef12685e307d1adee55a", "crates/perry-runtime/src/gc/cycle.rs": "77eadaf7c4157308c3b14be5e1ff11d7198b84b503a50ff2d244851248d3e800", - "crates/perry-runtime/src/gc/mod.rs": "78aca1306d4679a386e670cd96c4fa5f5046a6c28083e54fd6de7abec96da0e0", + "crates/perry-runtime/src/gc/mod.rs": "69bb74c7129709a9351f6624d7bb602a518816eacd3af8cf72ae8cb443312cf9", "crates/perry-runtime/src/gc/policy.rs": "dc9242ed40c0aa9c411d1ec0235c0219c6716dd82d56eb4d46578f7e889825d2", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } @@ -353,6 +366,12 @@ "verdict": "not_a_gc_pointer", "why": "#9965 verifier attribution: per-thread `Cell` tally of completed budgeted cycles. `note_incremental_completion` only increments the count and `incremental_completions_on_current_thread` only reads it; it cannot hold a heap pointer or NaN-boxed value." }, + { + "file": "crates/perry-runtime/src/gc/layout_tables.rs", + "name": "LAYOUT_RESIDUE_HISTOGRAM_ENTRIES", + "verdict": "not_a_gc_pointer", + "why": "#9976: how many entries the surviving per-object layout-mask residue histogram holds \u2014 a `Cell` COUNT for the diagnostic, never an address or a NaN-boxed value." + }, { "file": "crates/perry-runtime/src/gc/oldgen_defrag.rs", "name": "LAST_IDLE_PREDICTED_RELEASE", @@ -407,12 +426,6 @@ "verdict": "not_a_gc_pointer", "why": "Inline-cache miss diagnostics (`PERRY_IC_DIAG`). `IcDiag` is two `Instant`s, three counters, and `sites: HashMap` whose KEY is a PIC cache-slot address \u2014 malloc'd arena storage from `field_get_set/ic_slot.rs`, never GC heap \u2014 and whose value is a `String` plus counters. Nothing here is a managed pointer." }, - { - "file": "crates/perry-runtime/src/hot_diag.rs", - "name": "LAYOUT_DIAG", - "verdict": "not_a_gc_pointer", - "why": "#9807 layout-prune diagnostics (`PERRY_LAYOUT_DIAG`), off unless armed. Every field is a count, a length, or a running maximum (`prunes`, `typed_len`, `masks_len`, `typed_max`, `masks_max`, `filter_bits_*`, `useful_keys`, `rebuilt`). No field stores an address." - }, { "file": "crates/perry-runtime/src/hot_diag.rs", "name": "REGEX_DIAG", @@ -575,6 +588,12 @@ "verdict": "not_a_gc_pointer", "why": "Source order for Symbol-keyed class members: HashMap<(class_id, SymbolHeader::id, is_static), u32>. The u64 is the symbol's stable id (read once at registration in record_class_symbol_member_order), NOT its address, so the table needs no re-key when a Symbol is evacuated; the value is an order index. The member values themselves live in CLASS_SYMBOL_METHODS/CLASS_SYMBOL_ACCESSORS, which scan_class_symbol_member_keys_mut visits and rewrite_class_symbol_method_key_if_forwarded re-keys." }, + { + "file": "crates/perry-runtime/src/object/descriptor_state.rs", + "name": "TEST_SUPPRESS_DESCRIPTOR_YOUNG_NOTE", + "verdict": "not_a_gc_pointer", + "why": "#9976: the descriptor-table twin of TEST_SUPPRESS_CLOSURE_YOUNG_NOTE \u2014 a `Cell` test seam, not a pointer." + }, { "file": "crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs", "name": "TEST_TRANSITION_FAST_HITS", @@ -651,6 +670,12 @@ "verdict": "not_a_gc_pointer", "why": "#9893: how many by-name canonicality walks this process has done \u2014 a plain `u64` count whose whole purpose is to read 1 per realm and thereby prove the fast path is the path being taken. A NUMBER, never an address." }, + { + "file": "crates/perry-runtime/src/object/shapes.rs", + "name": "SHAPE_YOUNG_LOG_SUPPRESSED", + "verdict": "not_a_gc_pointer", + "why": "#9976: the shape-table twin of the same test seam \u2014 a `Cell` that forces the full shape walk instead of the young-log path. A flag, not a pointer." + }, { "file": "crates/perry-runtime/src/object/shapes_store.rs", "name": "ID_LIST_OP_STATS",