diff --git a/changelog.d/9807-layout-prune-single-pass.md b/changelog.d/9807-layout-prune-single-pass.md new file mode 100644 index 0000000000..ab8c8fe72c --- /dev/null +++ b/changelog.d/9807-layout-prune-single-pass.md @@ -0,0 +1,33 @@ +**The per-object layout death-prune walks its tables once instead of three +times, and no longer allocates a `Vec` of every live key** — 50.6 MB of a +compiled claude-code turn's 304 MB in the layout tables (#9792). + +`prune_dead_per_object_layout_owners` visited every surviving key three times +per collection: `retain` to drop the dead owners, then +`layout_addr_filter_rebuild` — which first collected all of them into a +`Vec` — and then `recount_young_layout_records` to re-derive the +nursery-key count. The last two want exactly the survivor set `retain` is +already walking, so both fold into its closure. The `Vec` is gone from the +rebuild's other caller too. + +The measurement that prompted it also found the accelerator these tables sit +behind unable to do its job. `layout_addr_filter_may_hold` is a 4,096-bit +one-hash sketch documented for "one or two entries, ~0.05 % false positives"; +a new `PERRY_LAYOUT_DIAG` instrument reports **162,258 live keys and 4,096 of +4,096 bits set** on one 400-character claude-code reply. Every probe answers +"may hold", so the early returns in `transfer_per_object_descriptor` and +`transfer_per_object_slot_mask` never fire, and each rebuild was an O(live +keys) walk restoring the all-ones state it started from. Past four times the +bit count — 16,384 keys, where the false-positive rate is already 98.2 % — the +rebuild now sets all ones directly, the same conservative answer reached in +O(1); below that the filter keeps exactly the selectivity it has today. The +instrument says so out loud rather than leaving it to be inferred. Widening the sketch is not available +from the runtime: its geometry and hash are mirrored in `perry-codegen`'s +`emit_gated_forget_object_layout`, and discriminating at 162k keys would take +~190 KB of inline thread-local storage per thread. + +`transfer_per_object_descriptor` also gained the emptiness test its shared +flag cannot express: the flag and the filter are common to both per-object +tables, so a full slot-mask table drags every relocation into the typed-layout +map as well — which on cc is permanently empty (typed=0, masks=162,258). One +`len` load replaces two hashes per evacuated object. diff --git a/crates/perry-runtime/src/gc/layout_tables.rs b/crates/perry-runtime/src/gc/layout_tables.rs index e5d71a1c03..420c2b1df0 100644 --- a/crates/perry-runtime/src/gc/layout_tables.rs +++ b/crates/perry-runtime/src/gc/layout_tables.rs @@ -168,21 +168,11 @@ fn note_new_layout_record(user_ptr: usize) { } } -/// Re-derive this thread's young-record count from the live keys and publish -/// the delta. Runs after every death prune (all cycle kinds) and whenever the -/// tables empty, so promotion (a key moving to an old page) and death both -/// bring the count back down. -fn recount_young_layout_records() { - let live = { - let masks = hot_layout_slot_masks().borrow(); - let typed = hot_typed_layouts().borrow(); - masks - .keys() - .chain(typed.keys()) - .filter(|key| layout_key_may_be_nursery(**key)) - .count() - }; - let live = u32::try_from(live).unwrap_or(u32::MAX); +/// Publish this thread's young-record count and the delta to the process +/// total. The count itself is derived by the death prune's single pass over +/// the live keys (all cycle kinds), so promotion (a key moving to an old page) +/// and death both bring it back down without a walk of their own. +fn publish_young_layout_records(live: u32) { let prev = hot_per_object_layout_hint().young_records.replace(live); use std::sync::atomic::Ordering::SeqCst; if live > prev { @@ -204,28 +194,97 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn( if !per_object_layouts_maybe_nonempty() { return; } + // ONE pass over each table, not three. The old shape visited every live + // key three times per collection — `retain`, then + // `layout_addr_filter_rebuild` (which first collected them all into a + // `Vec`), then `recount_young_layout_records` — and the survivor + // set the last two want is exactly what `retain` is already walking. On cc + // that was 162k keys x 47 prunes per 400-character reply, with the `Vec` + // alone allocating 50.6 MB of the turn's 304 MB (#9792). + let hint = hot_per_object_layout_hint(); + // A filter this occupancy has outgrown is not worth rebuilding: see + // [`layout_addr_filter_saturating_occupancy`]. Decide from the pre-prune + // size, which bounds the survivor count from above, so the decision is one + // branch captured by the closure rather than a test per key. + let occupancy = hot_layout_slot_masks().borrow().len() + hot_typed_layouts().borrow().len(); + let rebuild_filter = occupancy <= layout_addr_filter_saturating_occupancy(); + if rebuild_filter { + // Cleared FIRST so the bits set below describe survivors only — a key + // that dies in this pass must leave no bit behind, which is the whole + // point of rebuilding. + layout_addr_filter_clear(); + } else { + layout_addr_filter_saturate(); + } + let mut young: u32 = 0; + let mut keep = |key: usize| { + if is_dead_owner(key) { + return false; + } + if rebuild_filter { + let (word, bit) = layout_addr_filter_slot(key); + // SAFETY: the filter is a plain `UnsafeCell` in this thread's own + // hot slot and nothing else holds a reference to it here. This is + // the same single-threaded access every probe makes; the tables + // borrowed around it are different thread-locals. + unsafe { + (*hint.filter.get())[word] |= bit; + } + } + if layout_key_may_be_nursery(key) { + young = young.saturating_add(1); + } + true + }; let masks_emptied = { let mut masks = hot_layout_slot_masks().borrow_mut(); let had = !masks.is_empty(); - masks.retain(|key, _| !is_dead_owner(*key)); + masks.retain(|key, _| keep(*key)); had && masks.is_empty() }; let typed_emptied = { let mut typed = hot_typed_layouts().borrow_mut(); let had = !typed.is_empty(); - typed.retain(|key, _| !is_dead_owner(*key)); + typed.retain(|key, _| keep(*key)); had && typed.is_empty() }; + publish_young_layout_records(young); + // Runs last: when it finds both tables empty it disarms the flag, zeroes + // the young count published above and clears the filter, which is the + // correct end state whichever branch the pass took. refresh_per_object_layouts_flag(masks_emptied || typed_emptied); - if per_object_layouts_maybe_nonempty() { - // Stale filter bits are what the pruned keys leave behind; rebuilding - // from the survivors keeps the runtime probes as selective as the - // tables really are. - layout_addr_filter_rebuild(); - recount_young_layout_records(); + if crate::hot_diag::layout_on() { + layout_diag_note_prune(rebuild_filter); } } +/// `PERRY_LAYOUT_DIAG`'s per-prune sample. Out of line and behind +/// [`crate::hot_diag::layout_on`] so an unarmed build pays one relaxed load. +#[cold] +fn layout_diag_note_prune(rebuilt_filter: bool) { + let (typed_len, masks_len) = ( + hot_typed_layouts().borrow().len(), + hot_layout_slot_masks().borrow().len(), + ); + let hint = hot_per_object_layout_hint(); + // SAFETY: as in the pass above — this thread's own filter, no other + // reference live. + let set = unsafe { + (*hint.filter.get()) + .iter() + .map(|w| w.count_ones() as usize) + .sum::() + }; + crate::hot_diag::layout_note_prune( + typed_len, + masks_len, + set, + LAYOUT_ADDR_FILTER_BITS, + rebuilt_filter, + layout_addr_filter_saturating_occupancy(), + ); +} + #[cfg(test)] pub(in crate::gc) fn test_per_object_layout_present(user_ptr: usize) -> bool { hot_layout_slot_masks().borrow().contains_key(&user_ptr) @@ -366,18 +425,74 @@ pub(in crate::gc) fn layout_addr_filter_note(user_ptr: usize) { /// their own (two keys may share one), so this is what keeps a workload that /// genuinely churns per-object records from saturating the filter forever. fn layout_addr_filter_rebuild() { + let occupancy = hot_layout_slot_masks().borrow().len() + hot_typed_layouts().borrow().len(); + if occupancy > layout_addr_filter_saturating_occupancy() { + // Walking the keys would set almost every bit, so this is where the + // rebuild lands anyway — reached in O(1) instead of O(live keys). + layout_addr_filter_saturate(); + return; + } layout_addr_filter_clear(); - let keys: Vec = { - let masks = hot_layout_slot_masks().borrow(); - let typed = hot_typed_layouts().borrow(); - masks.keys().copied().chain(typed.keys().copied()).collect() - }; let hint = hot_per_object_layout_hint(); - for k in keys { + // Straight from the tables: the `Vec` of every live key that used + // to buffer this walk allocated 50.6 MB per 400-character cc reply, and + // bought nothing — no borrow held here conflicts with the filter, which + // lives in a different thread-local. + let set_bit = |k: usize| { let (word, bit) = layout_addr_filter_slot(k); + // SAFETY: this thread's own filter, no other reference live; the + // tables borrowed around it are different thread-locals. unsafe { (*hint.filter.get())[word] |= bit; } + }; + for k in hot_layout_slot_masks().borrow().keys() { + set_bit(*k); + } + for k in hot_typed_layouts().borrow().keys() { + set_bit(*k); + } + hint.sets.set(0); +} + +/// Live keys past which the 4,096-bit sketch stops being an accelerator. +/// +/// The filter is a one-hash bitmap, so `n` live keys leave it answering +/// "may hold" for about `1 - e^(-n/4096)` of all addresses: 12 % at 512 keys, +/// 63 % at 4,096, and indistinguishable from "always yes" past ~16k. cc holds +/// **162,258** (`PERRY_LAYOUT_DIAG`, one 400-character reply), i.e. every bit +/// set, every probe positive, and every rebuild an O(live keys) walk that +/// restores exactly the all-ones state it started from. +/// +/// Sizing the filter up is not available here: the geometry and hash are +/// mirrored in generated code (`perry-codegen`'s +/// `emit_gated_forget_object_layout`), so widening it is a codegen change, and +/// a sketch that discriminated at 162k keys would need ~1.5 Mbit — 190 KB of +/// inline thread-local storage on every thread, to serve a workload that has +/// already lost the fast path. What is available is to stop *paying* for a +/// gate that cannot pay back: past this occupancy the filter is set to all +/// ones, which is the conservative answer it would have reached anyway, and +/// the walk is skipped. Nothing downstream changes behaviour — `may_hold` is +/// a hint whose `true` every caller already handles. +/// +/// Four times the bit count is deliberately far past the point where the +/// filter merely *degrades*: at 16,384 keys its false-positive rate is 98.2 %, +/// so a rebuilt filter still proves absence for under one address in fifty +/// while costing a walk of every live key. Below that the filter is left +/// exactly as it was — a workload holding a few thousand records keeps the +/// selectivity it has today, and this branch never fires for it. +#[inline] +fn layout_addr_filter_saturating_occupancy() -> usize { + LAYOUT_ADDR_FILTER_BITS * 4 +} + +/// Set every bit: "may hold" for any address. Conservative by construction — +/// the filter's `false` is the only load-bearing answer. +fn layout_addr_filter_saturate() { + let hint = hot_per_object_layout_hint(); + // SAFETY: this thread's own filter, no other reference live. + unsafe { + (*hint.filter.get()).fill(u64::MAX); } hint.sets.set(0); } @@ -809,6 +924,15 @@ pub(in crate::gc) fn transfer_per_object_descriptor(old_user: usize, new_user: u return false; } let mut typed = hot_typed_layouts().borrow_mut(); + // The flag and the filter above are shared with `LAYOUT_SLOT_MASKS`, so a + // full mask table drags every relocation in here even when this map is + // empty — which is cc's steady state (`PERRY_LAYOUT_DIAG`: typed=0, + // masks=162,258). An empty map has nothing to remove at either address, so + // the two hashes below are pure loss; the `len` test that proves it is one + // load. #9792. + if typed.is_empty() { + return false; + } typed.remove(&new_user); match typed.remove(&old_user) { Some(layout) => { diff --git a/crates/perry-runtime/src/hot_diag.rs b/crates/perry-runtime/src/hot_diag.rs index 48216ced2c..c7850a6720 100644 --- a/crates/perry-runtime/src/hot_diag.rs +++ b/crates/perry-runtime/src/hot_diag.rs @@ -324,6 +324,154 @@ fn ic_sink() -> &'static Option { }) } +// --------------------------------------------------------------------------- +// Per-object layout tables: occupancy vs the address filter that gates them +// --------------------------------------------------------------------------- + +static LAYOUT_SINK: OnceLock> = OnceLock::new(); +static LAYOUT_ON: AtomicBool = AtomicBool::new(false); + +fn layout_sink() -> &'static Option { + LAYOUT_SINK.get_or_init(|| { + let sink = sink_from_env("PERRY_LAYOUT_DIAG"); + LAYOUT_ON.store(sink.is_some(), Ordering::Relaxed); + sink + }) +} + +/// Is the per-object-layout occupancy instrument armed? +#[inline] +pub fn layout_on() -> bool { + if LAYOUT_SINK.get().is_none() { + layout_sink(); + } + LAYOUT_ON.load(Ordering::Relaxed) +} + +/// One collection's view of the per-object layout tables and the 4096-bit +/// address filter that is supposed to keep evacuation off them. +/// +/// The question this exists to settle: `layout_addr_filter_may_hold` is +/// documented for "one or two entries … ~0.05 % false-positive rate", and +/// `transfer_per_object_descriptor` / `transfer_per_object_slot_mask` return +/// early only when it says no. If the tables hold far more keys than the +/// filter has bits, it answers "maybe" for every address, both early returns +/// stop firing, and every evacuated object pays the full two-map hash path — +/// while nothing in the system says so out loud. +/// +/// `keys` is also what a filter rebuild used to cost: one `Vec` of +/// every live key per prune, plus a second full walk to recount the young +/// records. +#[derive(Default)] +pub struct LayoutDiag { + prunes: u64, + typed_len: usize, + masks_len: usize, + typed_max: usize, + masks_max: usize, + /// Set bits in the address filter, and its capacity in bits. + filter_bits_set: usize, + filter_bits_total: usize, + filter_bits_set_max: usize, + /// Live keys past which a rebuild stops producing a selective filter. + useful_keys: usize, + /// Prunes that rebuilt the filter from the survivors, and prunes that + /// found it already outgrown and skipped the walk. + rebuilt: u64, + outgrown: u64, + /// Keys visited by prunes that DID rebuild — the walk that is still paid. + rebuilt_keys: u64, +} + +crate::perry_thread_local! { + static LAYOUT_DIAG: RefCell = RefCell::new(LayoutDiag::default()); +} + +/// Record one death-prune's occupancy. `rebuilt_filter` says whether this +/// prune rebuilt the address filter from its survivors, or found the tables +/// too full for a 4,096-bit sketch to discriminate and saturated it instead. +pub fn layout_note_prune( + typed_len: usize, + masks_len: usize, + filter_bits_set: usize, + filter_bits_total: usize, + rebuilt_filter: bool, + useful_keys: usize, +) { + LAYOUT_DIAG.with(|d| { + let mut d = d.borrow_mut(); + d.prunes += 1; + d.typed_len = typed_len; + d.masks_len = masks_len; + d.typed_max = d.typed_max.max(typed_len); + d.masks_max = d.masks_max.max(masks_len); + d.filter_bits_set = filter_bits_set; + d.filter_bits_total = filter_bits_total; + d.filter_bits_set_max = d.filter_bits_set_max.max(filter_bits_set); + d.useful_keys = useful_keys; + if rebuilt_filter { + d.rebuilt += 1; + d.rebuilt_keys += (typed_len + masks_len) as u64; + } else { + d.outgrown += 1; + } + let text = d.render(); + if let Some(sink) = layout_sink() { + write_sink(sink, &text); + } + }); +} + +impl LayoutDiag { + fn render(&self) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(512); + let pct = |n: usize, d: usize| { + if d == 0 { + 0.0 + } else { + 100.0 * n as f64 / d as f64 + } + }; + let keys = self.typed_len + self.masks_len; + let _ = writeln!( + out, + "[layout-diag] prunes={} keys_now={} (typed={} masks={}) keys_max={} \ + (typed={} masks={})", + self.prunes, + keys, + self.typed_len, + self.masks_len, + self.typed_max + self.masks_max, + self.typed_max, + self.masks_max + ); + let _ = writeln!( + out, + " filter: {}/{} bits set ({:.1} %), max {}/{} ({:.1} %); selective up to \ + {} keys, so this table is {}", + self.filter_bits_set, + self.filter_bits_total, + pct(self.filter_bits_set, self.filter_bits_total), + self.filter_bits_set_max, + self.filter_bits_total, + pct(self.filter_bits_set_max, self.filter_bits_total), + self.useful_keys, + if keys > self.useful_keys { + "OUTGROWN it -- every probe answers `may hold`" + } else { + "within it" + } + ); + let _ = writeln!( + out, + " filter rebuilds={} over {} keys walked; outgrown-and-skipped={}", + self.rebuilt, self.rebuilt_keys, self.outgrown + ); + out + } +} + /// Is the IC-miss instrument armed? One relaxed load once initialised. #[inline] pub fn ic_on() -> bool {