From 4cb12f10f18b5b2b8e6761b34b64698efd906996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 5 Sep 2026 14:01:14 +0200 Subject: [PATCH] perf(gc): one pass over the per-object layout tables per prune, and a filter that admits when it is outgrown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `prune_dead_per_object_layout_owners` walked every live key three times per collection — `retain`, `layout_addr_filter_rebuild` (which buffered them all into a `Vec` first), then `recount_young_layout_records`. The last two want exactly the survivor set `retain` already visits, so they fold into its closure and the `Vec` disappears; on the compiled claude-code TUI that `Vec` alone allocated 50.6 MB per 400-character reply. The new `PERRY_LAYOUT_DIAG` instrument reports what made this expensive: 162,258 live keys against a 4,096-bit address sketch documented for "one or two entries", with all 4,096 bits set. Every probe answers "may hold", so the early returns the sketch exists to serve never fire, and each rebuild is an O(live keys) walk that restores the all-ones state it started from. Past one eighth of the bits the rebuild now reaches that state in O(1) instead. Widening the sketch is a codegen change — the geometry and hash are mirrored in `emit_gated_forget_object_layout` — and would need ~190 KB of inline TLS per thread to discriminate at this occupancy. `transfer_per_object_descriptor` gains the emptiness test its shared flag cannot express: one `len` load instead of two hashes per evacuated object, for a map that is empty for the whole of a cc turn. `LAYOUT_DIAG` is declared with `crate::perry_thread_local!`, as `scripts/check_thread_locals.py` requires of every new declaration. Claude-Session: https://claude.ai/code/session_014UZWia6L37DpA93VLtNK9m --- changelog.d/9807-layout-prune-single-pass.md | 33 ++++ crates/perry-runtime/src/gc/layout_tables.rs | 182 ++++++++++++++++--- crates/perry-runtime/src/hot_diag.rs | 148 +++++++++++++++ 3 files changed, 334 insertions(+), 29 deletions(-) create mode 100644 changelog.d/9807-layout-prune-single-pass.md diff --git a/changelog.d/9807-layout-prune-single-pass.md b/changelog.d/9807-layout-prune-single-pass.md new file mode 100644 index 0000000000..ab8c8fe72c --- /dev/null +++ b/changelog.d/9807-layout-prune-single-pass.md @@ -0,0 +1,33 @@ +**The per-object layout death-prune walks its tables once instead of three +times, and no longer allocates a `Vec` of every live key** — 50.6 MB of a +compiled claude-code turn's 304 MB in the layout tables (#9792). + +`prune_dead_per_object_layout_owners` visited every surviving key three times +per collection: `retain` to drop the dead owners, then +`layout_addr_filter_rebuild` — which first collected all of them into a +`Vec` — and then `recount_young_layout_records` to re-derive the +nursery-key count. The last two want exactly the survivor set `retain` is +already walking, so both fold into its closure. The `Vec` is gone from the +rebuild's other caller too. + +The measurement that prompted it also found the accelerator these tables sit +behind unable to do its job. `layout_addr_filter_may_hold` is a 4,096-bit +one-hash sketch documented for "one or two entries, ~0.05 % false positives"; +a new `PERRY_LAYOUT_DIAG` instrument reports **162,258 live keys and 4,096 of +4,096 bits set** on one 400-character claude-code reply. Every probe answers +"may hold", so the early returns in `transfer_per_object_descriptor` and +`transfer_per_object_slot_mask` never fire, and each rebuild was an O(live +keys) walk restoring the all-ones state it started from. Past four times the +bit count — 16,384 keys, where the false-positive rate is already 98.2 % — the +rebuild now sets all ones directly, the same conservative answer reached in +O(1); below that the filter keeps exactly the selectivity it has today. The +instrument says so out loud rather than leaving it to be inferred. Widening the sketch is not available +from the runtime: its geometry and hash are mirrored in `perry-codegen`'s +`emit_gated_forget_object_layout`, and discriminating at 162k keys would take +~190 KB of inline thread-local storage per thread. + +`transfer_per_object_descriptor` also gained the emptiness test its shared +flag cannot express: the flag and the filter are common to both per-object +tables, so a full slot-mask table drags every relocation into the typed-layout +map as well — which on cc is permanently empty (typed=0, masks=162,258). One +`len` load replaces two hashes per evacuated object. diff --git a/crates/perry-runtime/src/gc/layout_tables.rs b/crates/perry-runtime/src/gc/layout_tables.rs index e5d71a1c03..420c2b1df0 100644 --- a/crates/perry-runtime/src/gc/layout_tables.rs +++ b/crates/perry-runtime/src/gc/layout_tables.rs @@ -168,21 +168,11 @@ fn note_new_layout_record(user_ptr: usize) { } } -/// Re-derive this thread's young-record count from the live keys and publish -/// the delta. Runs after every death prune (all cycle kinds) and whenever the -/// tables empty, so promotion (a key moving to an old page) and death both -/// bring the count back down. -fn recount_young_layout_records() { - let live = { - let masks = hot_layout_slot_masks().borrow(); - let typed = hot_typed_layouts().borrow(); - masks - .keys() - .chain(typed.keys()) - .filter(|key| layout_key_may_be_nursery(**key)) - .count() - }; - let live = u32::try_from(live).unwrap_or(u32::MAX); +/// Publish this thread's young-record count and the delta to the process +/// total. The count itself is derived by the death prune's single pass over +/// the live keys (all cycle kinds), so promotion (a key moving to an old page) +/// and death both bring it back down without a walk of their own. +fn publish_young_layout_records(live: u32) { let prev = hot_per_object_layout_hint().young_records.replace(live); use std::sync::atomic::Ordering::SeqCst; if live > prev { @@ -204,28 +194,97 @@ pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn( if !per_object_layouts_maybe_nonempty() { return; } + // ONE pass over each table, not three. The old shape visited every live + // key three times per collection — `retain`, then + // `layout_addr_filter_rebuild` (which first collected them all into a + // `Vec`), then `recount_young_layout_records` — and the survivor + // set the last two want is exactly what `retain` is already walking. On cc + // that was 162k keys x 47 prunes per 400-character reply, with the `Vec` + // alone allocating 50.6 MB of the turn's 304 MB (#9792). + let hint = hot_per_object_layout_hint(); + // A filter this occupancy has outgrown is not worth rebuilding: see + // [`layout_addr_filter_saturating_occupancy`]. Decide from the pre-prune + // size, which bounds the survivor count from above, so the decision is one + // branch captured by the closure rather than a test per key. + let occupancy = hot_layout_slot_masks().borrow().len() + hot_typed_layouts().borrow().len(); + let rebuild_filter = occupancy <= layout_addr_filter_saturating_occupancy(); + if rebuild_filter { + // Cleared FIRST so the bits set below describe survivors only — a key + // that dies in this pass must leave no bit behind, which is the whole + // point of rebuilding. + layout_addr_filter_clear(); + } else { + layout_addr_filter_saturate(); + } + let mut young: u32 = 0; + let mut keep = |key: usize| { + if is_dead_owner(key) { + return false; + } + if rebuild_filter { + let (word, bit) = layout_addr_filter_slot(key); + // SAFETY: the filter is a plain `UnsafeCell` in this thread's own + // hot slot and nothing else holds a reference to it here. This is + // the same single-threaded access every probe makes; the tables + // borrowed around it are different thread-locals. + unsafe { + (*hint.filter.get())[word] |= bit; + } + } + if layout_key_may_be_nursery(key) { + young = young.saturating_add(1); + } + true + }; let masks_emptied = { let mut masks = hot_layout_slot_masks().borrow_mut(); let had = !masks.is_empty(); - masks.retain(|key, _| !is_dead_owner(*key)); + masks.retain(|key, _| keep(*key)); had && masks.is_empty() }; let typed_emptied = { let mut typed = hot_typed_layouts().borrow_mut(); let had = !typed.is_empty(); - typed.retain(|key, _| !is_dead_owner(*key)); + typed.retain(|key, _| keep(*key)); had && typed.is_empty() }; + publish_young_layout_records(young); + // Runs last: when it finds both tables empty it disarms the flag, zeroes + // the young count published above and clears the filter, which is the + // correct end state whichever branch the pass took. refresh_per_object_layouts_flag(masks_emptied || typed_emptied); - if per_object_layouts_maybe_nonempty() { - // Stale filter bits are what the pruned keys leave behind; rebuilding - // from the survivors keeps the runtime probes as selective as the - // tables really are. - layout_addr_filter_rebuild(); - recount_young_layout_records(); + if crate::hot_diag::layout_on() { + layout_diag_note_prune(rebuild_filter); } } +/// `PERRY_LAYOUT_DIAG`'s per-prune sample. Out of line and behind +/// [`crate::hot_diag::layout_on`] so an unarmed build pays one relaxed load. +#[cold] +fn layout_diag_note_prune(rebuilt_filter: bool) { + let (typed_len, masks_len) = ( + hot_typed_layouts().borrow().len(), + hot_layout_slot_masks().borrow().len(), + ); + let hint = hot_per_object_layout_hint(); + // SAFETY: as in the pass above — this thread's own filter, no other + // reference live. + let set = unsafe { + (*hint.filter.get()) + .iter() + .map(|w| w.count_ones() as usize) + .sum::() + }; + crate::hot_diag::layout_note_prune( + typed_len, + masks_len, + set, + LAYOUT_ADDR_FILTER_BITS, + rebuilt_filter, + layout_addr_filter_saturating_occupancy(), + ); +} + #[cfg(test)] pub(in crate::gc) fn test_per_object_layout_present(user_ptr: usize) -> bool { hot_layout_slot_masks().borrow().contains_key(&user_ptr) @@ -366,18 +425,74 @@ pub(in crate::gc) fn layout_addr_filter_note(user_ptr: usize) { /// their own (two keys may share one), so this is what keeps a workload that /// genuinely churns per-object records from saturating the filter forever. fn layout_addr_filter_rebuild() { + let occupancy = hot_layout_slot_masks().borrow().len() + hot_typed_layouts().borrow().len(); + if occupancy > layout_addr_filter_saturating_occupancy() { + // Walking the keys would set almost every bit, so this is where the + // rebuild lands anyway — reached in O(1) instead of O(live keys). + layout_addr_filter_saturate(); + return; + } layout_addr_filter_clear(); - let keys: Vec = { - let masks = hot_layout_slot_masks().borrow(); - let typed = hot_typed_layouts().borrow(); - masks.keys().copied().chain(typed.keys().copied()).collect() - }; let hint = hot_per_object_layout_hint(); - for k in keys { + // Straight from the tables: the `Vec` of every live key that used + // to buffer this walk allocated 50.6 MB per 400-character cc reply, and + // bought nothing — no borrow held here conflicts with the filter, which + // lives in a different thread-local. + let set_bit = |k: usize| { let (word, bit) = layout_addr_filter_slot(k); + // SAFETY: this thread's own filter, no other reference live; the + // tables borrowed around it are different thread-locals. unsafe { (*hint.filter.get())[word] |= bit; } + }; + for k in hot_layout_slot_masks().borrow().keys() { + set_bit(*k); + } + for k in hot_typed_layouts().borrow().keys() { + set_bit(*k); + } + hint.sets.set(0); +} + +/// Live keys past which the 4,096-bit sketch stops being an accelerator. +/// +/// The filter is a one-hash bitmap, so `n` live keys leave it answering +/// "may hold" for about `1 - e^(-n/4096)` of all addresses: 12 % at 512 keys, +/// 63 % at 4,096, and indistinguishable from "always yes" past ~16k. cc holds +/// **162,258** (`PERRY_LAYOUT_DIAG`, one 400-character reply), i.e. every bit +/// set, every probe positive, and every rebuild an O(live keys) walk that +/// restores exactly the all-ones state it started from. +/// +/// Sizing the filter up is not available here: the geometry and hash are +/// mirrored in generated code (`perry-codegen`'s +/// `emit_gated_forget_object_layout`), so widening it is a codegen change, and +/// a sketch that discriminated at 162k keys would need ~1.5 Mbit — 190 KB of +/// inline thread-local storage on every thread, to serve a workload that has +/// already lost the fast path. What is available is to stop *paying* for a +/// gate that cannot pay back: past this occupancy the filter is set to all +/// ones, which is the conservative answer it would have reached anyway, and +/// the walk is skipped. Nothing downstream changes behaviour — `may_hold` is +/// a hint whose `true` every caller already handles. +/// +/// Four times the bit count is deliberately far past the point where the +/// filter merely *degrades*: at 16,384 keys its false-positive rate is 98.2 %, +/// so a rebuilt filter still proves absence for under one address in fifty +/// while costing a walk of every live key. Below that the filter is left +/// exactly as it was — a workload holding a few thousand records keeps the +/// selectivity it has today, and this branch never fires for it. +#[inline] +fn layout_addr_filter_saturating_occupancy() -> usize { + LAYOUT_ADDR_FILTER_BITS * 4 +} + +/// Set every bit: "may hold" for any address. Conservative by construction — +/// the filter's `false` is the only load-bearing answer. +fn layout_addr_filter_saturate() { + let hint = hot_per_object_layout_hint(); + // SAFETY: this thread's own filter, no other reference live. + unsafe { + (*hint.filter.get()).fill(u64::MAX); } hint.sets.set(0); } @@ -809,6 +924,15 @@ pub(in crate::gc) fn transfer_per_object_descriptor(old_user: usize, new_user: u return false; } let mut typed = hot_typed_layouts().borrow_mut(); + // The flag and the filter above are shared with `LAYOUT_SLOT_MASKS`, so a + // full mask table drags every relocation in here even when this map is + // empty — which is cc's steady state (`PERRY_LAYOUT_DIAG`: typed=0, + // masks=162,258). An empty map has nothing to remove at either address, so + // the two hashes below are pure loss; the `len` test that proves it is one + // load. #9792. + if typed.is_empty() { + return false; + } typed.remove(&new_user); match typed.remove(&old_user) { Some(layout) => { diff --git a/crates/perry-runtime/src/hot_diag.rs b/crates/perry-runtime/src/hot_diag.rs index 48216ced2c..c7850a6720 100644 --- a/crates/perry-runtime/src/hot_diag.rs +++ b/crates/perry-runtime/src/hot_diag.rs @@ -324,6 +324,154 @@ fn ic_sink() -> &'static Option { }) } +// --------------------------------------------------------------------------- +// Per-object layout tables: occupancy vs the address filter that gates them +// --------------------------------------------------------------------------- + +static LAYOUT_SINK: OnceLock> = OnceLock::new(); +static LAYOUT_ON: AtomicBool = AtomicBool::new(false); + +fn layout_sink() -> &'static Option { + LAYOUT_SINK.get_or_init(|| { + let sink = sink_from_env("PERRY_LAYOUT_DIAG"); + LAYOUT_ON.store(sink.is_some(), Ordering::Relaxed); + sink + }) +} + +/// Is the per-object-layout occupancy instrument armed? +#[inline] +pub fn layout_on() -> bool { + if LAYOUT_SINK.get().is_none() { + layout_sink(); + } + LAYOUT_ON.load(Ordering::Relaxed) +} + +/// One collection's view of the per-object layout tables and the 4096-bit +/// address filter that is supposed to keep evacuation off them. +/// +/// The question this exists to settle: `layout_addr_filter_may_hold` is +/// documented for "one or two entries … ~0.05 % false-positive rate", and +/// `transfer_per_object_descriptor` / `transfer_per_object_slot_mask` return +/// early only when it says no. If the tables hold far more keys than the +/// filter has bits, it answers "maybe" for every address, both early returns +/// stop firing, and every evacuated object pays the full two-map hash path — +/// while nothing in the system says so out loud. +/// +/// `keys` is also what a filter rebuild used to cost: one `Vec` of +/// every live key per prune, plus a second full walk to recount the young +/// records. +#[derive(Default)] +pub struct LayoutDiag { + prunes: u64, + typed_len: usize, + masks_len: usize, + typed_max: usize, + masks_max: usize, + /// Set bits in the address filter, and its capacity in bits. + filter_bits_set: usize, + filter_bits_total: usize, + filter_bits_set_max: usize, + /// Live keys past which a rebuild stops producing a selective filter. + useful_keys: usize, + /// Prunes that rebuilt the filter from the survivors, and prunes that + /// found it already outgrown and skipped the walk. + rebuilt: u64, + outgrown: u64, + /// Keys visited by prunes that DID rebuild — the walk that is still paid. + rebuilt_keys: u64, +} + +crate::perry_thread_local! { + static LAYOUT_DIAG: RefCell = RefCell::new(LayoutDiag::default()); +} + +/// Record one death-prune's occupancy. `rebuilt_filter` says whether this +/// prune rebuilt the address filter from its survivors, or found the tables +/// too full for a 4,096-bit sketch to discriminate and saturated it instead. +pub fn layout_note_prune( + typed_len: usize, + masks_len: usize, + filter_bits_set: usize, + filter_bits_total: usize, + rebuilt_filter: bool, + useful_keys: usize, +) { + LAYOUT_DIAG.with(|d| { + let mut d = d.borrow_mut(); + d.prunes += 1; + d.typed_len = typed_len; + d.masks_len = masks_len; + d.typed_max = d.typed_max.max(typed_len); + d.masks_max = d.masks_max.max(masks_len); + d.filter_bits_set = filter_bits_set; + d.filter_bits_total = filter_bits_total; + d.filter_bits_set_max = d.filter_bits_set_max.max(filter_bits_set); + d.useful_keys = useful_keys; + if rebuilt_filter { + d.rebuilt += 1; + d.rebuilt_keys += (typed_len + masks_len) as u64; + } else { + d.outgrown += 1; + } + let text = d.render(); + if let Some(sink) = layout_sink() { + write_sink(sink, &text); + } + }); +} + +impl LayoutDiag { + fn render(&self) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(512); + let pct = |n: usize, d: usize| { + if d == 0 { + 0.0 + } else { + 100.0 * n as f64 / d as f64 + } + }; + let keys = self.typed_len + self.masks_len; + let _ = writeln!( + out, + "[layout-diag] prunes={} keys_now={} (typed={} masks={}) keys_max={} \ + (typed={} masks={})", + self.prunes, + keys, + self.typed_len, + self.masks_len, + self.typed_max + self.masks_max, + self.typed_max, + self.masks_max + ); + let _ = writeln!( + out, + " filter: {}/{} bits set ({:.1} %), max {}/{} ({:.1} %); selective up to \ + {} keys, so this table is {}", + self.filter_bits_set, + self.filter_bits_total, + pct(self.filter_bits_set, self.filter_bits_total), + self.filter_bits_set_max, + self.filter_bits_total, + pct(self.filter_bits_set_max, self.filter_bits_total), + self.useful_keys, + if keys > self.useful_keys { + "OUTGROWN it -- every probe answers `may hold`" + } else { + "within it" + } + ); + let _ = writeln!( + out, + " filter rebuilds={} over {} keys walked; outgrown-and-skipped={}", + self.rebuilt, self.rebuilt_keys, self.outgrown + ); + out + } +} + /// Is the IC-miss instrument armed? One relaxed load once initialised. #[inline] pub fn ic_on() -> bool {