diff --git a/changelog.d/9802-outlined-ic-monomorphic-hit.md b/changelog.d/9802-outlined-ic-monomorphic-hit.md new file mode 100644 index 0000000000..a5afc8f6b4 --- /dev/null +++ b/changelog.d/9802-outlined-ic-monomorphic-hit.md @@ -0,0 +1,16 @@ +Gave the full-outline generic property get (`js_object_get_field_ic`, #5391 +path 3) the monomorphic inline-cache hit the inline diamond has. The outlined +helper observed typed feedback and then called `js_object_get_field_ic_miss` +unconditionally on every read of every heap receiver, so on a module past the +full-outline threshold — every minified bundle — the per-site cache was written +by every property read and consulted by nobody. Measured on the compiled +claude-code TUI, one 400-character reply: entries to the miss handler +2,725,376 → 649,216 and primes 2,180,102 → 114,732 over the same ~12,330 sites. +Guards mirror the emitted diamond's one for one; anything the hit path declines +still reaches the handler. `PERRY_IC_OUTLINE_FASTPATH=0` restores the previous +behaviour for measurement. + +Added `PERRY_IC_DIAG`'s prime split: every `pic_prime_get` is classified as +re-priming the token the site's MRU entry already held, priming a token that +was already in one of the four ways, or priming a genuinely new shape, with a +`PIC_WAY_STATE` census at prime time — globally and per site. diff --git a/crates/perry-runtime/src/hot_diag.rs b/crates/perry-runtime/src/hot_diag.rs index 48216ced2c..faa49fba65 100644 --- a/crates/perry-runtime/src/hot_diag.rs +++ b/crates/perry-runtime/src/hot_diag.rs @@ -405,6 +405,26 @@ struct SiteStat { key: String, misses: u64, by_reason: [u32; IC_MISS_REASONS], + /// Primes at this site whose token equals the one the site's MRU entry + /// ALREADY held. The cache was written with this shape, the next read of + /// the same shape came back to the miss handler anyway, and the handler + /// wrote the identical value again: the prime is not sticking. See + /// [`IcDiag::prime_same_token`]. + prime_same_token: u64, + /// Primes whose token differs from the MRU entry's — the site really is + /// seeing more than one receiver shape (polymorphism / megamorphism). + prime_new_token: u64, + /// Primes taken while the site's `PIC_WAY_STATE` was < 0, i.e. the ways + /// are latched off because the rotation was wider than they hold. + prime_while_megamorphic: u64, + /// Primes taken while the site had no way populated yet (state == 0). + prime_while_fresh: u64, + /// Primes taken while the site's ways were populated and live (state > 0). + prime_while_armed: u64, + /// Primes whose token was ALREADY sitting in one of the site's ways. The + /// cache held the right answer in a way and the read came back to the miss + /// handler regardless. See [`IcDiag::prime_in_ways`]. + prime_in_ways: u64, } #[derive(Default)] @@ -415,14 +435,96 @@ pub struct IcDiag { pub misses: u64, by_reason: [u64; IC_MISS_REASONS], sites: HashMap, + /// THE SPLIT. A site classified `own_inline_primed` misses, finds the + /// property as an own inline slot, and primes the cache — and then misses + /// again. Two mutually exclusive explanations, and the fix differs: + /// + /// * `prime_new_token` — the receiver's shape really did change between + /// the two reads. The site is polymorphic and the ways are (or should + /// be) doing their job; a fix would widen or re-tier them. + /// * `prime_same_token` — the site re-primed the shape it already had. + /// The cache holds the right answer and the emitted hit path did not + /// use it, or something invalidated it in between. That is a + /// priming/invalidation or IC-layout bug, not polymorphism. + /// + /// Measured on the claude-code TUI, shape identity is NOT the explanation + /// for the bulk of the misses (`{value, done}` iterator results share one + /// keys array since #7564 and their read sites still miss ~178k times per + /// turn), which is what this counter exists to settle. + pub prime_same_token: u64, + pub prime_new_token: u64, + pub prime_while_megamorphic: u64, + pub prime_while_fresh: u64, + pub prime_while_armed: u64, + /// The decisive counter for the `new_token` half. `prime_same_token` only + /// compares against the MRU entry (word 0), so a site rotating k <= 5 + /// shapes reports `new_token` on every prime even when the ways are doing + /// exactly what they were built for. This counts the primes whose token was + /// found in one of the four ways at prime time: the polymorphic cache + /// ALREADY held that shape's slot, and the emitted hit path still fell + /// through to the miss handler. + /// + /// So the three-way split of every prime is: + /// * `same_token` — re-primed the MRU shape (priming/invalidation), + /// * `new_token` + `in_ways` — the ways held it and were not consulted + /// (emitted gate / IC layout, i.e. codegen), + /// * `new_token` + not `in_ways` — a shape neither the MRU entry nor the + /// ways had (genuine polymorphism, or a first sighting). + pub prime_in_ways: u64, } crate::perry_thread_local! { static IC_DIAG: RefCell = RefCell::new(IcDiag::default()); } -/// Record one IC miss. `site` is the per-site cache slot address (stable for -/// the process lifetime), `key` the property-name string bytes. +/// Record one `pic_prime_get`, splitting it by whether the token the site is +/// being primed with is one it already held — in the MRU entry (`same`) or in +/// one of the ways (`in_ways`). See [`IcDiag::prime_same_token`] and +/// [`IcDiag::prime_in_ways`]. +/// +/// Diagnostic only: called from `pic_prime_get` behind [`ic_on`], and every +/// value it reads (`prev_tok`, `token`, `state`, the ways) is one the caller +/// already has in a register or in the cache line it has just touched. +pub fn ic_note_prime(site: usize, prev_tok: i64, token: i64, state: i64, in_ways: bool) { + IC_DIAG.with(|d| { + let mut d = d.borrow_mut(); + if d.started.is_none() { + d.started = Some(Instant::now()); + d.last_dump = d.started; + } + let same = prev_tok != 0 && prev_tok == token; + if same { + d.prime_same_token += 1; + } else { + d.prime_new_token += 1; + } + if in_ways { + d.prime_in_ways += 1; + } + match state.cmp(&0) { + std::cmp::Ordering::Less => d.prime_while_megamorphic += 1, + std::cmp::Ordering::Equal => d.prime_while_fresh += 1, + std::cmp::Ordering::Greater => d.prime_while_armed += 1, + } + let s = d.sites.entry(site).or_default(); + if same { + s.prime_same_token += 1; + } else { + s.prime_new_token += 1; + } + if in_ways { + s.prime_in_ways += 1; + } + match state.cmp(&0) { + std::cmp::Ordering::Less => s.prime_while_megamorphic += 1, + std::cmp::Ordering::Equal => s.prime_while_fresh += 1, + std::cmp::Ordering::Greater => s.prime_while_armed += 1, + } + }); +} + +/// Record one IC miss. `site` is the per-site cache address (stable for the +/// process lifetime), `key` the property-name string bytes. pub fn ic_note(site: usize, key: &[u8], reason: IcMissReason) { IC_DIAG.with(|d| { let mut d = d.borrow_mut(); @@ -470,9 +572,33 @@ impl IcDiag { } } out.push('\n'); + // THE SPLIT: of every prime, how many re-primed the token the site + // already held (the cache was right and was not used) versus a token + // it had not seen (real polymorphism)? + let primes = self.prime_same_token + self.prime_new_token; + if primes != 0 { + let pct = |n: u64| 100.0 * n as f64 / primes as f64; + let _ = writeln!( + out, + " primes={primes} same_token={} ({:.1} %) new_token={} ({:.1} %) \ + in_ways={} ({:.1} %) | way_state: fresh={} armed={} megamorphic={}", + self.prime_same_token, + pct(self.prime_same_token), + self.prime_new_token, + pct(self.prime_new_token), + self.prime_in_ways, + pct(self.prime_in_ways), + self.prime_while_fresh, + self.prime_while_armed, + self.prime_while_megamorphic + ); + } let mut rows: Vec<&SiteStat> = self.sites.values().collect(); rows.sort_by_key(|s| std::cmp::Reverse(s.misses)); - let _ = writeln!(out, " misses key reasons"); + let _ = writeln!( + out, + " misses same/new/inways fresh/armed/mega key reasons" + ); for s in rows.iter().take(40) { let mut reasons = String::new(); let mut idx: Vec = (0..IC_MISS_REASONS) @@ -482,7 +608,18 @@ impl IcDiag { for i in idx.iter().take(3) { let _ = write!(reasons, " {}={}", IC_REASON_NAMES[*i], s.by_reason[*i]); } - let _ = writeln!(out, " {:6} {:<24}{reasons}", s.misses, s.key); + let _ = writeln!( + out, + " {:6} {:>8}/{}/{:<8} {:>7}/{}/{:<8} {:<24}{reasons}", + s.misses, + s.prime_same_token, + s.prime_new_token, + s.prime_in_ways, + s.prime_while_fresh, + s.prime_while_armed, + s.prime_while_megamorphic, + s.key + ); } out } diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index e4c6020c66..7e4b9c40bf 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -315,6 +315,27 @@ pub(crate) unsafe fn pic_prime_get(cache: *mut PicCache, token: i64, slot: i64) let c = &mut *cache; let prev_tok = c[0]; let prev_slot = c[1]; + // `PERRY_IC_DIAG`: the prime split. `prev_tok == token` means this site is + // being primed with the shape its MRU entry ALREADY held — the cache was + // written, the next read of that same shape came back here anyway, and we + // are about to write the identical value again. That is a priming or + // invalidation problem, not polymorphism, and it is a different fix from + // `prev_tok != token` (the receiver really did change shape). Read before + // the write below, because the write destroys the evidence. + if crate::hot_diag::ic_on() { + // Was `token` already sitting in a WAY? The MRU comparison alone cannot + // tell a site rotating k <= PIC_WAYS+1 shapes (the ways doing their job) + // from one whose cached answer the emitted gate never consulted. Read + // here, before the loop below evicts `token` from its way. + let in_ways = (0..PIC_WAYS).any(|w| c[PIC_WAY_BASE + w * 2] == token); + crate::hot_diag::ic_note_prime( + cache as usize, + prev_tok, + token, + c[PIC_WAY_STATE], + in_ways, + ); + } c[0] = token; c[1] = slot; // Megamorphic. A rotation wider than the ways hold never hits one, so the @@ -498,7 +519,22 @@ fn ic_diag_note( std::slice::from_raw_parts(crate::string::string_data(key), (*key).byte_len as usize) } }; - crate::hot_diag::ic_note(cache_slot as usize, bytes, reason); + // Key the site by the RESOLVED cache, not by the slot that points at it, so + // these rows merge with the ones `pic_prime_get` records (it only ever has + // the resolved cache). A site that has never primed has no cache yet; key + // it by the slot, which is stable and has no prime rows to merge with. + // SAFETY: `cache_slot` is the codegen-emitted per-site slot (or null on the + // earliest exits, which `pic_slot_peek` handles); peeking only reads the + // published pointer and never allocates. + let site = unsafe { + let cache = pic_slot_peek(cache_slot); + if cache.is_null() { + cache_slot as usize + } else { + cache as usize + } + }; + crate::hot_diag::ic_note(site, bytes, reason); } #[no_mangle] @@ -977,6 +1013,115 @@ pub extern "C" fn js_object_get_field_ic_miss( /// - `site_id`: the typed-feedback site id /// - `cache_slot`: the per-site [`PicCacheSlot`] (resolved and primed by /// `..._ic_miss`) +/// `PERRY_IC_OUTLINE_FASTPATH=0` sends every outlined read back to the miss +/// handler, so the same binary can be measured with and without the hit path +/// one environment variable apart. Measurement only — nothing in the runtime +/// branches on it for behaviour, and the two settings are observationally +/// identical. +#[inline] +fn outlined_mru_hit_enabled() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + crate::gc::env_default_on_from_value( + std::env::var("PERRY_IC_OUTLINE_FASTPATH").ok().as_deref(), + ) + }) +} + +/// The inline-cache HIT the full-outline path never had. +/// +/// # Why this exists +/// +/// `js_object_get_field_ic` (#5391 path 3) replaces the inline generic-get +/// diamond with one call in oversized modules. The diamond's monomorphic +/// fast-load was traded away with it, and nothing replaced it: the helper +/// observed feedback and then called `js_object_get_field_ic_miss` +/// **unconditionally, on every read**, which primed the site's cache and +/// returned. The cache was written by every read and consulted by nobody. +/// +/// That is not a small trade on a minified bundle, because the threshold that +/// turns full-outlining on (4,000 callables) is met by the *whole module*, so +/// EVERY generic property read in such a program takes it. Measured on the +/// compiled claude-code TUI, one 400-character reply: 2,663,424 entries to the +/// miss handler over 12,326 sites, of which 2,122,626 primed, and **95.2 % of +/// those primes wrote the token the site's MRU entry already held** +/// (`PERRY_IC_DIAG`'s prime split). The four hottest sites — `.done`, +/// `.ambiguousAsWide`, `.value`, `.segment`, ~195k reads each — each recorded +/// exactly ONE new-token prime and ~195k same-token primes, with +/// `PIC_WAY_STATE` still 0. Perfectly monomorphic sites, a cache holding the +/// right answer, and the full miss ladder walked every time. +/// +/// So this is not a new cache or a new policy: it is the *existing* per-site +/// cache being read on the path that writes it. +/// +/// # The guards are the emitted diamond's, one for one +/// +/// Receiver is a real heap pointer (`>= HANDLE_BAND_MAX`), a `GC_TYPE_OBJECT` +/// with `OBJ_FLAG_HAS_DESCRIPTORS` clear, its shape stamp is non-zero and +/// equal to the cached token, and the cached slot carries no +/// `IC_SLOT_OVERFLOW_BIT`. Those are exactly the predicates +/// `lower_generic_property_get` emits before `pic.hit`, evaluated in the same +/// order, and the raw header loads are the same ones it emits — the caller has +/// already established the pointer tag, which is what licenses them there and +/// here. A `TAG_HOLE` in the slot is a deleted field and misses, as it does +/// there. +/// +/// Word 2 (the Array-subclass named-prefix token) and the polymorphic ways are +/// deliberately NOT served here: they are 2.5 % of primes between them and +/// each needs its own proof. They keep falling through to the handler. +/// +/// # Safety +/// `obj_handle` is the receiver with the NaN-box tag already masked off, and +/// the caller has established that the tag was `POINTER`/`STRING`. `cache_slot` +/// is the codegen-emitted per-site slot or null. +#[inline] +unsafe fn pic_outlined_mru_hit( + obj_handle: *const ObjectHeader, + cache_slot: *mut PicCacheSlot, +) -> Option { + if !outlined_mru_hit_enabled() { + return None; + } + let addr = obj_handle as usize; + if !crate::value::addr_class::is_above_handle_band(addr) { + return None; + } + // The site has never primed: there is nothing to hit, and resolving the + // slot is the miss handler's job. + let cache = pic_slot_peek(cache_slot); + if cache.is_null() { + return None; + } + let header = &*((addr - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader); + if header.obj_type != crate::gc::GC_TYPE_OBJECT + || header._reserved & crate::gc::OBJ_FLAG_HAS_DESCRIPTORS != 0 + { + return None; + } + // `object_shape_stamp` answers 0 for a receiver whose `parent_class_id` is + // not a ShapeId, which is what keeps a keyless receiver out of an empty + // cache slot (#809). + let stamp = crate::object::shapes::object_shape_stamp(obj_handle); + if stamp == 0 { + return None; + } + let c = &*cache; + if c[0] != (stamp as u64 | crate::object::shapes::PIC_ID_TOKEN_BIT) as i64 { + return None; + } + let slot = c[1]; + if (slot as u64) & u64::from(crate::proxy::IC_SLOT_OVERFLOW_BIT) != 0 { + return None; + } + let field = *((obj_handle as *const u8) + .add(std::mem::size_of::() + slot as usize * 8) + as *const f64); + if field.to_bits() == crate::value::TAG_HOLE { + return None; + } + Some(field) +} + #[no_mangle] pub extern "C" fn js_object_get_field_ic( obj_bits: i64, @@ -1016,6 +1161,13 @@ pub extern "C" fn js_object_get_field_ic( // is primed for any future inline sites sharing this global). if (tag & 0xFFFD) == 0x7FFD { crate::typed_feedback::js_typed_feedback_observe_property_get(site_id, obj_handle, key); + // The monomorphic hit the emitted diamond does inline. Everything it + // declines still reaches the handler below, so this only ever removes + // work. See `pic_outlined_mru_hit`. + if let Some(value) = unsafe { pic_outlined_mru_hit(obj_handle, cache_slot) } { + crate::typed_feedback::js_typed_feedback_record_guard_pass(site_id); + return value; + } return js_object_get_field_ic_miss(obj_handle, key, cache_slot); } // Invalid (non-pointer) receiver. `undefined`/`null` throw a TypeError (#462 — diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 714a4f81c6..db47f42621 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -6,6 +6,11 @@ #[cfg(feature = "regex-engine")] use regex::Regex; use std::cell::RefCell; +// Every use of `HashMap` in this file is inside a `#[cfg(feature = "regex-engine")]` +// block, so an unconditional import is an unused-import error under the +// `warnings` job's `-D warnings` when `perry`'s own binaries pull the runtime +// in without that feature. +#[cfg(feature = "regex-engine")] use std::collections::HashMap; use std::ptr; use std::sync::Arc;