perf(ic): the full-outline property get has no inline cache — give it the monomorphic hit

changelog.d/9802-outlined-ic-monomorphic-hit.md

@@ -0,0 +1,16 @@
+Gave the full-outline generic property get (`js_object_get_field_ic`, #5391
+path 3) the monomorphic inline-cache hit the inline diamond has. The outlined
+helper observed typed feedback and then called `js_object_get_field_ic_miss`
+unconditionally on every read of every heap receiver, so on a module past the
+full-outline threshold — every minified bundle — the per-site cache was written
+by every property read and consulted by nobody. Measured on the compiled
+claude-code TUI, one 400-character reply: entries to the miss handler
+2,725,376 → 649,216 and primes 2,180,102 → 114,732 over the same ~12,330 sites.
+Guards mirror the emitted diamond's one for one; anything the hit path declines
+still reaches the handler. `PERRY_IC_OUTLINE_FASTPATH=0` restores the previous
+behaviour for measurement.
+
+Added `PERRY_IC_DIAG`'s prime split: every `pic_prime_get` is classified as
+re-priming the token the site's MRU entry already held, priming a token that
+was already in one of the four ways, or priming a genuinely new shape, with a
+`PIC_WAY_STATE` census at prime time — globally and per site.

crates/perry-runtime/src/hot_diag.rs

@@ -405,6 +405,26 @@ struct SiteStat {
     key: String,
     misses: u64,
     by_reason: [u32; IC_MISS_REASONS],
+    /// Primes at this site whose token equals the one the site's MRU entry
+    /// ALREADY held. The cache was written with this shape, the next read of
+    /// the same shape came back to the miss handler anyway, and the handler
+    /// wrote the identical value again: the prime is not sticking. See
+    /// [`IcDiag::prime_same_token`].
+    prime_same_token: u64,
+    /// Primes whose token differs from the MRU entry's — the site really is
+    /// seeing more than one receiver shape (polymorphism / megamorphism).
+    prime_new_token: u64,
+    /// Primes taken while the site's `PIC_WAY_STATE` was < 0, i.e. the ways
+    /// are latched off because the rotation was wider than they hold.
+    prime_while_megamorphic: u64,
+    /// Primes taken while the site had no way populated yet (state == 0).
+    prime_while_fresh: u64,
+    /// Primes taken while the site's ways were populated and live (state > 0).
+    prime_while_armed: u64,
+    /// Primes whose token was ALREADY sitting in one of the site's ways. The
+    /// cache held the right answer in a way and the read came back to the miss
+    /// handler regardless. See [`IcDiag::prime_in_ways`].
+    prime_in_ways: u64,
 }
 
 #[derive(Default)]
@@ -415,14 +435,96 @@ pub struct IcDiag {
     pub misses: u64,
     by_reason: [u64; IC_MISS_REASONS],
     sites: HashMap<usize, SiteStat>,
+    /// THE SPLIT. A site classified `own_inline_primed` misses, finds the
+    /// property as an own inline slot, and primes the cache — and then misses
+    /// again. Two mutually exclusive explanations, and the fix differs:
+    ///
+    /// * `prime_new_token` — the receiver's shape really did change between
+    ///   the two reads. The site is polymorphic and the ways are (or should
+    ///   be) doing their job; a fix would widen or re-tier them.
+    /// * `prime_same_token` — the site re-primed the shape it already had.
+    ///   The cache holds the right answer and the emitted hit path did not
+    ///   use it, or something invalidated it in between. That is a
+    ///   priming/invalidation or IC-layout bug, not polymorphism.
+    ///
+    /// Measured on the claude-code TUI, shape identity is NOT the explanation
+    /// for the bulk of the misses (`{value, done}` iterator results share one
+    /// keys array since #7564 and their read sites still miss ~178k times per
+    /// turn), which is what this counter exists to settle.
+    pub prime_same_token: u64,
+    pub prime_new_token: u64,
+    pub prime_while_megamorphic: u64,
+    pub prime_while_fresh: u64,
+    pub prime_while_armed: u64,
+    /// The decisive counter for the `new_token` half. `prime_same_token` only
+    /// compares against the MRU entry (word 0), so a site rotating k <= 5
+    /// shapes reports `new_token` on every prime even when the ways are doing
+    /// exactly what they were built for. This counts the primes whose token was
+    /// found in one of the four ways at prime time: the polymorphic cache
+    /// ALREADY held that shape's slot, and the emitted hit path still fell
+    /// through to the miss handler.
+    ///
+    /// So the three-way split of every prime is:
+    /// * `same_token` — re-primed the MRU shape (priming/invalidation),
+    /// * `new_token` + `in_ways` — the ways held it and were not consulted
+    ///   (emitted gate / IC layout, i.e. codegen),
+    /// * `new_token` + not `in_ways` — a shape neither the MRU entry nor the
+    ///   ways had (genuine polymorphism, or a first sighting).
+    pub prime_in_ways: u64,
 }
 
 crate::perry_thread_local! {
     static IC_DIAG: RefCell<IcDiag> = RefCell::new(IcDiag::default());
 }
 
-/// Record one IC miss. `site` is the per-site cache slot address (stable for
-/// the process lifetime), `key` the property-name string bytes.
+/// Record one `pic_prime_get`, splitting it by whether the token the site is
+/// being primed with is one it already held — in the MRU entry (`same`) or in
+/// one of the ways (`in_ways`). See [`IcDiag::prime_same_token`] and
+/// [`IcDiag::prime_in_ways`].
+///
+/// Diagnostic only: called from `pic_prime_get` behind [`ic_on`], and every
+/// value it reads (`prev_tok`, `token`, `state`, the ways) is one the caller
+/// already has in a register or in the cache line it has just touched.
+pub fn ic_note_prime(site: usize, prev_tok: i64, token: i64, state: i64, in_ways: bool) {
+    IC_DIAG.with(|d| {
+        let mut d = d.borrow_mut();
+        if d.started.is_none() {
+            d.started = Some(Instant::now());
+            d.last_dump = d.started;
+        }
+        let same = prev_tok != 0 && prev_tok == token;
+        if same {
+            d.prime_same_token += 1;
+        } else {
+            d.prime_new_token += 1;
+        }
+        if in_ways {
+            d.prime_in_ways += 1;
+        }
+        match state.cmp(&0) {
+            std::cmp::Ordering::Less => d.prime_while_megamorphic += 1,
+            std::cmp::Ordering::Equal => d.prime_while_fresh += 1,
+            std::cmp::Ordering::Greater => d.prime_while_armed += 1,
+        }
+        let s = d.sites.entry(site).or_default();
+        if same {
+            s.prime_same_token += 1;
+        } else {
+            s.prime_new_token += 1;
+        }
+        if in_ways {
+            s.prime_in_ways += 1;
+        }
+        match state.cmp(&0) {
+            std::cmp::Ordering::Less => s.prime_while_megamorphic += 1,
+            std::cmp::Ordering::Equal => s.prime_while_fresh += 1,
+            std::cmp::Ordering::Greater => s.prime_while_armed += 1,
+        }
+    });
+}
+
+/// Record one IC miss. `site` is the per-site cache address (stable for the
+/// process lifetime), `key` the property-name string bytes.
 pub fn ic_note(site: usize, key: &[u8], reason: IcMissReason) {
     IC_DIAG.with(|d| {
         let mut d = d.borrow_mut();
@@ -470,9 +572,33 @@ impl IcDiag {
             }
         }
         out.push('\n');
+        // THE SPLIT: of every prime, how many re-primed the token the site
+        // already held (the cache was right and was not used) versus a token
+        // it had not seen (real polymorphism)?
+        let primes = self.prime_same_token + self.prime_new_token;
+        if primes != 0 {
+            let pct = |n: u64| 100.0 * n as f64 / primes as f64;
+            let _ = writeln!(
+                out,
+                "  primes={primes} same_token={} ({:.1} %) new_token={} ({:.1} %) \
+                 in_ways={} ({:.1} %) | way_state: fresh={} armed={} megamorphic={}",
+                self.prime_same_token,
+                pct(self.prime_same_token),
+                self.prime_new_token,
+                pct(self.prime_new_token),
+                self.prime_in_ways,
+                pct(self.prime_in_ways),
+                self.prime_while_fresh,
+                self.prime_while_armed,
+                self.prime_while_megamorphic
+            );
+        }
         let mut rows: Vec<&SiteStat> = self.sites.values().collect();
         rows.sort_by_key(|s| std::cmp::Reverse(s.misses));
-        let _ = writeln!(out, "  misses  key  reasons");
+        let _ = writeln!(
+            out,
+            "  misses   same/new/inways   fresh/armed/mega   key  reasons"
+        );
         for s in rows.iter().take(40) {
             let mut reasons = String::new();
             let mut idx: Vec<usize> = (0..IC_MISS_REASONS)
@@ -482,7 +608,18 @@ impl IcDiag {
             for i in idx.iter().take(3) {
                 let _ = write!(reasons, " {}={}", IC_REASON_NAMES[*i], s.by_reason[*i]);
             }
-            let _ = writeln!(out, "  {:6}  {:<24}{reasons}", s.misses, s.key);
+            let _ = writeln!(
+                out,
+                "  {:6}  {:>8}/{}/{:<8}  {:>7}/{}/{:<8}  {:<24}{reasons}",
+                s.misses,
+                s.prime_same_token,
+                s.prime_new_token,
+                s.prime_in_ways,
+                s.prime_while_fresh,
+                s.prime_while_armed,
+                s.prime_while_megamorphic,
+                s.key
+            );
         }
         out
     }

crates/perry-runtime/src/object/field_get_set/ic_miss.rs

@@ -315,6 +315,27 @@ pub(crate) unsafe fn pic_prime_get(cache: *mut PicCache, token: i64, slot: i64)
     let c = &mut *cache;
     let prev_tok = c[0];
     let prev_slot = c[1];
+    // `PERRY_IC_DIAG`: the prime split. `prev_tok == token` means this site is
+    // being primed with the shape its MRU entry ALREADY held — the cache was
+    // written, the next read of that same shape came back here anyway, and we
+    // are about to write the identical value again. That is a priming or
+    // invalidation problem, not polymorphism, and it is a different fix from
+    // `prev_tok != token` (the receiver really did change shape). Read before
+    // the write below, because the write destroys the evidence.
+    if crate::hot_diag::ic_on() {
+        // Was `token` already sitting in a WAY? The MRU comparison alone cannot
+        // tell a site rotating k <= PIC_WAYS+1 shapes (the ways doing their job)
+        // from one whose cached answer the emitted gate never consulted. Read
+        // here, before the loop below evicts `token` from its way.
+        let in_ways = (0..PIC_WAYS).any(|w| c[PIC_WAY_BASE + w * 2] == token);
+        crate::hot_diag::ic_note_prime(
+            cache as usize,
+            prev_tok,
+            token,
+            c[PIC_WAY_STATE],
+            in_ways,
+        );
+    }
     c[0] = token;
     c[1] = slot;
     // Megamorphic. A rotation wider than the ways hold never hits one, so the
@@ -498,7 +519,22 @@ fn ic_diag_note(
             std::slice::from_raw_parts(crate::string::string_data(key), (*key).byte_len as usize)
         }
     };
-    crate::hot_diag::ic_note(cache_slot as usize, bytes, reason);
+    // Key the site by the RESOLVED cache, not by the slot that points at it, so
+    // these rows merge with the ones `pic_prime_get` records (it only ever has
+    // the resolved cache). A site that has never primed has no cache yet; key
+    // it by the slot, which is stable and has no prime rows to merge with.
+    // SAFETY: `cache_slot` is the codegen-emitted per-site slot (or null on the
+    // earliest exits, which `pic_slot_peek` handles); peeking only reads the
+    // published pointer and never allocates.
+    let site = unsafe {
+        let cache = pic_slot_peek(cache_slot);
+        if cache.is_null() {
+            cache_slot as usize
+        } else {
+            cache as usize
+        }
+    };
+    crate::hot_diag::ic_note(site, bytes, reason);
 }
 
 #[no_mangle]
@@ -977,6 +1013,115 @@ pub extern "C" fn js_object_get_field_ic_miss(
 /// - `site_id`: the typed-feedback site id
 /// - `cache_slot`: the per-site [`PicCacheSlot`] (resolved and primed by
 ///   `..._ic_miss`)
+/// `PERRY_IC_OUTLINE_FASTPATH=0` sends every outlined read back to the miss
+/// handler, so the same binary can be measured with and without the hit path
+/// one environment variable apart. Measurement only — nothing in the runtime
+/// branches on it for behaviour, and the two settings are observationally
+/// identical.
+#[inline]
+fn outlined_mru_hit_enabled() -> bool {
+    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+    *ON.get_or_init(|| {
+        crate::gc::env_default_on_from_value(
+            std::env::var("PERRY_IC_OUTLINE_FASTPATH").ok().as_deref(),
+        )
+    })
+}
+
+/// The inline-cache HIT the full-outline path never had.
+///
+/// # Why this exists
+///
+/// `js_object_get_field_ic` (#5391 path 3) replaces the inline generic-get
+/// diamond with one call in oversized modules. The diamond's monomorphic
+/// fast-load was traded away with it, and nothing replaced it: the helper
+/// observed feedback and then called `js_object_get_field_ic_miss`
+/// **unconditionally, on every read**, which primed the site's cache and
+/// returned. The cache was written by every read and consulted by nobody.
+///
+/// That is not a small trade on a minified bundle, because the threshold that
+/// turns full-outlining on (4,000 callables) is met by the *whole module*, so
+/// EVERY generic property read in such a program takes it. Measured on the
+/// compiled claude-code TUI, one 400-character reply: 2,663,424 entries to the
+/// miss handler over 12,326 sites, of which 2,122,626 primed, and **95.2 % of
+/// those primes wrote the token the site's MRU entry already held**
+/// (`PERRY_IC_DIAG`'s prime split). The four hottest sites — `.done`,
+/// `.ambiguousAsWide`, `.value`, `.segment`, ~195k reads each — each recorded
+/// exactly ONE new-token prime and ~195k same-token primes, with
+/// `PIC_WAY_STATE` still 0. Perfectly monomorphic sites, a cache holding the
+/// right answer, and the full miss ladder walked every time.
+///
+/// So this is not a new cache or a new policy: it is the *existing* per-site
+/// cache being read on the path that writes it.
+///
+/// # The guards are the emitted diamond's, one for one
+///
+/// Receiver is a real heap pointer (`>= HANDLE_BAND_MAX`), a `GC_TYPE_OBJECT`
+/// with `OBJ_FLAG_HAS_DESCRIPTORS` clear, its shape stamp is non-zero and
+/// equal to the cached token, and the cached slot carries no
+/// `IC_SLOT_OVERFLOW_BIT`. Those are exactly the predicates
+/// `lower_generic_property_get` emits before `pic.hit`, evaluated in the same
+/// order, and the raw header loads are the same ones it emits — the caller has
+/// already established the pointer tag, which is what licenses them there and
+/// here. A `TAG_HOLE` in the slot is a deleted field and misses, as it does
+/// there.
+///
+/// Word 2 (the Array-subclass named-prefix token) and the polymorphic ways are
+/// deliberately NOT served here: they are 2.5 % of primes between them and
+/// each needs its own proof. They keep falling through to the handler.
+///
+/// # Safety
+/// `obj_handle` is the receiver with the NaN-box tag already masked off, and
+/// the caller has established that the tag was `POINTER`/`STRING`. `cache_slot`
+/// is the codegen-emitted per-site slot or null.
+#[inline]
+unsafe fn pic_outlined_mru_hit(
+    obj_handle: *const ObjectHeader,
+    cache_slot: *mut PicCacheSlot,
+) -> Option<f64> {
+    if !outlined_mru_hit_enabled() {
+        return None;
+    }
+    let addr = obj_handle as usize;
+    if !crate::value::addr_class::is_above_handle_band(addr) {
+        return None;
+    }
+    // The site has never primed: there is nothing to hit, and resolving the
+    // slot is the miss handler's job.
+    let cache = pic_slot_peek(cache_slot);
+    if cache.is_null() {
+        return None;
+    }
+    let header = &*((addr - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader);
+    if header.obj_type != crate::gc::GC_TYPE_OBJECT
+        || header._reserved & crate::gc::OBJ_FLAG_HAS_DESCRIPTORS != 0
+    {
+        return None;
+    }
+    // `object_shape_stamp` answers 0 for a receiver whose `parent_class_id` is
+    // not a ShapeId, which is what keeps a keyless receiver out of an empty
+    // cache slot (#809).
+    let stamp = crate::object::shapes::object_shape_stamp(obj_handle);
+    if stamp == 0 {
+        return None;
+    }
+    let c = &*cache;
+    if c[0] != (stamp as u64 | crate::object::shapes::PIC_ID_TOKEN_BIT) as i64 {
+        return None;
+    }
+    let slot = c[1];
+    if (slot as u64) & u64::from(crate::proxy::IC_SLOT_OVERFLOW_BIT) != 0 {
+        return None;
+    }
+    let field = *((obj_handle as *const u8)
+        .add(std::mem::size_of::<ObjectHeader>() + slot as usize * 8)
+        as *const f64);
+    if field.to_bits() == crate::value::TAG_HOLE {
+        return None;
+    }
+    Some(field)
+}
+
 #[no_mangle]
 pub extern "C" fn js_object_get_field_ic(
     obj_bits: i64,
@@ -1016,6 +1161,13 @@ pub extern "C" fn js_object_get_field_ic(
     // is primed for any future inline sites sharing this global).
     if (tag & 0xFFFD) == 0x7FFD {
         crate::typed_feedback::js_typed_feedback_observe_property_get(site_id, obj_handle, key);
+        // The monomorphic hit the emitted diamond does inline. Everything it
+        // declines still reaches the handler below, so this only ever removes
+        // work. See `pic_outlined_mru_hit`.
+        if let Some(value) = unsafe { pic_outlined_mru_hit(obj_handle, cache_slot) } {
+            crate::typed_feedback::js_typed_feedback_record_guard_pass(site_id);
+            return value;
+        }
         return js_object_get_field_ic_miss(obj_handle, key, cache_slot);
     }
     // Invalid (non-pointer) receiver. `undefined`/`null` throw a TypeError (#462 —

crates/perry-runtime/src/regex.rs

@@ -6,6 +6,11 @@
 #[cfg(feature = "regex-engine")]
 use regex::Regex;
 use std::cell::RefCell;
+// Every use of `HashMap` in this file is inside a `#[cfg(feature = "regex-engine")]`
+// block, so an unconditional import is an unused-import error under the
+// `warnings` job's `-D warnings` when `perry`'s own binaries pull the runtime
+// in without that feature.
+#[cfg(feature = "regex-engine")]
 use std::collections::HashMap;
 use std::ptr;
 use std::sync::Arc;