diff --git a/Cargo.toml b/Cargo.toml index 7ee6c9a59c..835d2cf2c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,45 +135,45 @@ strip = false # UI crates must NOT strip — they export #[no_mangle] extern "C" symbols [profile.release.package.perry-ui-macos] strip = false -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ui-gtk4] strip = false -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ui-windows] strip = false -codegen-units = 16 +codegen-units = 1 # WinUI scaffold (#4680): re-exports perry-ui-windows; the staticlib bundles # its #[no_mangle] extern "C" symbols, so it must not be stripped either. [profile.release.package.perry-ui-windows-winui] strip = false -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ui-geisterhand] strip = false -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ui-ios] strip = false -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ui-visionos] strip = false -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ui-tvos] strip = false -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ui-android] strip = false -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ui-watchos] strip = false -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-stdlib] opt-level = "s" # Optimize for size in stdlib @@ -190,7 +190,7 @@ opt-level = "s" # Optimize for size in stdlib # units + no strip keeps the exported C API in the staticlib. [profile.release.package.perry-ext-events] strip = false -codegen-units = 16 +codegen-units = 1 # Staticlib wrapper crates (#5422). perry-runtime / perry-stdlib are now # rlib-only; these wrappers re-export their #[no_mangle] C API into @@ -200,10 +200,10 @@ codegen-units = 16 # in the archive. [profile.release.package.perry-runtime-static] strip = false -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-stdlib-static] strip = false -codegen-units = 16 +codegen-units = 1 # Issue #5928: well-known "shared tokio" wrapper crates (#507) are built in # the SAME cargo invocation as perry-stdlib-static so cargo unifies their @@ -223,17 +223,17 @@ codegen-units = 16 # strip-dedup mechanism assumes same-named codegen units are byte-identical # and safe to drop duplicates from — an assumption this mismatch violated. [profile.release.package.perry-ext-fastify] -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ext-http] -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ext-ioredis] -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ext-net] -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ext-undici] -codegen-units = 16 +codegen-units = 1 [profile.release.package.perry-ext-ws] -codegen-units = 16 +codegen-units = 1 # Fast developer profile (#5422). Optimized enough for realistic local runs but # without the distribution-grade settings that dominate compile time, so the @@ -243,7 +243,7 @@ codegen-units = 16 [profile.perry-dev] inherits = "release" lto = false -codegen-units = 16 +codegen-units = 1 incremental = true strip = false opt-level = 1 @@ -278,61 +278,61 @@ opt-level = 3 strip = false [profile.dist.package.perry-ui-macos] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ui-gtk4] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ui-windows] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ui-windows-winui] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ui-geisterhand] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ui-ios] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ui-visionos] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ui-tvos] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ui-android] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ui-watchos] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-stdlib] opt-level = "s" [profile.dist.package.perry-ext-events] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-runtime-static] strip = false -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-stdlib-static] strip = false -codegen-units = 16 +codegen-units = 1 # Issue #5928: mirrors the [profile.release.package.perry-ext-*] block above # — see its comment for why matching `codegen-units` across every crate in # the "shared tokio" (#507) cargo invocation is required. [profile.dist.package.perry-ext-fastify] -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ext-http] -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ext-ioredis] -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ext-net] -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ext-undici] -codegen-units = 16 +codegen-units = 1 [profile.dist.package.perry-ext-ws] -codegen-units = 16 +codegen-units = 1 [workspace.package] version = "0.5.1520" @@ -532,3 +532,18 @@ perry-codegen-wasm = { path = "crates/perry-codegen-wasm" } perry-ui-testkit = { path = "crates/perry-ui-testkit" } perry-audio-miniaudio = { path = "crates/perry-audio-miniaudio" } perry-updater = { path = "crates/perry-updater" } + +# Release codegen with debug assertions ON, for the GC root-scanning guards +# that only exist under `cfg(debug_assertions)` — above all +# `gc::young_log::debug_assert_logged`, rule 2 of the young-entry-log design, +# which re-derives each table's minor-relevant set and panics on any key the +# log does not name. `[profile.release]` leaves debug-assertions off, so a +# release `cargo test` run does not enforce rule 1 at all, and the plain `dev` +# profile is too slow to run the GC suite comfortably. +# +# cargo test --profile gcaudit -p perry-runtime -- --test-threads=1 +[profile.gcaudit] +inherits = "release" +debug-assertions = true +lto = false +codegen-units = 1 diff --git a/benchmarks/gc_ratchet/README.md b/benchmarks/gc_ratchet/README.md index fa0a231aa1..909fb2130d 100644 --- a/benchmarks/gc_ratchet/README.md +++ b/benchmarks/gc_ratchet/README.md @@ -71,13 +71,14 @@ sessions x 7 repeats (21 runs per probe), plus 5 traced runs per probe: The GC accounting family is parsed from `PERRY_GC_DIAG=1` output in a separate, untimed pass; enabling the trace was verified not to change `heap_used_bytes`, so the traced pass observes the same collector the untimed pass measures. The -harness takes two traced runs on every invocation and fails if they disagree — -that is the harness proving, each time it runs, that the counters it is about to -gate on really are deterministic. +harness records two traced runs on every invocation. `check` fails if a gated +counter disagrees, even within its tolerance band. Documented probe overrides +apply here too; the measurement keeps both samples and runs every later probe. -Retention and GC accounting are semantic: they are a function of the allocation -sequence and collector policy, not of CPU speed, core count, or machine load. -That is why they can be gated on a shared CI runner and memory and time cannot. +Retention and GC accounting usually transfer across machine classes because +they describe allocation and collector work rather than CPU speed. They must +still demonstrate repeatability: block placement can change the collection +point and therefore the live cohort, as the #9790 investigation below shows. ## A probe may declare the collector it is a probe *of* (the large-Eden arm) @@ -332,10 +333,63 @@ carrying a non-deterministic gating cell cannot be *pinned*. Before #7554 the rule existed only in `tests/test_gc_ratchet.py`, which is why a bad pin could be committed and only wedge CI afterwards. -The section is currently **empty**, which is the goal state and not an -oversight. Its one entry — `12_large_live_set.heap_used_bytes` — was deleted by -#7558, which removed the *cause* rather than the cell. That is rule 4 working -as designed. +The former `12_large_live_set.heap_used_bytes` entry was deleted by #7558, +which removed its cause. The two current entries exclude only +`07_array_grow_evacuate.copied_bytes` and `.freed_bytes`, with the following +evidence. Removing the placement dependency would allow deleting these entries. + +### Array growth: placement changes the collection point (#9790) + +Twenty-one executions of one unchanged binary produced two counter tuples. +Every stdout matched Node 26.5.1. Retained heap, arena capacity, minor/step +counts, copied/promoted object counts, and promoted bytes were identical. +The [receipt](evidence/9790-array-growth-pacing.json) records raw byte-counter +samples, stable metrics, and compiler/runtime/probe hashes. + +Temporary logging in `move_young` cross-checked the counters against the actual +headers moved. Only the fifth minor's live cohort differed: an 8,208-byte array +with length 640, capacity 1,024, and seed 5,207 was reached through the ring's +remembered edge in one run. The other run instead copied a 144-byte array with +length 1, capacity 16, and seed 5,240 from the native stack. The difference is +exactly **8,064 bytes**, while both runs copy one object. Summing the other +copied headers gives the same result in both runs. + +The freed-byte difference also balances against the actual from-space usage: + +| Fifth minor | Higher copied bytes | Lower copied bytes | +|---|---:|---:| +| Eden bytes | 16,775,936 | 16,776,080 | +| Active survivor bytes | 517,248 | 517,248 | +| Copied bytes | 525,312 | 517,248 | +| Promoted bytes | 131,328 | 131,328 | +| Freed bytes | 16,636,544 | 16,644,752 | + +In each column, freed = Eden + active survivor - copied - promoted; malloc +reclamation is zero. There is no unexplained accounting remainder. + +Block-boundary logging located the pacing cause. `arena_cell_alloc` checks GC +pressure when its current block cannot satisfy an allocation. Promotion walks +address-keyed root tables, so equal total promoted bytes can fill individual +old blocks differently. In the higher-copy run, an old block overflowed on a +4,112-byte growth request after young occupancy had crossed the cap. In the +lower-copy run, that old-block rollover occurred earlier, below the cap; the +next nursery block overflow armed collection while starting the next array. +`js_array_grow` can fall back to old allocation when a growth cannot fit the +current nursery block, connecting this workload to that old-block geometry. +The earlier dirty-page statistics also vary with placement; they alone would +not have established the cause. + +The two byte counters therefore describe real, placement-dependent work on +this workload. They remain measured and displayed, with their existing bands; +only their ability to fail the gate is excluded. The probe's correctness, +retention, cycle counts, copied/promoted object counts, and promoted bytes +remain gated, as do these byte counters on every other probe. Runtime pacing +and the probe's allocation sequence are unchanged. + +The determinism check now runs in `check`, where the baseline's reviewed +overrides are available. An unlisted disagreement still fails even if its +median equals the baseline, and the full measurement artifact survives for +inspection. `assemble` still refuses to pin any nondeterministic gated cell. ### What that probe's non-determinism was, and where it went (#7558) diff --git a/benchmarks/gc_ratchet/baseline/gc-ratchet-v1.json b/benchmarks/gc_ratchet/baseline/gc-ratchet-v1.json index eb8998a45a..3d6db44e37 100644 --- a/benchmarks/gc_ratchet/baseline/gc-ratchet-v1.json +++ b/benchmarks/gc_ratchet/baseline/gc-ratchet-v1.json @@ -103,12 +103,12 @@ "job red. Every entry carries evidence that is checked, not merely stored --", "at least 21 runs (the same number every band above is justified by) and a", "spread that is actually non-zero, so a cell cannot be excluded on a hunch.", - "The section is EMPTY, and that is the goal state. Its one entry --", - "12_large_live_set.heap_used_bytes, added by #7554 -- was deleted by #7558,", - "which removed the cause rather than the cell: explicit gc() no longer forces", - "the conservative native-stack scan, so that reading is bit-identical again", - "and gates again. An empty section is not a disarmed rule; the evidence", - "checks and the never-gate-nothing rule still fail any entry added back.", + "The former 12_large_live_set.heap_used_bytes exclusion was removed by #7558.", + "#9790 excludes only 07_array_grow_evacuate.copied_bytes and freed_bytes:", + "promotion order changes block packing and when nursery pressure is checked.", + "The receipt records 21 runs and an allocation-level accounting cross-check.", + "Both cells remain measured and reported. All other cells retain their bands.", + "Deleting the placement dependency means deleting these exclusions.", "", "#7559 -- A heap_used_bytes band used to NOT be a statement about how much", "the collector retained. The reading is taken after the probe's own gc(),", @@ -295,7 +295,30 @@ "rationale": "GATED HERE ONLY. Worst cross-session spread of medians-of-7 was 0.751% on an idle box (load 1.7-2.0); worst raw within-session spread was 5.3%, which the median-of-7 damps out. 10% is ~13x the cross-session figure and ~2x the worst raw spread, so it will not fire on scheduler jitter but will catch the tens-of-percent slowdown a whole-stack conservative scan would introduce. The 15 ms floor covers the fastest probe (126 ms)." } }, - "probe_overrides": {} + "probe_overrides": { + "07_array_grow_evacuate": { + "copied_bytes": { + "gating": false, + "rationale": "NOT GATED ON THIS PROBE (#9790). Address-dependent promotion order changes old-block packing and the allocation boundary where nursery pressure is checked. The fifth minor observes a different live cohort: a completed 8,208-byte array or a new 144-byte array. Header-size sums and from-space reclamation account for the byte deltas; this is placement-dependent pacing, not an accounting discrepancy. Keep this cell visible; correctness, retention, cycle counts, object counts and promoted bytes remain gated. See evidence/9790-array-growth-pacing.json and the README investigation.", + "evidence": { + "observed_runs": 21, + "observed_spread": 8064, + "measured_on": "2026-09-05, macOS arm64, Perry 0.5.1520; 21 executions of one unchanged binary. Compiler/runtime hashes and raw counters: benchmarks/gc_ratchet/evidence/9790-array-growth-pacing.json.", + "issue": "https://github.com/PerryTS/perry/issues/9790" + } + }, + "freed_bytes": { + "gating": false, + "rationale": "NOT GATED ON THIS PROBE (#9790). Address-dependent promotion order changes old-block packing and the allocation boundary where nursery pressure is checked. The fifth minor observes a different live cohort: a completed 8,208-byte array or a new 144-byte array. Header-size sums and from-space reclamation account for the byte deltas; this is placement-dependent pacing, not an accounting discrepancy. Keep this cell visible; correctness, retention, cycle counts, object counts and promoted bytes remain gated. See evidence/9790-array-growth-pacing.json and the README investigation.", + "evidence": { + "observed_runs": 21, + "observed_spread": 8208, + "measured_on": "2026-09-05, macOS arm64, Perry 0.5.1520; 21 executions of one unchanged binary. Compiler/runtime hashes and raw counters: benchmarks/gc_ratchet/evidence/9790-array-growth-pacing.json.", + "issue": "https://github.com/PerryTS/perry/issues/9790" + } + } + } + } }, "notes": "Re-pinned for #8122-recover: the allocation census before minor #0 object-denominates the first nursery cap (first cycle fires earlier on small-object workloads), one descriptor lookup per traced object, untraced-promotion threshold 990 -> 980. Every GC-accounting fingerprint shifts; retention improves on 12_large_live_set (-75%) and 13_large_eden_survivors moves +85 KB because its cycle 0 now holds up an in-place promotion at 581 permille (main at cap 49 retains 651 KB the same way).", "probes": { diff --git a/benchmarks/gc_ratchet/evidence/9790-array-growth-pacing.json b/benchmarks/gc_ratchet/evidence/9790-array-growth-pacing.json new file mode 100644 index 0000000000..49b48be8d8 --- /dev/null +++ b/benchmarks/gc_ratchet/evidence/9790-array-growth-pacing.json @@ -0,0 +1,177 @@ +{ + "issue": 9790, + "measured_at": "2026-09-05", + "platform": "darwin-arm64", + "runtime_version": "0.5.1520", + "source": "benchmarks/gc_ratchet/probes/07_array_grow_evacuate.ts (unchanged workload)", + "run_env": { + "PERRY_GC_DIAG": "1", + "PERRY_GC_TRACE": "1" + }, + "oracle": { + "version": "v26.5.1", + "matching_runs": 21, + "stdout": "probe:07_array_grow_evacuate\nchecksum:21891160\nsurvivors:94\n" + }, + "binaries": { + "perry": { + "bytes": 152440456, + "sha256": "0e482d4099c396832d57acd1ed9a974107735901cafeeac609ae04ec7a3beb69" + }, + "libperry_runtime.a": { + "bytes": 87787184, + "sha256": "f979e631e6e1d468db1b40bc85882660afe7c79d75c9fc11c18a71fe58c5d591" + }, + "probe": { + "bytes": 16811144, + "sha256": "1f5e3b44052edf6c63fdb1cb51e3f62ec819db50435f264c6ef218da23f230d5" + } + }, + "constant_counters": { + "minor_cycles": 5, + "step_cycles": 5, + "copied_objects": 6745, + "promoted_objects": 6197, + "promoted_bytes": 844600 + }, + "constant_retention": { + "heap_used_bytes": 1214784, + "heap_total_bytes": 25165824 + }, + "samples": [ + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2639944, + "freed_bytes": 83567048 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2639944, + "freed_bytes": 83567048 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2639944, + "freed_bytes": 83567048 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2639944, + "freed_bytes": 83567048 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2639944, + "freed_bytes": 83567048 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2639944, + "freed_bytes": 83567048 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + }, + { + "copied_bytes": 2648008, + "freed_bytes": 83558840 + } + ], + "diagnostic": { + "method": "Temporary move/header and block-boundary logging in runtime source at c7361c87c73738fe97cf352c75c9f00ce0a9b346; manually relinked the same generated object. Both original counter tuples and Node stdout reproduced. Diagnostic edits are not part of the runtime fix.", + "high": { + "copied_bytes": 2648008, + "freed_bytes": 83558840, + "fifth_minor_copied_bytes": 525312, + "fifth_minor_promoted_bytes": 131328, + "fifth_minor_freed_bytes": 16636544, + "fifth_minor_eden_bytes": 16775936, + "fifth_minor_survivor_bytes": 517248, + "extra_array": { + "seed": 5207, + "length": 640, + "capacity": 1024, + "header_size": 8208, + "source": "remembered_set" + }, + "last_block_trigger": { + "generation": "Old", + "request": 4112, + "offset": 1045352, + "block_size": 1048576, + "young_bytes": 17293184 + } + }, + "low": { + "copied_bytes": 2639944, + "freed_bytes": 83567048, + "fifth_minor_copied_bytes": 517248, + "fifth_minor_promoted_bytes": 131328, + "fifth_minor_freed_bytes": 16644752, + "fifth_minor_eden_bytes": 16776080, + "fifth_minor_survivor_bytes": 517248, + "extra_array": { + "seed": 5240, + "length": 1, + "capacity": 16, + "header_size": 144, + "source": "mutable_root_slots/native_stack" + }, + "last_block_trigger": { + "generation": "Nursery", + "request": 144, + "offset": 1048496, + "block_size": 1048576, + "young_bytes": 17293184 + } + }, + "conservation": "fifth_minor_freed_bytes = fifth_minor_eden_bytes + fifth_minor_survivor_bytes - fifth_minor_copied_bytes - fifth_minor_promoted_bytes in both arms; malloc freed bytes are zero. Total copied-byte delta = 8208 - 144 = 8064." + } +} diff --git a/benchmarks/gc_ratchet/gc_ratchet.py b/benchmarks/gc_ratchet/gc_ratchet.py index 87eb80b83c..52cd1a41df 100644 --- a/benchmarks/gc_ratchet/gc_ratchet.py +++ b/benchmarks/gc_ratchet/gc_ratchet.py @@ -721,9 +721,11 @@ def measure( # Separate traced pass. PERRY_GC_DIAG writes one line per collection # phase, which perturbs wall time, so it must not share a pass with - # the timing samples. Two traced runs are taken and required to - # agree: that is the harness proving, every time it runs, that the - # counters it is about to gate on are actually deterministic. + # the timing samples. Keep both traced runs, including disagreement. + # `check` owns the gating policy (and documented probe overrides), + # so it rejects disagreement in every counter that remains gated. + # Aborting here would bypass those overrides and discard all later + # probes before the gate could report their results (#9790). traced = [ parse_gc_diag( run_once( @@ -733,12 +735,6 @@ def measure( ) for _ in range(2) ] - if traced[0] != traced[1]: - differing = sorted(k for k in traced[0] if traced[0][k] != traced[1][k]) - raise RatchetError( - f"{name}: GC counters are not deterministic across traced runs " - f"({', '.join(differing)}); they cannot be gated" - ) # Deliberately NOT rejecting minor_cycles == 0 here. A collector that # has stopped running copying minors at all is the single largest # regression this ratchet exists to catch, and it must surface as a @@ -1862,6 +1858,19 @@ def evaluate( for metric in ALL_METRICS: tolerance = resolve_tolerance(tolerances, overrides, name, metric) + # The two traced samples are an independent premise of the band: + # even a one-byte disagreement inside the allowance invalidates a + # gated counter. Read the samples themselves, not a cached spread. + # Only an explicit probe override (or the profile) can exclude it. + current_samples = cur_entry["metrics"][metric].get("samples", []) + unstable_current = metric in GC_METRICS and tolerance.gating and ( + len(current_samples) < 2 or len(set(current_samples)) != 1 + ) + if unstable_current: + failures.append( + f"{name}: {metric} is not deterministic across traced runs; " + "it cannot be gated" + ) # A cell the pinned artifact cannot support is demoted rather than # trusted: comparing against a number whose own premise failed would # dress a defect up as a verdict. The defect is already in @@ -1882,7 +1891,9 @@ def evaluate( else: breach = False - if breach and quarantined: + if unstable_current: + status = "UNFIT (traced samples disagree)" + elif breach and quarantined: status = "UNFIT (pinned cell unusable)" elif breach: status = "REGRESSION" if tolerance.gating else "drift (informational)" diff --git a/benchmarks/gc_ratchet/probes/07_array_grow_evacuate.ts b/benchmarks/gc_ratchet/probes/07_array_grow_evacuate.ts index cd9a463cba..8fdb35c13f 100644 --- a/benchmarks/gc_ratchet/probes/07_array_grow_evacuate.ts +++ b/benchmarks/gc_ratchet/probes/07_array_grow_evacuate.ts @@ -1,11 +1,13 @@ // GC ratchet probe: array element storage growth and reallocation. // -// A growing array repeatedly abandons its previous element storage, which is a -// separate allocation from the array header. Evacuation has to rewrite the -// header's pointer to the moved storage; per-object pinning would leave the old -// storage in place and fragment the region. The probe grows many arrays past +// A growing array replaces its inline header-plus-elements allocation and +// leaves a forwarding stub at the old address. Evacuation must repair holders +// of the current allocation. The probe grows many arrays past // several reallocation boundaries, keeps a small live sample, cycles the rest // through a ring so they are genuinely heap-allocated, then drops them. +// #9790: copied_bytes/freed_bytes remain informational on this probe because +// block placement changes when nursery pressure is checked. See the ratchet +// README and evidence/9790-array-growth-pacing.json; all other gates remain. declare function gc(): void; diff --git a/benchmarks/gc_ratchet/tolerances.json b/benchmarks/gc_ratchet/tolerances.json index 77d213cadd..4122e3f927 100644 --- a/benchmarks/gc_ratchet/tolerances.json +++ b/benchmarks/gc_ratchet/tolerances.json @@ -27,12 +27,12 @@ "job red. Every entry carries evidence that is checked, not merely stored --", "at least 21 runs (the same number every band above is justified by) and a", "spread that is actually non-zero, so a cell cannot be excluded on a hunch.", - "The section is EMPTY, and that is the goal state. Its one entry --", - "12_large_live_set.heap_used_bytes, added by #7554 -- was deleted by #7558,", - "which removed the cause rather than the cell: explicit gc() no longer forces", - "the conservative native-stack scan, so that reading is bit-identical again", - "and gates again. An empty section is not a disarmed rule; the evidence", - "checks and the never-gate-nothing rule still fail any entry added back.", + "The former 12_large_live_set.heap_used_bytes exclusion was removed by #7558.", + "#9790 excludes only 07_array_grow_evacuate.copied_bytes and freed_bytes:", + "promotion order changes block packing and when nursery pressure is checked.", + "The receipt records 21 runs and an allocation-level accounting cross-check.", + "Both cells remain measured and reported. All other cells retain their bands.", + "Deleting the placement dependency means deleting these exclusions.", "", "#7559 -- A heap_used_bytes band used to NOT be a statement about how much", "the collector retained. The reading is taken after the probe's own gc(),", @@ -222,5 +222,28 @@ } }, - "probe_overrides": {} + "probe_overrides": { + "07_array_grow_evacuate": { + "copied_bytes": { + "gating": false, + "rationale": "NOT GATED ON THIS PROBE (#9790). Address-dependent promotion order changes old-block packing and the allocation boundary where nursery pressure is checked. The fifth minor observes a different live cohort: a completed 8,208-byte array or a new 144-byte array. Header-size sums and from-space reclamation account for the byte deltas; this is placement-dependent pacing, not an accounting discrepancy. Keep this cell visible; correctness, retention, cycle counts, object counts and promoted bytes remain gated. See evidence/9790-array-growth-pacing.json and the README investigation.", + "evidence": { + "observed_runs": 21, + "observed_spread": 8064, + "measured_on": "2026-09-05, macOS arm64, Perry 0.5.1520; 21 executions of one unchanged binary. Compiler/runtime hashes and raw counters: benchmarks/gc_ratchet/evidence/9790-array-growth-pacing.json.", + "issue": "https://github.com/PerryTS/perry/issues/9790" + } + }, + "freed_bytes": { + "gating": false, + "rationale": "NOT GATED ON THIS PROBE (#9790). Address-dependent promotion order changes old-block packing and the allocation boundary where nursery pressure is checked. The fifth minor observes a different live cohort: a completed 8,208-byte array or a new 144-byte array. Header-size sums and from-space reclamation account for the byte deltas; this is placement-dependent pacing, not an accounting discrepancy. Keep this cell visible; correctness, retention, cycle counts, object counts and promoted bytes remain gated. See evidence/9790-array-growth-pacing.json and the README investigation.", + "evidence": { + "observed_runs": 21, + "observed_spread": 8208, + "measured_on": "2026-09-05, macOS arm64, Perry 0.5.1520; 21 executions of one unchanged binary. Compiler/runtime hashes and raw counters: benchmarks/gc_ratchet/evidence/9790-array-growth-pacing.json.", + "issue": "https://github.com/PerryTS/perry/issues/9790" + } + } + } + } } diff --git a/changelog.d/9755-gc-side-table-young-logs.md b/changelog.d/9755-gc-side-table-young-logs.md new file mode 100644 index 0000000000..5fdf78524d --- /dev/null +++ b/changelog.d/9755-gc-side-table-young-logs.md @@ -0,0 +1,38 @@ +### Performance + +- **Minor collections no longer walk every runtime side table.** A copying + minor's three root-scan passes — and a budgeted minor's initial root scan + and final remark — visited every entry of the closure dynamic-prop tables, + the string-keyed descriptor tables, the shape family/slot-index maps, the + transition cache and the shape cache on every collection, to discover that + nothing in them pointed at the nursery. On the compiled claude-code TUI + that was ~35k shape families, ~120k descriptors and ~13k closure owners + per walk, 41 minors per streamed reply, all reporting `slots=0`: 34–56 ms + of scanner time per minor. + + Each of those tables now keeps a **young-entry log** + (`crates/perry-runtime/src/gc/young_log.rs`): the keys of entries that may + hold a pointer a minor can act on (nursery, longlived or malloc-GC — an old + object is neither moved nor swept by any minor, and never becomes young + again). Writers note the key before publishing the entry; a minor-scoped + scanner (`RuntimeRootVisitor::young_scope`) visits only the logged keys, + with the same per-entry body as the full walk, and re-logs an entry iff it + is still relevant afterwards; a full trace walks everything and rebuilds + the log. The copied-minor and fallback-minor dead-owner prunes of the same + tables iterate the log as well. Under `debug_assertions` every minor-scoped + walk first re-derives the relevant set from the authoritative table and + panics on a key the log does not name, so a writer that forgets to note is + a red test rather than a silently collected object. `PERRY_GC_DIAG=1` + prints `[gc-young-log]` rows (logged / visited / kept / table size) per + table and cycle. + +- **The post-minor remembered-set coverage restore is proportional to what + the dirty scan could not cover.** `restore_surviving_dirty_coverage` + (#5029) re-walked every slot of every object on the pre-cycle dirty pages + after each copying minor. The minor's own dirty scan already re-remembers + each slot it visits with the same predicate, so objects it visited + completely (every pointer slot on a dirty page and inside the body) are + now skipped; multi-page arrays and owners of out-of-body buffers are still + walked. Debug builds walk the skipped objects too and panic if the walk + would have added a page. `[gc-restore-coverage]` reports objects + walked/skipped and pages added. diff --git a/changelog.d/9782-specialized-typedarray-lifetime.md b/changelog.d/9782-specialized-typedarray-lifetime.md new file mode 100644 index 0000000000..ca25b4bd9e --- /dev/null +++ b/changelog.d/9782-specialized-typedarray-lifetime.md @@ -0,0 +1,5 @@ +Keep a typed array alive through a specialized function call even when preparing +that call is the caller's last use of the array. The raw-pointer calling +convention still avoids repeated type checks, while native GC roots retain the +owner until the callee returns. This fixes collected typed-array storage and +incorrect checksums in all five full-collection representation stress arms. diff --git a/changelog.d/9783-stdin-fixture-handshake.md b/changelog.d/9783-stdin-fixture-handshake.md new file mode 100644 index 0000000000..e2dafb7879 --- /dev/null +++ b/changelog.d/9783-stdin-fixture-handshake.md @@ -0,0 +1,3 @@ +Make the stdin lifecycle parity fixture wait for each child to finish its toggle +and GC churn before sending the second input chunk. Removing the four fixed +2.5-second waits lets the Node oracle finish within the suite's 10-second budget. diff --git a/changelog.d/9785-9786-array-prototype-chains.md b/changelog.d/9785-9786-array-prototype-chains.md new file mode 100644 index 0000000000..efe6f10461 --- /dev/null +++ b/changelog.d/9785-9786-array-prototype-chains.md @@ -0,0 +1,3 @@ +Array indexed reads and membership checks now follow the full custom prototype chain, stop at explicit null prototypes, and invoke Proxy traps with the original receiver. Strict indexed writes also find inherited accessors and readonly properties beyond an array prototype, while writable own properties on intermediate prototypes continue to shadow ancestors. Fixes #9785 and #9786. + +Regression fixtures cover the reported chain-depth and Proxy cases plus accessor receivers, grown prototypes, undefined shadows, negative Proxy membership checks, nested reads inside traps, and trapless Proxy targets. No version bump. diff --git a/changelog.d/9788-iterator-prototype-symbols.md b/changelog.d/9788-iterator-prototype-symbols.md new file mode 100644 index 0000000000..16d66867d6 --- /dev/null +++ b/changelog.d/9788-iterator-prototype-symbols.md @@ -0,0 +1,4 @@ +Keep class iterator methods symbol-only in prototype own-key enumeration, and +observe replacements of Symbol.iterator during direct calls, spread, Array.from, +and for-of loops. Prototype accessors receive the instance and run once; an +explicit undefined replacement shadows the original class method. diff --git a/changelog.d/9790-gc-ratchet-array-growth.md b/changelog.d/9790-gc-ratchet-array-growth.md new file mode 100644 index 0000000000..f3abc2b10d --- /dev/null +++ b/changelog.d/9790-gc-ratchet-array-growth.md @@ -0,0 +1,4 @@ +GC Ratchet now applies traced-counter determinism checks where its documented +probe overrides are available, preserving the full measurement report. Record +why array-growth block placement makes two byte counters informational while +keeping the probe's correctness, retention, cycle and object counts gated. diff --git a/changelog.d/9791-provider-stream-constructor.md b/changelog.d/9791-provider-stream-constructor.md new file mode 100644 index 0000000000..ac949a649c --- /dev/null +++ b/changelog.d/9791-provider-stream-constructor.md @@ -0,0 +1,5 @@ +Fix the native-root provider gate's missing ReadableStream and Response helpers, +and prevent separately built runtime providers from assigning different TLS +values to the same shared cache slot. Provider copies of a thread-local now +claim one declaration identity and reuse its existing storage, preserving the +class registry and GC state while streamed Responses run under moving GC. diff --git a/changelog.d/9801-regex-program-cache-coherence.md b/changelog.d/9801-regex-program-cache-coherence.md new file mode 100644 index 0000000000..8e8162fea4 --- /dev/null +++ b/changelog.d/9801-regex-program-cache-coherence.md @@ -0,0 +1,33 @@ +### Bug Fixes + +- **A lookbehind or backreference literal could stop matching for good.** + The three compiled-program caches are capped independently and each + `clear()`s wholesale on overflow, and `compile_and_cache_regex_checked` + returns early whenever `REGEX_CACHE` already holds the pattern — so it never + re-runs the fancy-regex or repeat-matcher build. For a pattern only + `fancy-regex` accepts, the `REGEX_CACHE` entry is the never-match + placeholder and the real program is the one in `FANCY_CACHE`; once + `FANCY_CACHE` reached its 512-entry cap and cleared while that placeholder + survived, `get_or_compile_regex` handed back a program matching nothing and + nothing rebuilt the fallback. + + Since `lookup_fancy_regex` treats a built header as authoritative (a null + `fancy_ptr` beside a non-null `regex_ptr` IS the answer) and + `site_cache::install_programs` memoizes that triple against the pattern + text, this is not one bad header: every later construction of the same + literal is born with it, until the site-cache entry is evicted. The same + shape applies to `REPEAT_MATCHER_CACHE`, where the wrong answer is quieter — + the linear engine's capture assignment instead of ECMA-262's RepeatMatcher + semantics. + + `lazy::build_and_install_programs` now repairs a missing program before + publishing the header and before memoizing the triple: if the standard + program is the never-match placeholder and no fancy program came back, it + rebuilds the fancy one; if no repeat matcher came back, it re-derives it + (`repeat_matcher::compile` is a byte scan that returns immediately unless a + capture group sits under a quantifier, so it is free for the patterns that + do not need it). A built header therefore always carries every program its + pattern needs, which is exactly the invariant the header-authoritative + lookups and the construction cache depend on. + + (`a_single_program_cache_clear_cannot_disarm_a_lookbehind_literal`) diff --git a/changelog.d/9802-outlined-ic-monomorphic-hit.md b/changelog.d/9802-outlined-ic-monomorphic-hit.md new file mode 100644 index 0000000000..a5afc8f6b4 --- /dev/null +++ b/changelog.d/9802-outlined-ic-monomorphic-hit.md @@ -0,0 +1,16 @@ +Gave the full-outline generic property get (`js_object_get_field_ic`, #5391 +path 3) the monomorphic inline-cache hit the inline diamond has. The outlined +helper observed typed feedback and then called `js_object_get_field_ic_miss` +unconditionally on every read of every heap receiver, so on a module past the +full-outline threshold — every minified bundle — the per-site cache was written +by every property read and consulted by nobody. Measured on the compiled +claude-code TUI, one 400-character reply: entries to the miss handler +2,725,376 → 649,216 and primes 2,180,102 → 114,732 over the same ~12,330 sites. +Guards mirror the emitted diamond's one for one; anything the hit path declines +still reaches the handler. `PERRY_IC_OUTLINE_FASTPATH=0` restores the previous +behaviour for measurement. + +Added `PERRY_IC_DIAG`'s prime split: every `pic_prime_get` is classified as +re-priming the token the site's MRU entry already held, priming a token that +was already in one of the four ways, or priming a genuinely new shape, with a +`PIC_WAY_STATE` census at prime time — globally and per site. diff --git a/changelog.d/9803-module-presized-array-growth.md b/changelog.d/9803-module-presized-array-growth.md new file mode 100644 index 0000000000..d46375cf34 --- /dev/null +++ b/changelog.d/9803-module-presized-array-growth.md @@ -0,0 +1 @@ +Fix out-of-bounds numeric array accesses when a static `new Array(n)` length exceeds the runtime’s dense allocation limit. Large module-level fills now use the guarded storage-growth path. diff --git a/changelog.d/9814-train126-gate-followups.md b/changelog.d/9814-train126-gate-followups.md new file mode 100644 index 0000000000..b5fe8d038d --- /dev/null +++ b/changelog.d/9814-train126-gate-followups.md @@ -0,0 +1,19 @@ +**Gate follow-ups for train126.** + +- `PASS1_MARKED`'s `non_moving_snapshot` window re-pinned after #9755 + restructured `gc/cycle.rs`. Its hunks are all root-scan machinery + (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration + state), which runs before mark propagation completes; the bracketing is + unchanged — `census_pass1_if_armed` inside `step_mark_propagation`, + `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a + synchronous full mark-sweep still moves nothing between them. +- `TRANSITION_CACHE_YOUNG` and `SHAPE_CACHE_YOUNG` classified. Both are + `YoungLog` remembered sets holding slot indices and shape ids — a `u32` + cannot hold a 48-bit pointer — and the pointer-bearing entries they index are + visited by registered scanners. +- Shape-descriptor callsite baseline updated for #9755's relocation of + `visit_raw_mut_ptr_slot(&mut entry.keys_array)` into the new + `object/side_table_roots.rs`. +- `regex.rs` crossed the 2000-line cap by one line, so the `replaceAll` / + `matchAll` non-global receiver guards moved to `regex/global_guards.rs`, + gated on `regex-engine` like their siblings. diff --git a/crates/perry-codegen/src/collectors/ptr_numarray.rs b/crates/perry-codegen/src/collectors/ptr_numarray.rs index d00e795a5f..fcff52edc5 100644 --- a/crates/perry-codegen/src/collectors/ptr_numarray.rs +++ b/crates/perry-codegen/src/collectors/ptr_numarray.rs @@ -8,7 +8,8 @@ //! as a **numeric-array pointer local** when static analysis proves that for //! the local's entire lifetime: //! -//! 1. every element slot in `[0, length)` holds canonical raw-f64 number bits +//! 1. the initial length fits in the allocated backing store, and every +//! element slot in `[0, length)` holds canonical raw-f64 number bits //! or `TAG_HOLE` (never a NaN-boxed pointer/string/bool/undefined), and //! 2. `length` never shrinks below the allocation length, and //! 3. the binding can never go stale (no growth path exists that fails to @@ -22,7 +23,8 @@ //! ## Why it is sound (provenance + containment + density) //! //! * **Provenance**: the local is initialized by exactly one `Stmt::Let` whose -//! init is `new Array()` (runtime hole-fills every slot, sets +//! init is `new Array()` within the runtime's fresh dense limit +//! (runtime hole-fills every slot, sets //! `GC_ARRAY_RAW_F64_HOLES`, and stamps the pointer-free GC layout — //! `js_array_constructor_single`) or an EMPTY array literal `[]` (length 0; //! nothing to observe until a numeric push). Density: `new Array(n)` ⇒ @@ -382,6 +384,7 @@ struct UseWalk<'a> { impl<'a> UseWalk<'a> { /// Resolve a static non-negative array-allocation length: an integer /// literal or a module-level `const` recorded in `compile_time_constants`. + /// The length must also be backed by dense storage at allocation time. fn static_alloc_length(&self, e: &Expr) -> Option { let value = match e { Expr::Integer(v) => *v as f64, @@ -389,7 +392,11 @@ impl<'a> UseWalk<'a> { Expr::LocalGet(id) => *self.compile_time_constants.get(id)?, _ => return None, }; - if !value.is_finite() || value.fract() != 0.0 || !(0.0..=16_000_000.0).contains(&value) { + // #9784: match MAX_FRESH_DENSE_ARRAY_LENGTH in runtime/array/alloc.rs. + // Above this limit `new Array(n)` has logical length n but only a + // small backing store. A length proof cannot justify an unchecked + // slot access there; the guarded tiers must grow/consult storage. + if !value.is_finite() || value.fract() != 0.0 || !(0.0..=1_000_000.0).contains(&value) { return None; } Some(value as i64) @@ -1023,6 +1030,30 @@ mod tests { assert_eq!(fact.proven_initial_length, 0); } + #[test] + fn fresh_array_proof_requires_allocated_storage_for_the_initial_length() { + for (length, should_promote) in [(1_000_000, true), (1_000_001, false)] { + assert_eq!(is_promoted(&[alloc_let(length)]), should_promote); + + // Module-level constants take a separate provenance input from + // literal lengths; both must respect the runtime allocation cap. + let stmts = [let_with( + ARR, + num_array_ty(), + new_array(vec![Expr::LocalGet(OTHER)]), + )]; + let collected = collect_num_array_locals( + &stmts, + &HashSet::new(), + &HashMap::new(), + &facts_for(vec![]), + &HashMap::from([(OTHER, length as f64)]), + &HashSet::new(), + ); + assert_eq!(collected.contains_key(&ARR), should_promote); + } + } + #[test] fn promotes_contained_uses() { // Element read, numeric-valued element write (including a read of the diff --git a/crates/perry-codegen/src/lower_call/func_ref.rs b/crates/perry-codegen/src/lower_call/func_ref.rs index d69e121e22..2b41338717 100644 --- a/crates/perry-codegen/src/lower_call/func_ref.rs +++ b/crates/perry-codegen/src/lower_call/func_ref.rs @@ -145,6 +145,21 @@ fn try_emit_spec_static_call( raw_args_storage } + // TaPtr entries hoist raw header/data pointers and rely on caller roots. + // The caller's binding can otherwise die at its last use while preparing + // this call: native GC liveness follows SSA uses, not lexical scope. Keep + // the boxed owner live through the call, even when no later JS read exists. + fn keep_ta_owners_alive(ctx: &mut FnCtx<'_>, raw_plan: &[RawArg], lowered: &[String]) { + for entry in raw_plan { + if let RawArg::TaPtr(i) = entry { + let bits = ctx.block().bitcast_double_to_i64(&lowered[*i]); + ctx.block().emit_raw(format!( + "call void asm sideeffect \"\", \"r\"(i64 {bits}) \"gc-leaf-function\"" + )); + } + } + } + let check_descriptors = matches!(plan.dispatch, crate::codegen::SpecDispatch::Static); if !range_checked.is_empty() || (check_descriptors && plan.guards.iter().any(Option::is_some)) { // One diamond for the whole call: every range-checked slot's test is @@ -206,6 +221,7 @@ fn try_emit_spec_static_call( .map(|(ty, v)| (*ty, v.as_str())) .collect(); let fast_value = ctx.block().call(DOUBLE, &spec_name, &call_args); + keep_ta_owners_alive(ctx, &raw_plan, lowered); let after_fast = ctx.block().label.clone(); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); @@ -263,6 +279,7 @@ fn try_emit_spec_static_call( .map(|(ty, v)| (*ty, v.as_str())) .collect(); let result = ctx.block().call(DOUBLE, &spec_name, &call_args); + keep_ta_owners_alive(ctx, &raw_plan, lowered); ctx.record_lowered_value( "Call", None, diff --git a/crates/perry-codegen/src/native_root_coverage/harness_self_tests.rs b/crates/perry-codegen/src/native_root_coverage/harness_self_tests.rs index 81c7f60f54..3f3cac5fe9 100644 --- a/crates/perry-codegen/src/native_root_coverage/harness_self_tests.rs +++ b/crates/perry-codegen/src/native_root_coverage/harness_self_tests.rs @@ -205,3 +205,13 @@ fn no_root_alloca_survives_the_statepoint_rewrite() { ); } } + +#[test] +fn the_statepoint_parser_accepts_quoted_specialized_callees() { + let name = format!("perry_fn_probe${}", "spec_ta4x256"); + let fixture = FIXTURE.replace("@js_array_alloc", &format!("@\"{name}\"")); + let line = fixture.lines().find(|line| line.contains(&name)).unwrap(); + let point = super::parse_statepoint(line); + assert_eq!(point.callee, name); + assert_eq!(point.live, vec!["%a", "%b"]); +} diff --git a/crates/perry-codegen/src/native_root_coverage/mod.rs b/crates/perry-codegen/src/native_root_coverage/mod.rs index e1624f5f4c..d4e6ba1852 100644 --- a/crates/perry-codegen/src/native_root_coverage/mod.rs +++ b/crates/perry-codegen/src/native_root_coverage/mod.rs @@ -98,6 +98,7 @@ use perry_hir::{Expr, Function, Module, ModuleInitKind, Param, Stmt}; mod harness_self_tests; mod mechanics; +mod specialized_calls; /// The two targets native roots ship on, one per object format and /// architecture. Pinned rather than host-derived — see the module docs. @@ -508,8 +509,11 @@ fn callee_after_elementtype(line: &str) -> Option { let group = group_after(line, "elementtype(")?; let after = line[line.find("elementtype(")? + "elementtype(".len() + group.len() + 1..].trim_start(); - let name: String = after - .strip_prefix('@')? + // LLVM quotes names containing the specialized-entry separator `$`. + let after_at = after.strip_prefix('@')?; + let name: String = after_at + .strip_prefix('"') + .unwrap_or(after_at) .chars() .take_while(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '$')) .collect(); diff --git a/crates/perry-codegen/src/native_root_coverage/specialized_calls.rs b/crates/perry-codegen/src/native_root_coverage/specialized_calls.rs new file mode 100644 index 0000000000..0454cef049 --- /dev/null +++ b/crates/perry-codegen/src/native_root_coverage/specialized_calls.rs @@ -0,0 +1,101 @@ +//! #9782: a raw typed-array argument still needs its boxed owner alive in +//! the caller while the specialized callee executes. + +use super::*; + +#[test] +fn a_last_use_typed_array_is_live_across_the_specialized_call() { + let mut module = bare_module("last_use_typed_array.ts"); + module.functions.push(Function { + id: 1, + name: "consume".to_string(), + type_params: Vec::new(), + params: vec![Param { + id: 10, + name: "array".to_string(), + ty: Type::Any, + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + }], + return_type: Type::Number, + body: vec![ + Stmt::Expr(Expr::MapNew), + Stmt::Return(Some(Expr::IndexGet { + object: Box::new(Expr::LocalGet(10)), + index: Box::new(Expr::Integer(0)), + })), + ], + is_async: false, + is_generator: false, + is_strict: true, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }); + module.init = vec![ + let_stmt( + 20, + "owner", + Expr::TypedArrayNew { + kind: perry_hir::TYPED_ARRAY_KIND_INT32, + arg: Some(Box::new(Expr::Integer(256))), + }, + ), + Stmt::Expr(Expr::Call { + callee: Box::new(Expr::FuncRef(1)), + args: vec![Expr::LocalGet(20)], + type_args: Vec::new(), + byte_offset: 0, + }), + ]; + + for target in NATIVE_TARGETS { + let ir = native_ir(&module, target, true); + let prefix = format!("perry_fn_last_use_typed_array_ts__consume${}", "spec_"); + let specialized = ir + .lines() + .filter(|line| line.starts_with("define ")) + .filter_map(|line| line.split_once('@')?.1.split_once('(').map(|p| p.0)) + .find(|name| name.starts_with(&prefix)) + .expect("fixture must select a raw typed-array entry"); + let points = statepoints_of(&ir, target, "main"); + for point in points.at(specialized) { + assert!( + !point.live.is_empty(), + "[{target}] raw argument lost its owner: {point:?}" + ); + } + + // Move the lifetime use (and its reloads) above the raw call. Merely + // deleting the asm leaves dead post-call SSA uses that RS4GC still + // considers live before the later dead-code elimination pass. + let mut lines: Vec<_> = ir.lines().collect(); + let call = lines + .iter() + .position(|line| line.contains(&format!("call double @{specialized}("))) + .expect("fixture must call its specialized entry"); + let end = lines + .iter() + .enumerate() + .skip(call + 1) + .find_map(|(i, line)| { + line.contains("call void asm sideeffect \"\", \"r\"(i64") + .then_some(i) + }) + .expect("post-call lifetime use must exist"); + let raw_call = lines.remove(call); + lines.insert(end, raw_call); + let broken = lines.join("\n"); + let broken_points = statepoints_of(&broken, target, "main"); + for point in broken_points.at(specialized) { + assert!( + point.live.is_empty(), + "[{target}] control still roots the owner: {point:?}" + ); + } + } +} diff --git a/crates/perry-hir/src/lower/stmt_loops.rs b/crates/perry-hir/src/lower/stmt_loops.rs index cb8460f179..8d498c4056 100644 --- a/crates/perry-hir/src/lower/stmt_loops.rs +++ b/crates/perry-hir/src/lower/stmt_loops.rs @@ -866,8 +866,8 @@ pub(super) fn lower_stmt_for_of_inner( // Also detect: for (const x of new Range(...)) where Range // defines `*[Symbol.iterator]()`. We lowered that method as // a synthesized top-level generator function taking `this` - // as its first parameter; the for-of here dispatches by - // calling that function with the lowered receiver. + // as its first parameter; this identifies the iterator-protocol loop. + // The actual iterator method is looked up at runtime so mutations count. let iter_from_class: Option = if let ast::Expr::New(new_expr) = &*for_of_stmt.right { if let ast::Expr::Ident(ident) = new_expr.callee.as_ref() { @@ -902,22 +902,17 @@ pub(super) fn lower_stmt_for_of_inner( { // Lower to iterator protocol: // let __iter = genFunc(...); // generator-fn path - // let __iter = __perry_iter_Range(new Range(...)); // class path + // let __iter = GetIterator(new Range(...)); // class path // let __iter = readable.iterator(); // node:stream path // let __result = __iter.next(); // while (!__result.done) { const x = __result.value; body; __result = __iter.next(); } let for_scope_mark = ctx.push_block_scope(); let iter_expr = lower_expr(ctx, &for_of_stmt.right)?; - // For the class path we wrap the lowered `new Range(..)` - // in a direct FuncRef call to the synthesized iterator - // function (which has `this` as its first parameter). - let iter_expr = if let Some(iter_fn_id) = iter_from_class { - Expr::Call { - callee: Box::new(Expr::FuncRef(iter_fn_id)), - args: vec![iter_expr], - type_args: vec![], - byte_offset: 0, - } + // A fresh instance still observes prototype mutations at loop entry. + let iter_expr = if iter_from_class.is_some() { + // Resolve the current Symbol.iterator property, including + // prototype replacements, once at loop entry (#9788). + Expr::GetIterator(Box::new(iter_expr)) } else if is_filehandle_readlines_for_await || is_fs_dir_for_await { async_iterator_method_call(iter_expr) } else if is_node_readable_for_await { diff --git a/crates/perry-hir/src/lower_decl/body_stmt.rs b/crates/perry-hir/src/lower_decl/body_stmt.rs index a3343d29bf..2441bd7651 100644 --- a/crates/perry-hir/src/lower_decl/body_stmt.rs +++ b/crates/perry-hir/src/lower_decl/body_stmt.rs @@ -1326,13 +1326,10 @@ fn lower_body_stmt_impl(ctx: &mut LoweringContext, stmt: &ast::Stmt) -> Result { - if array_has_own_index(proto_arr, index) { - inherited_owner = proto_arr as usize; - } + default_chain = false; + inherited_owner = array_object_proto_index_owner( + crate::value::js_nanbox_pointer(proto_arr as i64).to_bits(), + &key, + ); } None => {} } diff --git a/crates/perry-runtime/src/array/indexing_proto_chain.rs b/crates/perry-runtime/src/array/indexing_proto_chain.rs index 1f612c8774..d16b310044 100644 --- a/crates/perry-runtime/src/array/indexing_proto_chain.rs +++ b/crates/perry-runtime/src/array/indexing_proto_chain.rs @@ -33,12 +33,19 @@ pub(super) unsafe fn array_oob_prototype_get(receiver: usize, index: u32) -> f64 match array_custom_prototype(arr) { Some(ArrayCustomProto::Null) => return TAG_UNDEFINED_F64, Some(ArrayCustomProto::Other(bits)) => { - return array_object_proto_index_get(arr, bits, index).unwrap_or(TAG_UNDEFINED_F64) + return array_object_proto_index_get( + crate::value::js_nanbox_pointer(receiver as i64), + bits, + index, + ) + .unwrap_or(TAG_UNDEFINED_F64) } Some(ArrayCustomProto::Array(proto_arr)) => { - if index < (*proto_arr).length && array_has_own_index(proto_arr, index) { - return js_array_get_f64(proto_arr, index); - } + return array_spec_get_with_receiver( + proto_arr, + index, + crate::value::js_nanbox_pointer(receiver as i64), + ); } None => {} } @@ -74,20 +81,15 @@ pub(crate) fn array_spec_has_index(arr: *const ArrayHeader, index: u32) -> bool return true; } // An explicit `Object.setPrototypeOf(arr, p)` REPLACES the default - // chain. A real-array `p` keeps the original lane (its own indices - // first, then the implicit `Array.prototype` tail below — test262 - // copyWithin/coerced-values-start-change-*). #9192: any other `p` - // answers the whole question by itself, so the default-chain tail must - // not run after it. + // chain. Every custom prototype answers the whole lookup, including + // an array whose own prototype may be retargeted or null (#9785). match array_custom_prototype(arr) { Some(ArrayCustomProto::Null) => return false, Some(ArrayCustomProto::Other(bits)) => { return array_object_proto_index_has(bits, index) } Some(ArrayCustomProto::Array(proto_arr)) => { - if index < (*proto_arr).length && array_has_own_index(proto_arr, index) { - return true; - } + return array_spec_has_index(proto_arr, index); } None => {} } @@ -120,8 +122,8 @@ pub(crate) enum ArrayCustomProto { /// `Object.setPrototypeOf(arr, null)`: nothing is inherited, and the /// implicit `Array.prototype` → `Object.prototype` chain is gone too. Null, - /// The recorded prototype is itself a real array — the original lane, kept - /// bit-for-bit (test262 copyWithin/coerced-values-start-change-*). + /// The recorded prototype is itself a real array. Its own prototype is + /// authoritative after an own-index miss, just as for any other object. Array(*const ArrayHeader), /// Any other object: resolved through the generic object machinery with the /// array as the receiver, so prototype accessors see the right `this` and @@ -140,11 +142,11 @@ pub(crate) unsafe fn array_custom_prototype(arr: *const ArrayHeader) -> Option Option { - // The caller may still hold a pre-grow forwarding stub; the receiver an - // inherited accessor observes must be the live head. - let arr = clean_arr_ptr(arr); - if arr.is_null() { - return None; - } +unsafe fn array_object_proto_index_get(receiver: f64, proto_bits: u64, index: u32) -> Option { let scope = crate::gc::RuntimeHandleScope::new(); - let receiver = scope.root_nanbox_f64(crate::value::js_nanbox_pointer(arr as i64)); + let receiver = scope.root_nanbox_f64(receiver); let proto = scope.root_heap_word_u64(proto_bits); let key = index.to_string(); let key_hdr = crate::string::js_string_from_bytes(key.as_ptr(), key.len() as u32); @@ -246,60 +238,46 @@ unsafe fn array_object_proto_index_get( .map(|v| f64::from_bits(v.bits())) } -/// #9192: the first object in a NON-array custom `[[Prototype]]` chain that -/// owns `key` with a descriptor — the owner whose accessor / attributes the -/// spec `Set` must observe before creating an own element on the array. A plain -/// writable data property carries no side-table entry and correctly reports no -/// owner: the Set then creates the own element, as the spec requires. +/// Find the first own indexed property in the actual custom prototype chain. +/// Stop at writable data too: it shadows a non-writable ancestor. The runtime's +/// GetPrototypeOf handles real arrays and synthetic Object.create prototypes +/// without interpreting an ArrayHeader as an ObjectHeader (#9785). pub(crate) unsafe fn array_object_proto_index_owner(proto_bits: u64, key: &str) -> usize { - let mut bits = proto_bits; + let scope = crate::gc::RuntimeHandleScope::new(); + let proto = scope.root_heap_word_u64(proto_bits); + let key_ptr = crate::string::js_string_from_bytes(key.as_ptr(), key.len() as u32); + if key_ptr.is_null() { + return 0; + } + let key_handle = scope.root_nanbox_f64(crate::value::nanbox_string_key(key_ptr)); for _ in 0..64 { - if bits == crate::value::TAG_NULL { - return 0; - } - if crate::proxy::js_proxy_is_proxy(f64::from_bits(bits)) != 0 { + let bits = proto.get_heap_word_u64(); + if bits == crate::value::TAG_NULL + || crate::proxy::js_proxy_is_proxy(f64::from_bits(bits)) != 0 + { return 0; } let Some(addr) = pointer_bits_of_recorded_prototype(bits) else { return 0; }; - // Pair the band predicate with the validity check (#6279): a handle - // value sits below HANDLE_BAND_MAX and would otherwise be dereferenced - // as if it were an object pointer. - if !crate::value::addr_class::is_above_handle_band(addr as usize) + if !crate::value::addr_class::is_above_handle_band(addr) || !crate::object::is_valid_obj_ptr(addr as *const u8) { return 0; } - if crate::object::get_accessor_descriptor(addr, key).is_some() - || crate::object::get_property_attrs(addr, key).is_some() - { - return addr; + let addr = crate::value::resolve_forwarding(addr); + let value = crate::value::js_nanbox_pointer(addr as i64); + proto.set_heap_word_u64(value.to_bits()); + if crate::object::obj_value_has_own_key(value, key_handle.get_nanbox_f64()) { + return crate::value::js_nanbox_get_pointer(f64::from_bits(proto.get_heap_word_u64())) + as usize; } - match crate::object::prototype_chain::object_static_prototype(addr) { - Some(next) => bits = next, - // #9220: `Object.create(p)` does NOT record `p` in the observable - // prototype side table — `js_object_create` models the link with a - // SYNTHETIC CLASS ID whose `class_prototype_object` entry is `p` - // (#809). The recorded-prototype hop alone therefore stops one link - // short, and an inherited accessor / non-writable index that the - // READ side already resolves (`js_object_get_field_by_name`'s - // `class_id != 0` branch, reached through - // `resolve_inherited_field_from_prototype`) was silently replaced by - // a new own element on the array. Take the same hop the read walk - // takes so `[[Set]]` and `[[Get]]` agree on the chain. - None => { - let class_id = (*(addr as *const crate::ObjectHeader)).class_id; - if class_id == 0 { - return 0; - } - let synth = crate::object::class_prototype_object(class_id); - if synth.is_null() || synth as usize == addr { - return 0; - } - bits = crate::value::js_nanbox_pointer(synth as i64).to_bits(); - } + let next = + crate::object::js_object_get_prototype_of(f64::from_bits(proto.get_heap_word_u64())); + if next.to_bits() == proto.get_heap_word_u64() { + return 0; } + proto.set_heap_word_u64(next.to_bits()); } 0 } @@ -323,29 +301,32 @@ unsafe fn array_object_proto_index_has(proto_bits: u64, index: u32) -> bool { /// (firing index accessors via `js_array_get_f64`) or, for an absent own index, /// the inherited `Array.prototype[index]`. Returns `undefined` when absent. pub(crate) fn array_spec_get(arr: *const ArrayHeader, index: u32) -> f64 { + let arr = clean_arr_ptr(arr); + array_spec_get_with_receiver(arr, index, crate::value::js_nanbox_pointer(arr as i64)) +} + +fn array_spec_get_with_receiver(arr: *const ArrayHeader, index: u32, receiver: f64) -> f64 { const TAG_UNDEFINED_F64: f64 = f64::from_bits(0x7FFC_0000_0000_0001u64); let arr = clean_arr_ptr(arr); if arr.is_null() { return TAG_UNDEFINED_F64; } unsafe { - let receiver = crate::value::js_nanbox_pointer(arr as i64); let scope = crate::gc::RuntimeHandleScope::new(); let receiver = scope.root_nanbox_f64(receiver); if array_has_own_index(arr, index) { - return js_array_get_f64(arr, index); + return array_inherited_index_get(arr, index, receiver.get_nanbox_f64()); } // #9192: see `array_spec_has_index` — a non-array custom prototype // replaces the default chain outright. match array_custom_prototype(arr) { Some(ArrayCustomProto::Null) => return TAG_UNDEFINED_F64, Some(ArrayCustomProto::Other(bits)) => { - return array_object_proto_index_get(arr, bits, index).unwrap_or(TAG_UNDEFINED_F64) + return array_object_proto_index_get(receiver.get_nanbox_f64(), bits, index) + .unwrap_or(TAG_UNDEFINED_F64) } Some(ArrayCustomProto::Array(proto_arr)) => { - if index < (*proto_arr).length && array_has_own_index(proto_arr, index) { - return array_inherited_index_get(proto_arr, index, receiver.get_nanbox_f64()); - } + return array_spec_get_with_receiver(proto_arr, index, receiver.get_nanbox_f64()); } None => {} } diff --git a/crates/perry-runtime/src/closure/dynamic_props.rs b/crates/perry-runtime/src/closure/dynamic_props.rs index 4e6a96cec9..d3ea542188 100644 --- a/crates/perry-runtime/src/closure/dynamic_props.rs +++ b/crates/perry-runtime/src/closure/dynamic_props.rs @@ -95,6 +95,38 @@ fn get_closure_props() -> &'static Mutex> { CLOSURE_PROPS.get_or_init(|| Mutex::new(new_ptr_hash_map())) } +crate::perry_thread_local! { + /// #9754: this thread's young-entry log for the three closure side tables + /// (`CLOSURE_PROPS`, `CLOSURE_STATIC_PROTOTYPES`, `CLOSURE_DELETED_KEYS`) — + /// the owners whose entry may hold a pointer a minor can act on, as key + /// or as value. Thread-local although the tables are process-global: an + /// entry's addresses belong to the inserting thread's heap, and only that + /// thread's minors can move or free them. See `gc/young_log.rs`. + static CLOSURE_YOUNG_OWNERS: std::cell::RefCell> = + const { std::cell::RefCell::new(crate::gc::young_log::YoungLog::new()) }; +} + +const CLOSURE_YOUNG_LOG_NAME: &str = "closure.dynamic_props"; + +/// Rule 1 of `gc/young_log.rs`: log `owner` BEFORE the entry is published +/// when the owner or the value being stored can matter to a minor. +#[inline] +fn note_young_closure_owner(owner: usize, value_bits: u64) { + if crate::gc::young_log::addr_is_minor_relevant(owner) + || crate::gc::young_log::bits_are_minor_relevant(value_bits) + { + CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().note(owner)); + } +} + +/// A re-keyed entry keeps whatever values it had, so the new owner is logged +/// unconditionally; the next minor-scoped walk drops it if nothing in it is +/// relevant any more. +#[inline] +fn note_young_closure_owner_rekeyed(new_owner: usize) { + CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().note(new_owner)); +} + per_test_global! { /// #3655: keys deleted off a closure via `delete fn.name` etc. /// @@ -123,6 +155,7 @@ pub fn closure_mark_key_deleted(ptr: usize, key: &str) { if ptr == 0 { return; } + note_young_closure_owner(ptr, 0); if let Ok(mut map) = get_closure_deleted_keys().lock() { map.entry(ptr).or_default().insert(key.to_string()); } @@ -177,6 +210,7 @@ pub fn closure_set_static_prototype(closure_ptr: usize, proto_bits: u64) { return; } let mut slot_addr = 0usize; + note_young_closure_owner(closure_ptr, proto_bits); if let Ok(mut map) = get_closure_prototypes().lock() { let slot = map.entry(closure_ptr).or_insert(0); *slot = proto_bits; @@ -301,10 +335,29 @@ pub(crate) fn prune_dead_closure_side_table_owners(is_dead_closure: &dyn Fn(usiz } } +/// [`prune_dead_closure_side_table_owners`] for a MINOR: only a young owner +/// can be dead, and a young owner is always in the young log (it was noted +/// at insert and is re-logged by every minor-scoped walk while it stays +/// young), so the log is the complete candidate set (#9754). +pub(crate) fn prune_dead_closure_side_table_owners_young(is_dead_closure: &dyn Fn(usize) -> bool) { + super::prune_dead_closure_box_capture_owners(is_dead_closure); + let candidates = CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().take_sorted()); + let mut kept = Vec::with_capacity(candidates.len()); + for owner in candidates { + if is_dead_closure(owner) { + clear_closure_side_tables_for_dead_ptr(owner); + } else { + kept.push(owner); + } + } + CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().extend(kept)); +} + pub(crate) fn closure_dynamic_props_owner_moved(old_owner: usize, new_owner: usize) { if old_owner == 0 || new_owner == 0 || old_owner == new_owner { return; } + note_young_closure_owner_rekeyed(new_owner); if let Ok(mut props) = get_closure_props().lock() { if let Some(old_props) = props.remove(&old_owner) { merge_closure_prop_map(&mut props, new_owner, old_props); @@ -396,21 +449,159 @@ pub(crate) fn visit_closure_static_prototype_slot_mut( /// transitive contents were reachable only via the side table (e.g. /// ajv's `validate.errors = [{ msg }]`) had its element objects freed /// behind the still-live array. +/// +/// #9754: a minor-scoped pass (`visitor.young_scope()`) visits only the +/// owners in `CLOSURE_YOUNG_OWNERS`; a full pass walks every owner and +/// rebuilds the log. Both go through [`scan_closure_owner`], so the per-entry +/// work is identical and only the candidate set differs. pub fn scan_closure_dynamic_props_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - let prop_owners = get_closure_props() - .lock() - .ok() - .map(|props| props.keys().copied().collect::>()) - .unwrap_or_default(); - for owner in prop_owners { - let Some(mut closure_props) = get_closure_props() + if visitor.young_scope() { + scan_closure_side_tables_young(visitor); + return; + } + let mut owners: Vec = Vec::new(); + if let Ok(props) = get_closure_props().lock() { + owners.extend(props.keys().copied()); + } + if let Ok(prototypes) = get_closure_prototypes().lock() { + owners.extend(prototypes.keys().copied()); + } + if let Ok(deleted) = get_closure_deleted_keys().lock() { + owners.extend(deleted.keys().copied()); + } + owners.sort_unstable(); + owners.dedup(); + let table_len = owners.len() as u64; + // A full walk is authoritative: rebuild the log from what it finds. + // Notes made by owner-move hooks while the walk runs land in the emptied + // log and are kept — they name entries this walk already visited under + // their old key, so the duplicate is harmless. + let _ = CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().take_sorted()); + let mut kept = CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().take_spare()); + for owner in owners { + let (new_owner, relevant) = scan_closure_owner(visitor, owner); + if relevant { + kept.push(new_owner); + } + } + let kept_len = kept.len() as u64; + CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + CLOSURE_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: false, + logged: table_len, + visited: table_len, + kept: kept_len, + table_len, + }, + ); +} + +/// The minor-scoped walk: only logged owners. Rounds repeat while visits +/// trigger owner-move hooks that log new keys (`note_young_closure_owner_rekeyed`), +/// which is also what closes the pre-#9754 gap where an entry re-keyed +/// mid-walk was skipped by the mark pass and only rewritten later. +fn scan_closure_side_tables_young(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + let table_len = { + let props = get_closure_props().lock().map(|m| m.len()).unwrap_or(0); + let prototypes = get_closure_prototypes() .lock() - .ok() - .and_then(|mut props| props.remove(&owner)) - else { - continue; - }; + .map(|m| m.len()) + .unwrap_or(0); + let deleted = get_closure_deleted_keys() + .lock() + .map(|m| m.len()) + .unwrap_or(0); + (props + prototypes + deleted) as u64 + }; + #[cfg(debug_assertions)] + debug_assert_closure_young_log_complete(); + let mut logged = 0u64; + let mut visited = 0u64; + let mut kept = CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().take_spare()); + loop { + let batch = CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().take_sorted()); + if batch.is_empty() { + break; + } + logged += batch.len() as u64; + for owner in batch { + visited += 1; + let (new_owner, relevant) = scan_closure_owner(visitor, owner); + if relevant { + kept.push(new_owner); + } + } + } + let kept_len = kept.len() as u64; + CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + CLOSURE_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: true, + logged, + visited, + kept: kept_len, + table_len, + }, + ); +} +/// Rule 2 of `gc/young_log.rs`: re-derive the relevant owners from the three +/// tables and require the log to name each one. +#[cfg(debug_assertions)] +fn debug_assert_closure_young_log_complete() { + use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; + let mut relevant = Vec::new(); + if let Ok(props) = get_closure_props().lock() { + for (&owner, entry) in props.iter() { + if addr_is_minor_relevant(owner) + || entry + .values + .values() + .any(|value| bits_are_minor_relevant(value.to_bits())) + { + relevant.push(owner); + } + } + } + if let Ok(prototypes) = get_closure_prototypes().lock() { + for (&owner, &proto_bits) in prototypes.iter() { + if addr_is_minor_relevant(owner) || bits_are_minor_relevant(proto_bits) { + relevant.push(owner); + } + } + } + if let Ok(deleted) = get_closure_deleted_keys().lock() { + for &owner in deleted.keys() { + if addr_is_minor_relevant(owner) { + relevant.push(owner); + } + } + } + CLOSURE_YOUNG_OWNERS.with(|log| { + log.borrow() + .debug_assert_logged(CLOSURE_YOUNG_LOG_NAME, &relevant) + }); +} + +/// Visit one owner's entries in all three tables — the per-entry body both +/// walks share. Returns the owner's post-visit key and whether the entry can +/// still matter to a minor (its key or any value is not old). +fn scan_closure_owner( + visitor: &mut crate::gc::RuntimeRootVisitor<'_>, + owner: usize, +) -> (usize, bool) { + use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; + let mut relevant = false; + let mut current_owner = owner; + + if let Some(mut closure_props) = get_closure_props() + .lock() + .ok() + .and_then(|mut props| props.remove(&owner)) + { // Metadata key rewrite. Only fires in rewrite-phase modes; mark phases // return `false` here without recording the key as a root (so the // side-table entry doesn't itself keep the closure alive). @@ -422,58 +613,52 @@ pub fn scan_closure_dynamic_props_roots_mut(visitor: &mut crate::gc::RuntimeRoot // value was forwarded. for value in closure_props.values.values_mut() { visitor.visit_nanbox_f64_slot(value); + relevant |= bits_are_minor_relevant(value.to_bits()); } if new_owner == owner { new_owner = forwarded_heap_owner(owner).unwrap_or(owner); } - + current_owner = new_owner; if let Ok(mut props) = get_closure_props().lock() { merge_closure_prop_map(&mut props, new_owner, closure_props); } } - let prototype_owners = get_closure_prototypes() + if let Some(mut proto_bits) = get_closure_prototypes() .lock() .ok() - .map(|prototypes| prototypes.keys().copied().collect::>()) - .unwrap_or_default(); - for owner in prototype_owners { - let Some(mut proto_bits) = get_closure_prototypes() - .lock() - .ok() - .and_then(|mut prototypes| prototypes.remove(&owner)) - else { - continue; - }; - + .and_then(|mut prototypes| prototypes.remove(&owner)) + { let mut new_owner = owner; visitor.visit_metadata_usize_slot(&mut new_owner); visitor.visit_nanbox_u64_slot(&mut proto_bits); + relevant |= bits_are_minor_relevant(proto_bits); if new_owner == owner { new_owner = forwarded_heap_owner(owner).unwrap_or(owner); } - + current_owner = new_owner; if let Ok(mut prototypes) = get_closure_prototypes().lock() { prototypes.insert(new_owner, proto_bits); } } + // #3655: re-key the deleted-keys side table when a closure moves. The // entries are pure metadata (string keys, no JS references), so the // metadata-key visitor only records a re-key; nothing to trace. - let mut moved_deleted = Vec::new(); if let Ok(mut deleted) = get_closure_deleted_keys().lock() { - for owner in deleted.keys().copied().collect::>() { + if deleted.contains_key(&owner) { let mut new_owner = owner; - if visitor.visit_metadata_usize_slot(&mut new_owner) { - moved_deleted.push((owner, new_owner)); - } - } - for (old_owner, new_owner) in moved_deleted { - if let Some(keys) = deleted.remove(&old_owner) { - deleted.entry(new_owner).or_default().extend(keys); + if visitor.visit_metadata_usize_slot(&mut new_owner) && new_owner != owner { + if let Some(keys) = deleted.remove(&owner) { + deleted.entry(new_owner).or_default().extend(keys); + } + current_owner = new_owner; } } } + + relevant |= addr_is_minor_relevant(current_owner); + (current_owner, relevant) } /// Check if a raw pointer points to a ClosureHeader by checking CLOSURE_MAGIC at offset 12. @@ -852,6 +1037,7 @@ pub(crate) fn closure_set_via_function_prototype_descriptor( /// Set a dynamic property on a closure. pub fn closure_set_dynamic_prop(ptr: usize, prop: &str, value: f64) { + note_young_closure_owner(ptr, value.to_bits()); if let Ok(mut props) = get_closure_props().lock() { let closure_props = props.entry(ptr).or_default(); closure_props.insert(prop.to_string(), value); @@ -890,6 +1076,7 @@ pub fn closure_delete_own_dynamic_prop(ptr: usize, prop: &str) -> bool { #[cfg(test)] pub(crate) fn test_clear_closure_side_tables() { + CLOSURE_YOUNG_OWNERS.with(|log| log.borrow_mut().clear()); if let Ok(mut props) = get_closure_props().lock() { props.clear(); } diff --git a/crates/perry-runtime/src/closure/mod.rs b/crates/perry-runtime/src/closure/mod.rs index 01f3e6e22d..20666f8b15 100644 --- a/crates/perry-runtime/src/closure/mod.rs +++ b/crates/perry-runtime/src/closure/mod.rs @@ -79,7 +79,8 @@ pub(crate) use dynamic_props::{ clear_closure_side_tables_for_dead_ptr, clone_closure_rebind_this, closure_dynamic_props_owner_moved, closure_dynamic_side_tables_nonempty, closure_set_via_function_prototype_descriptor, prune_dead_closure_side_table_owners, - visit_closure_dynamic_prop_value_slots_mut, visit_closure_static_prototype_slot_mut, + prune_dead_closure_side_table_owners_young, visit_closure_dynamic_prop_value_slots_mut, + visit_closure_static_prototype_slot_mut, }; pub use dynamic_props::{ closure_delete_own_dynamic_prop, closure_dynamic_props_snapshot, closure_get_dynamic_prop, diff --git a/crates/perry-runtime/src/gc/barrier/mod.rs b/crates/perry-runtime/src/gc/barrier/mod.rs index 35c37c0454..808a66427c 100644 --- a/crates/perry-runtime/src/gc/barrier/mod.rs +++ b/crates/perry-runtime/src/gc/barrier/mod.rs @@ -523,7 +523,7 @@ pub(super) unsafe fn scan_dirty_header_once( stats.old_objects_considered += 1; stats.valid_roots += 1; stats.dirty_objects_scanned += 1; - scan_dirty_object_slots(header, dirty_pages, stats, visit_slot); + let _ = scan_dirty_object_slots(header, dirty_pages, stats, visit_slot); } #[inline] @@ -564,18 +564,40 @@ pub(super) unsafe fn scan_dirty_slot_with_layout( visit_slot(slot, stats); } +/// Scan the slots of `header` that lie on `dirty_pages`. +/// +/// Returns whether the scan was COMPLETE for the object (#9754): every pointer +/// slot it owns lies on a dirty page AND inside its own allocation. For such +/// an object the per-slot re-remembering the copying minor does in +/// `visit_slot_with_parent` is exactly what the post-cycle +/// `restore_surviving_dirty_coverage` re-derives — the same child predicate on +/// the same post-visit value, and the same `external` verdict (an in-body slot +/// of an old parent is not external under either the page rule used here or +/// the containment rule used there) — so the restore may skip it. An object +/// with a slot on a clean page (a multi-page array) or outside its body (a +/// lazy array's sparse cache, #7538) is reported incomplete and the restore +/// walks it as before. pub(super) unsafe fn scan_dirty_object_slots( header: *mut GcHeader, dirty_pages: &crate::fast_hash::PtrHashSet, stats: &mut RememberedSetTraceStats, visit_slot: &mut dyn FnMut(*mut u64, &mut RememberedSetTraceStats), -) { +) -> bool { + let body_start = header as usize; + let body_end = body_start.saturating_add((*header).size as usize); + let in_body = |slot: *mut u64| { + let addr = slot as usize; + addr >= body_start && addr < body_end + }; + let mut complete = true; visit_gc_rewrite_slot_descriptors(header, |descriptor| unsafe { match descriptor { GcMutableSlotDescriptor::Slot(slot) => { if crate::weakref::is_weak_target_trace_slot(header, slot.slot) { return; } + complete &= in_body(slot.slot) + && dirty_pages_contains_addr(dirty_pages, slot.slot as usize); if let Some(layout_kind) = slot.layout_kind { scan_dirty_slot_with_layout( slot.slot, @@ -607,7 +629,13 @@ pub(super) unsafe fn scan_dirty_object_slots( // reads its target's (cold) GC header — prefetch ahead. let parent_has_weak_slots = crate::weakref::header_may_hold_weak_target_slots(header); + let count = range.slot_count(); + if count != 0 { + complete &= in_body(range.slot(0)) && in_body(range.slot(count - 1)); + } + let mut visited_slots = 0usize; for (start, end) in dirty_slot_ranges_for(range, dirty_pages, stats) { + visited_slots += end - start; stats.dirty_slot_ranges_scanned += 1; let mut acct_page = usize::MAX; let mut acct_slots = 0usize; @@ -642,10 +670,12 @@ pub(super) unsafe fn scan_dirty_object_slots( crate::arena::old_page_account_dirty_slots(acct_page, acct_slots); } } + complete &= visited_slots == count; } GcMutableSlotDescriptor::PointerFreeRange(_) => {} } }); + complete } // --------------------------------------------------------------------------- diff --git a/crates/perry-runtime/src/gc/copying.rs b/crates/perry-runtime/src/gc/copying.rs index 6c53d857e8..be5dfdab47 100644 --- a/crates/perry-runtime/src/gc/copying.rs +++ b/crates/perry-runtime/src/gc/copying.rs @@ -750,6 +750,7 @@ pub(super) fn last_untraced_decline_reason() -> &'static str { pub(super) fn scan_remembered_dirty_slots_copying( snapshot: &RememberedDirtySnapshot, + mut covered: Option<&mut crate::fast_hash::PtrHashSet>, mut visit: impl FnMut(*mut u64, *mut GcHeader, bool, &mut RememberedSetTraceStats), ) -> RememberedSetTraceStats { let mut stats = RememberedSetTraceStats { @@ -793,7 +794,13 @@ pub(super) fn scan_remembered_dirty_slots_copying( visit(slot, header, external, stats); changed |= *slot != before; }; - scan_dirty_object_slots(header, &snapshot.dirty_pages, stats, &mut visit_slot); + let complete = + scan_dirty_object_slots(header, &snapshot.dirty_pages, stats, &mut visit_slot); + if complete { + if let Some(covered) = covered.as_deref_mut() { + covered.insert(header as usize); + } + } if changed { run_gc_rewrite_hook((*header).obj_type, user); } @@ -1058,9 +1065,13 @@ impl CopiedMinorEligibility { let snapshot = remembered_dirty_snapshot(); let mut dirty_checker = CopyingNurseryPreflight::new(ptrs, CopiedMinorFallbackReason::PinnedYoungDirtySlot); - scan_remembered_dirty_slots_copying(&snapshot, |slot, _header, _external, _stats| unsafe { - dirty_checker.check_bits(*slot); - }); + scan_remembered_dirty_slots_copying( + &snapshot, + None, + |slot, _header, _external, _stats| unsafe { + dirty_checker.check_bits(*slot); + }, + ); unsafe { dirty_checker.drain(); } @@ -1343,10 +1354,14 @@ pub(super) fn run_copied_minor_attempt( // arming happens. Skipping it would leave the barrier unarmed for the next // cycle — a missing-edge bug one collection later. let snapshot = remembered_dirty_snapshot(); + // #9754: objects whose every slot the dirty scan visited in-body — the + // post-cycle coverage restore skips them (see `scan_dirty_object_slots`). + let mut dirty_scan_covered = crate::fast_hash::new_ptr_hash_set(); if !untraced { let _phase = super::pin::CopyingWalkPhaseGuard::enter("remembered_set"); let remembered_stats = scan_remembered_dirty_slots_copying( &snapshot, + Some(&mut dirty_scan_covered), |slot, header, external, stats| unsafe { let before = *slot; collector.visit_slot_with_parent(slot, header, external); @@ -1600,7 +1615,7 @@ pub(super) fn run_copied_minor_attempt( remembered_set_clear(); collector.sticky.restore(); if !collector.skip_remembering { - restore_surviving_dirty_coverage(&snapshot); + restore_surviving_dirty_coverage(&snapshot, &dirty_scan_covered, "copying_minor"); } let malloc_freed_bytes = if malloc_sweep_due { let phase_start = trace_phase_start(trace); diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 33e0252f8c..b34e9eff3e 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -297,234 +297,8 @@ enum RootScanSubphase { Done, } -struct MutableRegisteredRootScanState { - scanners: Vec, - scanner_states: Vec>>, - ffi_scanners: Vec, - ffi_named_scanners: Vec<(PerryFfiNamedMutableRootScanner, usize)>, - scanner_cursor: usize, - ffi_cursor: usize, - ffi_named_cursor: usize, - recorded_counts: bool, -} - -impl MutableRegisteredRootScanState { - fn new() -> Self { - let scanners = MUTABLE_ROOT_SCANNERS.with(|s| s.borrow().clone()); - let scanner_states = scanners - .iter() - .map(|entry| entry.budgeted_state_factory.map(|factory| factory())) - .collect(); - Self { - scanners, - scanner_states, - ffi_scanners: FFI_MUTABLE_ROOT_SCANNERS.with(|s| s.borrow().clone()), - ffi_named_scanners: FFI_NAMED_MUTABLE_ROOT_SCANNERS.with(|s| s.borrow().clone()), - scanner_cursor: 0, - ffi_cursor: 0, - ffi_named_cursor: 0, - recorded_counts: false, - } - } - - fn step( - &mut self, - valid_ptrs: &ValidPointerSet, - mut root_sources: Option<&mut RootSourcesTraceStats>, - budget: usize, - allow_synchronous_scanners: bool, - ) -> bool { - if !self.recorded_counts { - if let Some(sources) = &mut root_sources { - sources.runtime_handles.record_registered_scanners( - self.scanners - .iter() - .filter(|entry| entry.source == MutableRootScannerSource::RuntimeHandles) - .count(), - ); - sources.runtime_mutable_scanners.record_registered_scanners( - self.scanners - .iter() - .filter(|entry| { - entry.source == MutableRootScannerSource::RuntimeMutableScanner - }) - .count(), - ); - sources.ffi_mutable_scanners.record_registered_scanners( - self.ffi_scanners.len() + self.ffi_named_scanners.len(), - ); - } - self.recorded_counts = true; - } - - let mut remaining = budget; - let mut visitor = RuntimeRootVisitor::for_mark(valid_ptrs); - while self.scanner_cursor < self.scanners.len() { - if remaining == 0 { - return false; - } - let entry = self.scanners[self.scanner_cursor]; - let stats = match &mut root_sources { - Some(sources) => match entry.source { - MutableRootScannerSource::RuntimeHandles => { - Some(&mut sources.runtime_handles as *mut RootSourceSlotTraceStats) - } - MutableRootScannerSource::RuntimeMutableScanner => { - Some(&mut sources.runtime_mutable_scanners as *mut RootSourceSlotTraceStats) - } - }, - None => None, - }; - let previous = visitor.set_root_source_stats(stats); - let done = if let Some(scanner) = entry.budgeted_scanner { - let state = self.scanner_states[self.scanner_cursor] - .as_deref_mut() - .expect("budgeted scanner state exists"); - let before = remaining; - let done = scanner(&mut visitor, state, &mut remaining); - if done && remaining == before && remaining != usize::MAX { - remaining -= 1; - } - done - } else { - if !allow_synchronous_scanners { - return false; - } - remaining -= 1; - (entry.scanner)(&mut visitor); - true - }; - visitor.set_root_source_stats(previous); - if !done { - return false; - } - self.scanner_cursor += 1; - } - - if !allow_synchronous_scanners - && (self.ffi_cursor < self.ffi_scanners.len() - || self.ffi_named_cursor < self.ffi_named_scanners.len()) - { - return false; - } - - while remaining > 0 && self.ffi_cursor < self.ffi_scanners.len() { - let scanner = self.ffi_scanners[self.ffi_cursor]; - self.ffi_cursor += 1; - remaining -= 1; - let stats = match &mut root_sources { - Some(sources) => { - Some(&mut sources.ffi_mutable_scanners as *mut RootSourceSlotTraceStats) - } - None => None, - }; - let previous = visitor.set_root_source_stats(stats); - let ctx = &mut visitor as *mut RuntimeRootVisitor<'_> as *mut c_void; - scanner(perry_ffi_visit_mutable_root_slot, ctx); - visitor.set_root_source_stats(previous); - } - - while remaining > 0 && self.ffi_named_cursor < self.ffi_named_scanners.len() { - let (scanner, scanner_id) = self.ffi_named_scanners[self.ffi_named_cursor]; - self.ffi_named_cursor += 1; - remaining -= 1; - let stats = match &mut root_sources { - Some(sources) => { - Some(&mut sources.ffi_mutable_scanners as *mut RootSourceSlotTraceStats) - } - None => None, - }; - let previous = visitor.set_root_source_stats(stats); - let ctx = &mut visitor as *mut RuntimeRootVisitor<'_> as *mut c_void; - scanner(scanner_id, perry_ffi_visit_mutable_root_slot, ctx); - visitor.set_root_source_stats(previous); - } - - self.scanner_cursor >= self.scanners.len() - && self.ffi_cursor >= self.ffi_scanners.len() - && self.ffi_named_cursor >= self.ffi_named_scanners.len() - } -} - -struct LegacyRegisteredRootScanState { - scanners: Vec, - ffi_scanners: Vec, - scanner_cursor: usize, - ffi_cursor: usize, - stats: LegacyRootTraceStats, -} - -impl LegacyRegisteredRootScanState { - fn new() -> Self { - let scanners: Vec = ROOT_SCANNERS.with(|s| s.borrow().clone()); - let ffi_scanners: Vec = FFI_ROOT_SCANNERS.with(|s| s.borrow().clone()); - let stats = LegacyRootTraceStats { - registered_rust_scanners: scanners.len(), - registered_ffi_scanners: ffi_scanners.len(), - ..LegacyRootTraceStats::default() - }; - Self { - scanners, - ffi_scanners, - scanner_cursor: 0, - ffi_cursor: 0, - stats, - } - } - - fn step( - &mut self, - valid_ptrs: &ValidPointerSet, - pin_discoveries: bool, - budget: usize, - allow_synchronous_scanners: bool, - ) -> bool { - if !allow_synchronous_scanners - && (self.scanner_cursor < self.scanners.len() - || self.ffi_cursor < self.ffi_scanners.len()) - { - return false; - } - let mut remaining = budget; - while remaining > 0 && self.scanner_cursor < self.scanners.len() { - let scanner = self.scanners[self.scanner_cursor]; - self.scanner_cursor += 1; - remaining -= 1; - scanner(&mut |value: f64| { - record_copy_only_scanner_mark_emission( - value.to_bits(), - valid_ptrs, - &mut self.stats, - ); - if let Some(bytes) = - mark_copy_only_scanner_bits(value.to_bits(), valid_ptrs, pin_discoveries) - { - self.stats.pinned_roots += 1; - self.stats.pinned_bytes += bytes; - } - }); - } - - while remaining > 0 && self.ffi_cursor < self.ffi_scanners.len() { - let scanner = self.ffi_scanners[self.ffi_cursor]; - self.ffi_cursor += 1; - remaining -= 1; - let mut ctx = RegisteredRootMarkContext { - valid_ptrs: valid_ptrs as *const ValidPointerSet, - pin_discoveries, - legacy_stats: &mut self.stats as *mut LegacyRootTraceStats, - }; - let ctx = &mut ctx as *mut RegisteredRootMarkContext as *mut c_void; - scanner(perry_ffi_mark_root, ctx); - } - - self.scanner_cursor >= self.scanners.len() && self.ffi_cursor >= self.ffi_scanners.len() - } - - fn stats(&self) -> LegacyRootTraceStats { - self.stats - } -} +mod registered_root_scan; +use registered_root_scan::{LegacyRegisteredRootScanState, MutableRegisteredRootScanState}; struct RootScanCycleState { subphase: RootScanSubphase, @@ -648,8 +422,15 @@ impl RootScanCycleState { Some(&mut trace.root_sources), budget, allow_synchronous_scanners, + self.minor_only, + ), + None => state.step( + valid_ptrs, + None, + budget, + allow_synchronous_scanners, + self.minor_only, ), - None => state.step(valid_ptrs, None, budget, allow_synchronous_scanners), }; if done { self.subphase = RootScanSubphase::LegacyRegisteredScanners; @@ -1805,7 +1586,15 @@ impl GcCycleState { sticky.restore(); } if let Some(snapshot) = self.pre_clear_dirty_snapshot.take() { - restore_surviving_dirty_coverage(&snapshot); + // No coverage set: a budgeted cycle interleaves + // with the mutator, and a store into an already + // dirty page leaves no trace, so nothing scanned + // earlier can be declared covered here. + restore_surviving_dirty_coverage( + &snapshot, + &crate::fast_hash::new_ptr_hash_set(), + "budgeted_cycle", + ); } let reclaim_state = self.reclaim_state.as_mut().expect("reclaim state exists"); @@ -1914,6 +1703,13 @@ impl GcCycleState { ReclaimSubphase::Publish => { let reclaim_start = trace_phase_start(&self.trace); self.publish_reclaim_outcome(); + // #9754: per-table young-log rows for this cycle's root + // scans (initial + final remark), labelled by cycle kind. + super::young_log::report_and_reset(if self.minor.is_some() { + "budgeted_minor" + } else { + "budgeted_full" + }); trace_phase_record(&mut self.trace, "reclaim", reclaim_start); self.reclaim_state .as_mut() diff --git a/crates/perry-runtime/src/gc/cycle/registered_root_scan.rs b/crates/perry-runtime/src/gc/cycle/registered_root_scan.rs new file mode 100644 index 0000000000..559a8fe66c --- /dev/null +++ b/crates/perry-runtime/src/gc/cycle/registered_root_scan.rs @@ -0,0 +1,244 @@ +//! The two registered-root scan cursors of the budgeted root scan. +//! +//! `RootScanCycleState` drives its `MutableRegisteredScanners` and +//! `LegacyRegisteredScanners` subphases through these; each keeps a snapshot +//! of the registry it walks plus a cursor, so a budgeted cycle can stop +//! between scanners and resume where it left off without re-reading a +//! registry the mutator may have grown in between. +//! +//! They live beside `cycle.rs` rather than in it because that file is at the +//! file-size gate; nothing else about the split is meaningful. + +use super::*; + +pub(super) struct MutableRegisteredRootScanState { + scanners: Vec, + scanner_states: Vec>>, + ffi_scanners: Vec, + ffi_named_scanners: Vec<(PerryFfiNamedMutableRootScanner, usize)>, + scanner_cursor: usize, + ffi_cursor: usize, + ffi_named_cursor: usize, + recorded_counts: bool, +} + +impl MutableRegisteredRootScanState { + pub(super) fn new() -> Self { + let scanners = MUTABLE_ROOT_SCANNERS.with(|s| s.borrow().clone()); + let scanner_states = scanners + .iter() + .map(|entry| entry.budgeted_state_factory.map(|factory| factory())) + .collect(); + Self { + scanners, + scanner_states, + ffi_scanners: FFI_MUTABLE_ROOT_SCANNERS.with(|s| s.borrow().clone()), + ffi_named_scanners: FFI_NAMED_MUTABLE_ROOT_SCANNERS.with(|s| s.borrow().clone()), + scanner_cursor: 0, + ffi_cursor: 0, + ffi_named_cursor: 0, + recorded_counts: false, + } + } + + pub(super) fn step( + &mut self, + valid_ptrs: &ValidPointerSet, + mut root_sources: Option<&mut RootSourcesTraceStats>, + budget: usize, + allow_synchronous_scanners: bool, + minor_only: bool, + ) -> bool { + if !self.recorded_counts { + if let Some(sources) = &mut root_sources { + sources.runtime_handles.record_registered_scanners( + self.scanners + .iter() + .filter(|entry| entry.source == MutableRootScannerSource::RuntimeHandles) + .count(), + ); + sources.runtime_mutable_scanners.record_registered_scanners( + self.scanners + .iter() + .filter(|entry| { + entry.source == MutableRootScannerSource::RuntimeMutableScanner + }) + .count(), + ); + sources.ffi_mutable_scanners.record_registered_scanners( + self.ffi_scanners.len() + self.ffi_named_scanners.len(), + ); + } + self.recorded_counts = true; + } + + let mut remaining = budget; + // #9754: a minor-only trace is a young-scoped visit for the logged + // side tables (`gc/young_log.rs`); a full trace walks everything. + let mut visitor = RuntimeRootVisitor::for_mark_scoped(valid_ptrs, minor_only); + while self.scanner_cursor < self.scanners.len() { + if remaining == 0 { + return false; + } + let entry = self.scanners[self.scanner_cursor]; + let stats = match &mut root_sources { + Some(sources) => match entry.source { + MutableRootScannerSource::RuntimeHandles => { + Some(&mut sources.runtime_handles as *mut RootSourceSlotTraceStats) + } + MutableRootScannerSource::RuntimeMutableScanner => { + Some(&mut sources.runtime_mutable_scanners as *mut RootSourceSlotTraceStats) + } + }, + None => None, + }; + let previous = visitor.set_root_source_stats(stats); + let done = if let Some(scanner) = entry.budgeted_scanner { + let state = self.scanner_states[self.scanner_cursor] + .as_deref_mut() + .expect("budgeted scanner state exists"); + let before = remaining; + let done = scanner(&mut visitor, state, &mut remaining); + if done && remaining == before && remaining != usize::MAX { + remaining -= 1; + } + done + } else { + if !allow_synchronous_scanners { + return false; + } + remaining -= 1; + (entry.scanner)(&mut visitor); + true + }; + visitor.set_root_source_stats(previous); + if !done { + return false; + } + self.scanner_cursor += 1; + } + + if !allow_synchronous_scanners + && (self.ffi_cursor < self.ffi_scanners.len() + || self.ffi_named_cursor < self.ffi_named_scanners.len()) + { + return false; + } + + while remaining > 0 && self.ffi_cursor < self.ffi_scanners.len() { + let scanner = self.ffi_scanners[self.ffi_cursor]; + self.ffi_cursor += 1; + remaining -= 1; + let stats = match &mut root_sources { + Some(sources) => { + Some(&mut sources.ffi_mutable_scanners as *mut RootSourceSlotTraceStats) + } + None => None, + }; + let previous = visitor.set_root_source_stats(stats); + let ctx = &mut visitor as *mut RuntimeRootVisitor<'_> as *mut c_void; + scanner(perry_ffi_visit_mutable_root_slot, ctx); + visitor.set_root_source_stats(previous); + } + + while remaining > 0 && self.ffi_named_cursor < self.ffi_named_scanners.len() { + let (scanner, scanner_id) = self.ffi_named_scanners[self.ffi_named_cursor]; + self.ffi_named_cursor += 1; + remaining -= 1; + let stats = match &mut root_sources { + Some(sources) => { + Some(&mut sources.ffi_mutable_scanners as *mut RootSourceSlotTraceStats) + } + None => None, + }; + let previous = visitor.set_root_source_stats(stats); + let ctx = &mut visitor as *mut RuntimeRootVisitor<'_> as *mut c_void; + scanner(scanner_id, perry_ffi_visit_mutable_root_slot, ctx); + visitor.set_root_source_stats(previous); + } + + self.scanner_cursor >= self.scanners.len() + && self.ffi_cursor >= self.ffi_scanners.len() + && self.ffi_named_cursor >= self.ffi_named_scanners.len() + } +} + +pub(super) struct LegacyRegisteredRootScanState { + scanners: Vec, + ffi_scanners: Vec, + scanner_cursor: usize, + ffi_cursor: usize, + stats: LegacyRootTraceStats, +} + +impl LegacyRegisteredRootScanState { + pub(super) fn new() -> Self { + let scanners: Vec = ROOT_SCANNERS.with(|s| s.borrow().clone()); + let ffi_scanners: Vec = FFI_ROOT_SCANNERS.with(|s| s.borrow().clone()); + let stats = LegacyRootTraceStats { + registered_rust_scanners: scanners.len(), + registered_ffi_scanners: ffi_scanners.len(), + ..LegacyRootTraceStats::default() + }; + Self { + scanners, + ffi_scanners, + scanner_cursor: 0, + ffi_cursor: 0, + stats, + } + } + + pub(super) fn step( + &mut self, + valid_ptrs: &ValidPointerSet, + pin_discoveries: bool, + budget: usize, + allow_synchronous_scanners: bool, + ) -> bool { + if !allow_synchronous_scanners + && (self.scanner_cursor < self.scanners.len() + || self.ffi_cursor < self.ffi_scanners.len()) + { + return false; + } + let mut remaining = budget; + while remaining > 0 && self.scanner_cursor < self.scanners.len() { + let scanner = self.scanners[self.scanner_cursor]; + self.scanner_cursor += 1; + remaining -= 1; + scanner(&mut |value: f64| { + record_copy_only_scanner_mark_emission( + value.to_bits(), + valid_ptrs, + &mut self.stats, + ); + if let Some(bytes) = + mark_copy_only_scanner_bits(value.to_bits(), valid_ptrs, pin_discoveries) + { + self.stats.pinned_roots += 1; + self.stats.pinned_bytes += bytes; + } + }); + } + + while remaining > 0 && self.ffi_cursor < self.ffi_scanners.len() { + let scanner = self.ffi_scanners[self.ffi_cursor]; + self.ffi_cursor += 1; + remaining -= 1; + let mut ctx = RegisteredRootMarkContext { + valid_ptrs: valid_ptrs as *const ValidPointerSet, + pin_discoveries, + legacy_stats: &mut self.stats as *mut LegacyRootTraceStats, + }; + let ctx = &mut ctx as *mut RegisteredRootMarkContext as *mut c_void; + scanner(perry_ffi_mark_root, ctx); + } + + self.scanner_cursor >= self.scanners.len() && self.ffi_cursor >= self.ffi_scanners.len() + } + + pub(super) fn stats(&self) -> LegacyRootTraceStats { + self.stats + } +} diff --git a/crates/perry-runtime/src/gc/dead_owner.rs b/crates/perry-runtime/src/gc/dead_owner.rs index 2c617264af..563e404c93 100644 --- a/crates/perry-runtime/src/gc/dead_owner.rs +++ b/crates/perry-runtime/src/gc/dead_owner.rs @@ -226,10 +226,14 @@ pub(super) fn prune_dead_owner_side_tables_post_trace( crate::object::shapes::rotate_old_carrier_epoch_after_full_trace(); } let probe = PostTraceProbe::new(full_trace); + // #9754: a minor can only find a young owner dead (`owner_is_dead` refuses + // tenured and old-generation owners on a minor), so the young-logged + // tables prune from their logs instead of walking. fan_out( &|addr| probe.owner_is_dead(addr, None), &|addr| probe.owner_is_dead(addr, Some(GC_TYPE_CLOSURE)), &|addr| probe.owner_is_dead(addr, Some(GC_TYPE_STRING)), + /* young_only = */ !full_trace, ); // #6182: drop dead weak-target HOLDERS (WeakRef / FinalizationRegistry / // WeakMap-WeakSet entry — all GC_TYPE_OBJECT) from the registry so the @@ -251,6 +255,7 @@ pub(super) fn prune_dead_owner_side_tables_copied_minor() { &|addr| owner_is_dead_copied_minor_from_space(addr, None), &|addr| owner_is_dead_copied_minor_from_space(addr, Some(GC_TYPE_CLOSURE)), &|addr| owner_is_dead_copied_minor_from_space(addr, Some(GC_TYPE_STRING)), + /* young_only = */ true, ); } @@ -279,9 +284,18 @@ pub(super) struct DeadKeyPrune { #[cfg_attr(not(test), allow(dead_code))] pub(super) table: &'static str, pub(super) owner: DeadKeyOwner, - pub(super) prune: fn(&dyn Fn(usize) -> bool), + pub(super) prune: DeadKeyPruneFn, + /// #9754: the same prune restricted to the table's young-entry log + /// (`gc/young_log.rs`). A MINOR can only find a young owner dead, and a + /// young owner is always in the log, so on a minor's fan-out this visits + /// the candidates instead of the whole table. `None` keeps the full walk + /// on every cycle. + pub(super) young_prune: Option, } +/// A prune: drops every entry whose owner the predicate reports dead. +pub(super) type DeadKeyPruneFn = fn(&dyn Fn(usize) -> bool); + /// THE REGISTRY (#8174). /// /// `fan_out` iterates this instead of naming a dozen prunes inline, and @@ -306,11 +320,13 @@ pub(super) const DEAD_KEY_PRUNES: &[DeadKeyPrune] = &[ table: "WASM_MEMORY_BINDINGS", owner: DeadKeyOwner::Any, prune: crate::webassembly::prune_dead_wasm_memory_bindings, + young_prune: None, }, DeadKeyPrune { table: "ARRAY_NAMED_PROPS", owner: DeadKeyOwner::Any, prune: crate::array::prune_dead_array_named_property_owners, + young_prune: None, }, // Re-keyed by the per-object move hook (`transfer_per_object_slot_mask` / // `transfer_per_object_descriptor`), not by a metadata visitor. Dropping @@ -320,17 +336,20 @@ pub(super) const DEAD_KEY_PRUNES: &[DeadKeyPrune] = &[ table: "LAYOUT_SLOT_MASKS + TYPED_LAYOUTS", owner: DeadKeyOwner::Any, prune: crate::gc::layout_tables::prune_dead_per_object_layout_owners, + young_prune: None, }, // Re-keyed by the per-object move hook, not by a metadata visitor. DeadKeyPrune { table: "ELEMENT_SHAPES", owner: DeadKeyOwner::Any, prune: crate::array::prune_dead_element_shape_owners, + young_prune: None, }, DeadKeyPrune { table: "MAP_ITERATOR_ARRAYS", owner: DeadKeyOwner::Any, prune: crate::map::prune_dead_map_iterator_array_owners, + young_prune: None, }, // Re-keyed by `map_header_moved_for_gc`; a dead Map's squeeze history // serves no cursor. @@ -338,32 +357,38 @@ pub(super) const DEAD_KEY_PRUNES: &[DeadKeyPrune] = &[ table: "MAP_COMPACTION_LOG", owner: DeadKeyOwner::Any, prune: crate::map::prune_dead_map_compaction_log_owners, + young_prune: None, }, DeadKeyPrune { table: "SET_ITERATOR_ARRAYS", owner: DeadKeyOwner::Any, prune: crate::set::prune_dead_set_iterator_array_owners, + young_prune: None, }, DeadKeyPrune { table: "SET_COMPACTION_LOG", owner: DeadKeyOwner::Any, prune: crate::set::prune_dead_set_compaction_log_owners, + young_prune: None, }, DeadKeyPrune { table: "state().descriptors.property_descriptors + .accessor_descriptors", owner: DeadKeyOwner::Any, prune: crate::object::prune_dead_descriptor_owner_entries, + young_prune: Some(crate::object::prune_dead_descriptor_owner_entries_young), }, DeadKeyPrune { table: "ARGUMENTS_OBJECTS", owner: DeadKeyOwner::Any, prune: crate::object::prune_dead_arguments_object_entries, + young_prune: None, }, // Re-keyed by the per-object move hook, not by a metadata visitor. DeadKeyPrune { table: "OBJECT_PROTOTYPES", owner: DeadKeyOwner::Any, prune: crate::object::prototype_chain::prune_dead_object_prototype_owners, + young_prune: None, }, // #6759 C1: shape records are keyed on keys_array addresses; drop the // ones whose keys_array died (memory only — per-hit validation covers @@ -372,33 +397,39 @@ pub(super) const DEAD_KEY_PRUNES: &[DeadKeyPrune] = &[ table: "state().shapes.inner (descriptors + indices)", owner: DeadKeyOwner::Any, prune: crate::object::shapes::prune_dead_shape_keys, + young_prune: Some(crate::object::shapes::prune_dead_shape_keys_young), }, // Re-keyed by the per-object move hook, not by a metadata visitor. DeadKeyPrune { table: "state().exotic_expando.entries", owner: DeadKeyOwner::Any, prune: crate::object::exotic_expando::prune_dead_exotic_expando_owners, + young_prune: None, }, DeadKeyPrune { table: "SYMBOL_PROPERTIES + SYMBOL_PROPERTY_ATTRS", owner: DeadKeyOwner::Any, prune: crate::symbol::prune_dead_symbol_property_owners, + young_prune: None, }, DeadKeyPrune { table: "SYMBOL_POINTERS", owner: DeadKeyOwner::Symbol, prune: crate::symbol::prune_dead_symbol_pointers, + young_prune: None, }, DeadKeyPrune { table: "CLOSURE_PROPS + CLOSURE_STATIC_PROTOTYPES + CLOSURE_DELETED_KEYS + CLOSURE_BOX_CELLS", owner: DeadKeyOwner::Closure, prune: crate::closure::prune_dead_closure_side_table_owners, + young_prune: Some(crate::closure::prune_dead_closure_side_table_owners_young), }, DeadKeyPrune { table: "BUILTIN_CLOSURE_LENGTH + BUILTIN_CLOSURE_NON_CONSTRUCTABLE", owner: DeadKeyOwner::Closure, prune: crate::object::prune_dead_builtin_closure_metadata_owners, + young_prune: None, }, // #8040: `FUNCTION_CLASS_IDS` is keyed by a synthetic-class function // value's closure address, and is REKEYED (not re-derived) when that @@ -408,16 +439,19 @@ pub(super) const DEAD_KEY_PRUNES: &[DeadKeyPrune] = &[ table: "FUNCTION_CLASS_IDS", owner: DeadKeyOwner::Closure, prune: crate::object::prune_dead_function_class_id_keys, + young_prune: None, }, DeadKeyPrune { table: "VM_CONTEXTS + VM_SCRIPTS + VM_FUNCTIONS", owner: DeadKeyOwner::Any, prune: crate::node_vm::prune_dead_vm_owner_entries, + young_prune: None, }, DeadKeyPrune { table: "FILEHANDLE_OBJECT_FDS", owner: DeadKeyOwner::Any, prune: crate::fs::prune_dead_filehandle_fd_entries, + young_prune: None, }, // #8190/#8191/#8192/#8194: four more REKEYED tables that the #8174 audit // found had no death story at all. Each is the #8040 shape — see this @@ -427,27 +461,32 @@ pub(super) const DEAD_KEY_PRUNES: &[DeadKeyPrune] = &[ table: "CONSOLE_INSTANCES", owner: DeadKeyOwner::Any, prune: crate::builtins::prune_dead_console_instance_owners, + young_prune: None, }, DeadKeyPrune { table: "BOXED_PRIMITIVE_PAYLOADS", owner: DeadKeyOwner::Any, prune: crate::builtins::prune_dead_boxed_primitive_payload_owners, + young_prune: None, }, DeadKeyPrune { table: "TRANSITION_CACHE_GLOBAL", owner: DeadKeyOwner::Any, prune: crate::object::prune_dead_transition_cache_entries, + young_prune: Some(crate::object::prune_dead_transition_cache_entries_young), }, DeadKeyPrune { table: "REFLECT_METADATA", owner: DeadKeyOwner::Any, prune: crate::proxy::prune_dead_reflect_metadata_targets, + young_prune: None, }, #[cfg(feature = "node-api-host")] DeadKeyPrune { table: "NODE_API_OBJECT_METADATA", owner: DeadKeyOwner::Any, prune: crate::node_api_host::prune_dead_object_meta_owners, + young_prune: None, }, ]; @@ -455,6 +494,7 @@ fn fan_out( is_dead_owner: &dyn Fn(usize) -> bool, is_dead_closure: &dyn Fn(usize) -> bool, is_dead_symbol: &dyn Fn(usize) -> bool, + young_only: bool, ) { // Interned key pointers cached in the store-plan cache may die in this // collection — flush every cached verdict. Pointer identity only: the @@ -468,6 +508,9 @@ fn fan_out( DeadKeyOwner::Closure => is_dead_closure, DeadKeyOwner::Symbol => is_dead_symbol, }; - (entry.prune)(is_dead); + match entry.young_prune { + Some(young_prune) if young_only => young_prune(is_dead), + _ => (entry.prune)(is_dead), + } } } diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 6a7424daca..cb0acd9a6b 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -163,6 +163,10 @@ mod forwarding; /// Per-scanner root attribution for the copied-minor root scan (#7915). mod scanner_profile; mod sticky_remembered; +/// #9754: per-side-table young-entry logs (remembered sets for the runtime +/// side tables), so a minor-scoped root scan visits only the entries that +/// can hold a pointer a minor acts on. +pub(crate) mod young_log; use copying::*; use copying_first_cycle::*; // Named rather than glob-imported: a glob does not propagate through the diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index d243c4e976..054bee16f7 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -708,118 +708,8 @@ pub(crate) fn try_mark_value_or_raw(word: u64, valid_ptrs: &ValidPointerSet) -> true } -/// Specialized mark-and-enqueue for trace-phase field walks. -/// -/// Descriptor-driven trace walks all share the same pattern: read a -/// heap-field word that is either a NaN-boxed JSValue or a raw I64 -/// pointer at an object start, mark it if live, and push the marked -/// header onto the local worklist. The generic -/// `try_mark_value_or_raw` is general enough to also handle -/// conservative stack scans (raw interior pointers via -/// `enclosing_object`) and root scans (push to MARK_SEEDS so the -/// trace-marked-objects entry point can pick them up), but BOTH of -/// those features are pure overhead inside `drain_trace_worklist`: -/// -/// 1. Field words never hold interior pointers — they're written via -/// `arr[i] = x` / `obj.f = x` / closure capture stores, all of -/// which use the object-start user pointer. Skipping -/// `enclosing_object` saves a binary-search lookup per field. -/// -/// 2. The MARK_SEEDS push happens once per newly-marked object during -/// trace, but the same header is also pushed onto the local -/// worklist by the caller (so the trace drain visits it). The -/// extra MARK_SEEDS push goes onto a TLS vec, gets cleared at the -/// start of the next cycle, and is pure waste while we're already -/// in the trace phase. Skipping it saves a TLS slot deref + -/// Vec::push per marked object. -/// -/// 3. The caller-side re-decode of the NaN-tag (to figure out -/// POINTER_MASK extraction vs raw-pointer extraction) is folded -/// into this function, so the caller doesn't pay that switch a -/// second time. -/// -/// The valid-pointer hashset check is still load-bearing here — we -/// only elide the secondary `enclosing_object` fallback. -#[inline(always)] -#[cfg(target_os = "macos")] -pub(super) fn get_stack_bottom() -> usize { - extern "C" { - fn pthread_self() -> *mut std::ffi::c_void; - fn pthread_get_stackaddr_np(thread: *mut std::ffi::c_void) -> *mut std::ffi::c_void; - } - unsafe { - let thread = pthread_self(); - pthread_get_stackaddr_np(thread) as usize - } -} - -#[cfg(target_os = "linux")] -pub(super) fn get_stack_bottom() -> usize { - crate::native_stack::stack_top() -} - -// Windows: read TEB.StackBase. Works on every supported Windows version -// (Windows 7+) without needing GetCurrentThreadStackLimits (Win8+), so it -// stays correct on the `--min-windows-version=7` build path. The TEB lives -// at GS:[0] on x86_64 (FS:[0] on x86); StackBase sits at offset 0x08 -// (the highest address — i.e. where the stack starts and grows down from). -// This is the same pointer kernel32!GetCurrentThreadStackLimits returns as -// `HighLimit`, just read directly from the TEB to avoid the kernel32 dep. -// -// Without this, conservative stack scan early-returns with stack_bottom=0, -// the GC sees no stack roots, and any heap pointer that lives only in a -// stack slot during a callback gets swept (issues #385/#386/#387 — the -// `Array.prototype.map` / `JSON.parse(...).property` / supported_features -// segfaults all traced back to here). -#[cfg(all(target_os = "windows", target_arch = "x86_64"))] -pub(super) fn get_stack_bottom() -> usize { - let stack_base: usize; - unsafe { - std::arch::asm!( - "mov {out}, gs:[0x08]", - out = out(reg) stack_base, - options(nostack, preserves_flags, readonly), - ); - } - stack_base -} - -#[cfg(all(target_os = "windows", target_arch = "x86"))] -pub(super) fn get_stack_bottom() -> usize { - let stack_base: usize; - unsafe { - std::arch::asm!( - "mov {out}, fs:[0x04]", - out = out(reg) stack_base, - options(nostack, preserves_flags, readonly), - ); - } - stack_base -} - -#[cfg(all(target_os = "windows", target_arch = "aarch64"))] -pub(super) fn get_stack_bottom() -> usize { - // ARM64 Windows: TEB pointer is in x18; StackBase at offset 0x08. - let stack_base: usize; - unsafe { - let teb: usize; - std::arch::asm!("mov {}, x18", out(reg) teb, options(nostack, preserves_flags, readonly)); - stack_base = *((teb + 0x08) as *const usize); - } - stack_base -} - -#[cfg(not(any( - target_os = "macos", - target_os = "linux", - all( - target_os = "windows", - any(target_arch = "x86_64", target_arch = "x86", target_arch = "aarch64") - ), -)))] -pub(super) fn get_stack_bottom() -> usize { - 0 // Stack scanning not supported on this OS/arch -} +mod stack_bottom; +pub(super) use stack_bottom::get_stack_bottom; pub(super) enum RuntimeRootVisitMode<'a> { Mark { @@ -856,6 +746,13 @@ pub(super) enum RuntimeRootVisitMode<'a> { pub struct RuntimeRootVisitor<'a> { pub(super) mode: RuntimeRootVisitMode<'a>, pub(super) root_source_stats: Option<*mut RootSourceSlotTraceStats>, + /// #9754: this visit belongs to a MINOR-scoped pass — the copying minor's + /// preflight/mark/rewrite, or a `GcCollectionKind::Minor` trace's root + /// scan — which neither moves nor sweeps an old-generation object. A + /// side table with a young-entry log (`gc/young_log.rs`) may then visit + /// only its logged entries. A full trace, an evacuation rewrite/verify + /// pass and the legacy copy mode walk everything. + pub(super) young_scope: bool, } impl<'a> RuntimeRootVisitor<'a> { @@ -863,6 +760,18 @@ impl<'a> RuntimeRootVisitor<'a> { Self { mode: RuntimeRootVisitMode::Mark { valid_ptrs }, root_source_stats: None, + young_scope: false, + } + } + + /// `for_mark` for a cycle whose trace is minor-only: old objects are + /// black leaves and their marks are consumed by nothing, so the + /// young-logged side tables may skip their old entries (#9754). + pub(super) fn for_mark_scoped(valid_ptrs: &'a ValidPointerSet, minor_only: bool) -> Self { + Self { + mode: RuntimeRootVisitMode::Mark { valid_ptrs }, + root_source_stats: None, + young_scope: minor_only, } } @@ -870,6 +779,7 @@ impl<'a> RuntimeRootVisitor<'a> { Self { mode: RuntimeRootVisitMode::Rewrite { valid_ptrs }, root_source_stats: None, + young_scope: false, } } @@ -877,6 +787,7 @@ impl<'a> RuntimeRootVisitor<'a> { Self { mode: RuntimeRootVisitMode::CopyingCheck { checker }, root_source_stats: None, + young_scope: true, } } @@ -884,6 +795,7 @@ impl<'a> RuntimeRootVisitor<'a> { Self { mode: RuntimeRootVisitMode::CopyingMark { collector }, root_source_stats: None, + young_scope: true, } } @@ -891,6 +803,7 @@ impl<'a> RuntimeRootVisitor<'a> { Self { mode: RuntimeRootVisitMode::CopyingRewrite { collector }, root_source_stats: None, + young_scope: true, } } @@ -901,6 +814,7 @@ impl<'a> RuntimeRootVisitor<'a> { surface, }, root_source_stats: None, + young_scope: false, } } @@ -908,9 +822,18 @@ impl<'a> RuntimeRootVisitor<'a> { Self { mode: RuntimeRootVisitMode::Copy { mark }, root_source_stats: None, + young_scope: false, } } + /// #9754: true when this pass can only act on non-old objects, so a + /// young-logged side table may confine its walk to the logged entries. + /// See `gc/young_log.rs` for the argument. + #[inline] + pub(crate) fn young_scope(&self) -> bool { + self.young_scope + } + #[inline] pub(super) fn set_root_source_stats( &mut self, diff --git a/crates/perry-runtime/src/gc/roots/stack_bottom.rs b/crates/perry-runtime/src/gc/roots/stack_bottom.rs new file mode 100644 index 0000000000..6cf6aa9fbf --- /dev/null +++ b/crates/perry-runtime/src/gc/roots/stack_bottom.rs @@ -0,0 +1,126 @@ +//! `get_stack_bottom` — the per-platform top of the current thread's stack. +//! +//! The conservative stack scan needs the address the stack grows down from, +//! and every OS answers differently (`pthread_get_stackaddr_np` on macOS, +//! `pthread_getattr_np` + `pthread_attr_getstack` on Linux, the TEB on +//! Windows, and a `0` fallback that disables stack scanning elsewhere). The +//! four `#[cfg]` arms plus their `extern "C"` blocks are the only +//! platform-conditional code in the root scanner, so they live here and +//! `roots.rs` stays under the file-size gate. +//! +//! The doc comment on the first arm describes a trace-phase mark helper, not +//! `get_stack_bottom`; it was already attached to this item and is moved +//! verbatim rather than re-pointed at the next unrelated item. + +/// Specialized mark-and-enqueue for trace-phase field walks. +/// +/// Descriptor-driven trace walks all share the same pattern: read a +/// heap-field word that is either a NaN-boxed JSValue or a raw I64 +/// pointer at an object start, mark it if live, and push the marked +/// header onto the local worklist. The generic +/// `try_mark_value_or_raw` is general enough to also handle +/// conservative stack scans (raw interior pointers via +/// `enclosing_object`) and root scans (push to MARK_SEEDS so the +/// trace-marked-objects entry point can pick them up), but BOTH of +/// those features are pure overhead inside `drain_trace_worklist`: +/// +/// 1. Field words never hold interior pointers — they're written via +/// `arr[i] = x` / `obj.f = x` / closure capture stores, all of +/// which use the object-start user pointer. Skipping +/// `enclosing_object` saves a binary-search lookup per field. +/// +/// 2. The MARK_SEEDS push happens once per newly-marked object during +/// trace, but the same header is also pushed onto the local +/// worklist by the caller (so the trace drain visits it). The +/// extra MARK_SEEDS push goes onto a TLS vec, gets cleared at the +/// start of the next cycle, and is pure waste while we're already +/// in the trace phase. Skipping it saves a TLS slot deref + +/// Vec::push per marked object. +/// +/// 3. The caller-side re-decode of the NaN-tag (to figure out +/// POINTER_MASK extraction vs raw-pointer extraction) is folded +/// into this function, so the caller doesn't pay that switch a +/// second time. +/// +/// The valid-pointer hashset check is still load-bearing here — we +/// only elide the secondary `enclosing_object` fallback. +#[inline(always)] +#[cfg(target_os = "macos")] +pub(crate) fn get_stack_bottom() -> usize { + extern "C" { + fn pthread_self() -> *mut std::ffi::c_void; + fn pthread_get_stackaddr_np(thread: *mut std::ffi::c_void) -> *mut std::ffi::c_void; + } + unsafe { + let thread = pthread_self(); + pthread_get_stackaddr_np(thread) as usize + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn get_stack_bottom() -> usize { + crate::native_stack::stack_top() +} + +// Windows: read TEB.StackBase. Works on every supported Windows version +// (Windows 7+) without needing GetCurrentThreadStackLimits (Win8+), so it +// stays correct on the `--min-windows-version=7` build path. The TEB lives +// at GS:[0] on x86_64 (FS:[0] on x86); StackBase sits at offset 0x08 +// (the highest address — i.e. where the stack starts and grows down from). +// This is the same pointer kernel32!GetCurrentThreadStackLimits returns as +// `HighLimit`, just read directly from the TEB to avoid the kernel32 dep. +// +// Without this, conservative stack scan early-returns with stack_bottom=0, +// the GC sees no stack roots, and any heap pointer that lives only in a +// stack slot during a callback gets swept (issues #385/#386/#387 — the +// `Array.prototype.map` / `JSON.parse(...).property` / supported_features +// segfaults all traced back to here). +#[cfg(all(target_os = "windows", target_arch = "x86_64"))] +pub(crate) fn get_stack_bottom() -> usize { + let stack_base: usize; + unsafe { + std::arch::asm!( + "mov {out}, gs:[0x08]", + out = out(reg) stack_base, + options(nostack, preserves_flags, readonly), + ); + } + stack_base +} + +#[cfg(all(target_os = "windows", target_arch = "x86"))] +pub(crate) fn get_stack_bottom() -> usize { + let stack_base: usize; + unsafe { + std::arch::asm!( + "mov {out}, fs:[0x04]", + out = out(reg) stack_base, + options(nostack, preserves_flags, readonly), + ); + } + stack_base +} + +#[cfg(all(target_os = "windows", target_arch = "aarch64"))] +pub(crate) fn get_stack_bottom() -> usize { + // ARM64 Windows: TEB pointer is in x18; StackBase at offset 0x08. + let stack_base: usize; + unsafe { + let teb: usize; + std::arch::asm!("mov {}, x18", out(reg) teb, options(nostack, preserves_flags, readonly)); + stack_base = *((teb + 0x08) as *const usize); + } + stack_base +} + +#[cfg(not(any( + target_os = "macos", + target_os = "linux", + all( + target_os = "windows", + any(target_arch = "x86_64", target_arch = "x86", target_arch = "aarch64") + ), +)))] +pub(crate) fn get_stack_bottom() -> usize { + 0 // Stack scanning not supported on this OS/arch +} diff --git a/crates/perry-runtime/src/gc/scanner_profile.rs b/crates/perry-runtime/src/gc/scanner_profile.rs index 954767b3f9..2e95310269 100644 --- a/crates/perry-runtime/src/gc/scanner_profile.rs +++ b/crates/perry-runtime/src/gc/scanner_profile.rs @@ -132,6 +132,7 @@ pub(super) fn report_and_reset(cycle_label: &str) { if !scanner_profile_enabled() { return; } + super::young_log::report_and_reset(cycle_label); let mut rows = SCANNER_PROFILE.with(|rows| std::mem::take(&mut *rows.borrow_mut())); if rows.is_empty() { return; diff --git a/crates/perry-runtime/src/gc/sticky_remembered.rs b/crates/perry-runtime/src/gc/sticky_remembered.rs index 8e9b2f3acc..3402f6cbd7 100644 --- a/crates/perry-runtime/src/gc/sticky_remembered.rs +++ b/crates/perry-runtime/src/gc/sticky_remembered.rs @@ -54,6 +54,51 @@ impl StickyRememberedSet { } } + /// [`Self::restore`], reporting how many entries were NOT already in the + /// remembered set — the pages this restore genuinely added (#9754). + pub(super) fn restore_counted(&self) -> usize { + let mut added = 0usize; + for &page in &self.old_pages { + // `mark_dirty_old_page`'s return is not "inserted" (its uncached + // arm answers the ever-dirty question), so ask the set first. + let already = super::barrier::DIRTY_OLD_PAGES.with(|s| s.borrow().contains(&page)); + mark_dirty_old_page(page); + if !already { + added += 1; + } + } + for &(header, page) in &self.external_pages { + if mark_dirty_external_slot_page(header, page) { + added += 1; + } + } + added + } + + /// How many of this set's entries the remembered set does NOT hold yet — + /// what `restore` would add. Read-only: the debug check of the coverage + /// restore asks this about objects it skipped. + #[cfg(debug_assertions)] + pub(super) fn count_not_yet_dirty(&self) -> usize { + let old_missing = super::barrier::DIRTY_OLD_PAGES.with(|s| { + let s = s.borrow(); + self.old_pages + .iter() + .filter(|page| !s.contains(page)) + .count() + }); + let external_missing = super::barrier::EXTERNAL_DIRTY_SLOT_PAGES.with(|s| { + let s = s.borrow(); + self.external_pages + .iter() + .filter(|(header, page)| { + !s.get(page).is_some_and(|headers| headers.contains(header)) + }) + .count() + }); + old_missing + external_missing + } + pub(super) fn extend(&mut self, other: StickyRememberedSet) { self.old_pages.extend(other.old_pages); self.external_pages.extend(other.external_pages); diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index ad0ae26bcc..ac6d08f01e 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -64,3 +64,4 @@ mod triggers; mod typed_layout_intact_residual; mod u8_inline_cache; mod weak_read_barrier; +mod young_log_tests; diff --git a/crates/perry-runtime/src/gc/tests/young_log_tests.rs b/crates/perry-runtime/src/gc/tests/young_log_tests.rs new file mode 100644 index 0000000000..db66fadbf6 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/young_log_tests.rs @@ -0,0 +1,594 @@ +//! #9754 — the side-table young-entry logs (`gc/young_log.rs`). +//! +//! Each table gets the same three-part proof: +//! +//! * a YOUNG entry reachable only through the table is moved by a copying +//! minor and the entry re-keyed — through the minor-scoped walk, which the +//! recorded walk row proves was PARTIAL (visited only the logged keys); +//! * an OLD entry is not visited at all — the row proves the skip fired +//! (`visited == 0` while the table is non-empty), which is rule 3 of the +//! design note: a latch that never skips looks landed while doing nothing; +//! * a DEAD young owner is pruned by the young-only prune. +//! +//! Sabotage contract (rule 2): delete any one `note` call in the tables' +//! writers and the matching "moves" test here goes red — under +//! `debug_assertions` on the log-completeness assertion the walk runs first, +//! and in release on the stale address the un-visited entry keeps. + +use super::super::*; +use super::support::*; + +fn young_closure() -> usize { + let ptr = crate::arena::arena_alloc_gc( + std::mem::size_of::(), + std::mem::align_of::(), + GC_TYPE_CLOSURE, + ); + unsafe { init_test_closure(ptr) }; + ptr as usize +} + +fn old_closure() -> usize { + let ptr = crate::arena::arena_alloc_gc_old( + std::mem::size_of::(), + std::mem::align_of::(), + GC_TYPE_CLOSURE, + ); + unsafe { init_test_closure(ptr) }; + ptr as usize +} + +unsafe fn young_keys_array() -> *mut crate::array::ArrayHeader { + let arr = crate::arena::arena_alloc_gc( + std::mem::size_of::(), + std::mem::align_of::(), + GC_TYPE_ARRAY, + ) as *mut crate::array::ArrayHeader; + (*arr).length = 0; + (*arr).capacity = 0; + arr +} + +fn walk(table: &'static str) -> young_log::YoungLogWalk { + young_log::last_walk(table).unwrap_or_else(|| panic!("no walk recorded for {table}")) +} + +// ---------------------------------------------------------------- closures + +#[test] +fn young_closure_prop_value_is_moved_through_the_log() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::closure::scan_closure_dynamic_props_roots_mut); + + let owner = young_closure(); + js_shadow_slot_set(0, ptr_bits(owner)); + // The value is reachable ONLY through the side table. + let value = young_leaf(); + crate::closure::closure_set_dynamic_prop(owner, "memo", f64::from_bits(string_bits(value))); + + let _ = gc_collect_minor(); + + let owner_after = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!( + owner_after, owner, + "the rooted owner must have been evacuated" + ); + let bits = crate::closure::closure_get_own_dynamic_prop(owner_after, "memo") + .expect("entry must follow its owner to the new address") + .to_bits(); + let value_after = (bits & POINTER_MASK) as usize; + assert_eq!(bits & TAG_MASK, STRING_TAG); + assert_ne!( + value_after, value, + "the value must have been evacuated, not left in from-space" + ); + assert!(crate::arena::pointer_in_nursery(value_after)); + assert!( + crate::closure::closure_get_own_dynamic_prop(owner, "memo").is_none(), + "the stale owner key must be gone" + ); + let row = walk("closure.dynamic_props"); + assert!( + row.partial, + "a copying minor must take the young-scoped walk" + ); + assert!( + row.visited >= 1, + "the logged owner must have been visited: {row:?}" + ); + assert!( + row.kept >= 1, + "a survivor still young must stay logged: {row:?}" + ); +} + +#[test] +fn young_value_under_an_old_closure_owner_is_logged_by_the_value() { + let _guard = CopyingNurseryTestGuard::new(0); + gc_register_mutable_root_scanner(crate::closure::scan_closure_dynamic_props_roots_mut); + + let owner = old_closure(); + let value = young_leaf(); + crate::closure::closure_set_dynamic_prop(owner, "memo", f64::from_bits(string_bits(value))); + let proto = young_leaf(); + crate::closure::closure_set_static_prototype(owner, string_bits(proto)); + + let _ = gc_collect_minor(); + + let bits = crate::closure::closure_get_own_dynamic_prop(owner, "memo") + .expect("old owner keeps its entry") + .to_bits(); + let value_after = (bits & POINTER_MASK) as usize; + assert_ne!(value_after, value); + assert!(crate::arena::pointer_in_nursery(value_after)); + let proto_after = (crate::closure::closure_static_prototype(owner).expect("prototype kept") + & POINTER_MASK) as usize; + assert_ne!(proto_after, proto); + assert!(crate::arena::pointer_in_nursery(proto_after)); + assert!(walk("closure.dynamic_props").partial); +} + +#[test] +fn old_closure_entries_are_skipped_by_a_minor() { + let _guard = CopyingNurseryTestGuard::new(0); + gc_register_mutable_root_scanner(crate::closure::scan_closure_dynamic_props_roots_mut); + + let owner = old_closure(); + crate::closure::closure_set_dynamic_prop(owner, "count", 42.0); + crate::closure::closure_mark_key_deleted(owner, "name"); + + let _ = gc_collect_minor(); + + assert_eq!( + crate::closure::closure_get_own_dynamic_prop(owner, "count"), + Some(42.0) + ); + assert!(crate::closure::closure_is_key_deleted(owner, "name")); + let row = walk("closure.dynamic_props"); + assert!(row.partial); + assert!(row.table_len >= 2, "{row:?}"); + assert_eq!( + row.visited, 0, + "an old owner with no heap values must not be visited by a minor: {row:?}" + ); +} + +#[test] +fn dead_young_closure_owner_is_pruned_by_the_young_prune() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::closure::scan_closure_dynamic_props_roots_mut); + // One rooted young object so the minor has real work; the owner is not it. + js_shadow_slot_set(0, string_bits(young_leaf())); + + let dead = young_closure(); + crate::closure::closure_set_dynamic_prop(dead, "memo", 42.0); + crate::closure::closure_set_static_prototype(dead, crate::value::TAG_NULL); + assert!(crate::closure::closure_get_own_dynamic_prop(dead, "memo").is_some()); + + let _ = gc_collect_minor(); + + assert!( + crate::closure::closure_get_own_dynamic_prop(dead, "memo").is_none(), + "the dead young owner's CLOSURE_PROPS entry must be pruned from the log" + ); + assert!(crate::closure::closure_static_prototype(dead).is_none()); +} + +// -------------------------------------------------------------- descriptors + +#[test] +fn young_accessor_getter_is_moved_through_the_log() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::object::descriptor_state::scan_descriptor_roots_mut); + + let (owner, _) = unsafe { alloc_nursery_test_object(0) }; + let owner = owner as usize; + js_shadow_slot_set(0, ptr_bits(owner)); + // The getter closure is reachable ONLY through the accessor table. + let getter = young_closure(); + crate::object::set_accessor_descriptor( + owner, + "g".to_string(), + crate::object::AccessorDescriptor { + get: ptr_bits(getter), + set: 0, + }, + ); + + let _ = gc_collect_minor(); + + let owner_after = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!(owner_after, owner); + let acc = crate::object::get_accessor_descriptor(owner_after, "g") + .expect("accessor must follow its owner to the new address"); + let getter_after = (acc.get & POINTER_MASK) as usize; + assert_ne!(getter_after, getter, "the getter must have been evacuated"); + assert!(crate::arena::pointer_in_nursery(getter_after)); + assert!(crate::object::get_accessor_descriptor(owner, "g").is_none()); + let row = walk("object.descriptors"); + assert!(row.partial); + assert!(row.visited >= 1, "{row:?}"); +} + +#[test] +fn old_descriptor_owners_are_skipped_by_a_minor() { + let _guard = CopyingNurseryTestGuard::new(0); + gc_register_mutable_root_scanner(crate::object::descriptor_state::scan_descriptor_roots_mut); + + // The descriptor tables are agent state that outlives every test on this + // thread, and the FIRST descriptor install on a thread bootstraps the + // lazy `globalThis` realm (#7975), which installs ~1.8k builtin + // descriptors on young objects. Warm that up, take a minor, then measure + // the delta: the old entry must add index rows but no visit. + let (warm, _) = unsafe { alloc_old_test_object(0) }; + crate::object::set_property_attrs( + warm as usize, + "warm".to_string(), + crate::object::PropertyAttrs::new(false, true, true), + ); + let _ = gc_collect_minor(); + let before = walk("object.descriptors"); + + let (owner, _) = unsafe { alloc_old_test_object(0) }; + let owner = owner as usize; + let getter = old_closure(); + crate::object::set_accessor_descriptor( + owner, + "g".to_string(), + crate::object::AccessorDescriptor { + get: ptr_bits(getter), + set: 0, + }, + ); + crate::object::set_property_attrs( + owner, + "p".to_string(), + crate::object::PropertyAttrs::new(false, true, true), + ); + + let _ = gc_collect_minor(); + + assert_eq!( + crate::object::get_accessor_descriptor(owner, "g").map(|acc| acc.get), + Some(ptr_bits(getter)) + ); + let row = walk("object.descriptors"); + assert!(row.partial); + // The first minor's prune can drop dead realm owners between the two + // walks, so the exact count is `kept` minus whatever died; the new old + // entry can only NOT add to it. + assert!( + row.visited <= before.kept, + "old owner, old getter: the new entry must not add a visit: {before:?} -> {row:?}" + ); + assert!( + row.visited < row.table_len, + "the walk must stay partial: {row:?}" + ); +} + +#[test] +fn dead_young_descriptor_owner_is_pruned_by_the_young_prune() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::object::descriptor_state::scan_descriptor_roots_mut); + js_shadow_slot_set(0, string_bits(young_leaf())); + + let (dead, _) = unsafe { alloc_nursery_test_object(0) }; + let dead = dead as usize; + crate::object::set_property_attrs( + dead, + "p".to_string(), + crate::object::PropertyAttrs::new(false, true, true), + ); + assert!(crate::object::get_property_attrs(dead, "p").is_some()); + + let _ = gc_collect_minor(); + + assert!( + crate::object::get_property_attrs(dead, "p").is_none(), + "the dead young owner's descriptor must be pruned from the log" + ); +} + +// ------------------------------------------------------------------- shapes + +#[test] +fn young_keys_array_family_is_rekeyed_through_the_log() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::object::shapes::scan_shape_table_rekey_mut); + + let keys = unsafe { young_keys_array() }; + js_shadow_slot_set(0, ptr_bits(keys as usize)); + let id = crate::object::shapes::shape_descriptor_ensure(keys, 0, 0).expect("shape id"); + assert_eq!( + crate::object::shapes::shape_descriptor_by_id(id).map(|d| d.keys), + Some(keys as u64) + ); + + let _ = gc_collect_minor(); + + let keys_after = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!( + keys_after, keys as usize, + "the rooted keys array must have moved" + ); + assert_eq!( + crate::object::shapes::shape_descriptor_by_id(id).map(|d| d.keys), + Some(keys_after as u64), + "the family's descriptor must be re-keyed to the evacuated keys array" + ); + let row = walk("shapes.families+indices"); + assert!(row.partial); + assert!(row.visited >= 1, "{row:?}"); +} + +#[test] +fn old_shape_families_are_skipped_by_a_minor() { + let _guard = CopyingNurseryTestGuard::new(0); + gc_register_mutable_root_scanner(crate::object::shapes::scan_shape_table_rekey_mut); + + let keys = crate::arena::arena_alloc_gc_old( + std::mem::size_of::(), + std::mem::align_of::(), + GC_TYPE_ARRAY, + ) as *mut crate::array::ArrayHeader; + unsafe { + (*keys).length = 0; + (*keys).capacity = 0; + } + let id = crate::object::shapes::shape_descriptor_ensure(keys, 0, 0).expect("shape id"); + + let _ = gc_collect_minor(); + + assert_eq!( + crate::object::shapes::shape_descriptor_by_id(id).map(|d| d.keys), + Some(keys as u64) + ); + let row = walk("shapes.families+indices"); + assert!(row.partial); + assert!(row.table_len >= 1, "{row:?}"); + assert_eq!( + row.visited, 0, + "an old keys array's family must not be visited: {row:?}" + ); +} + +// ------------------------------------------------------------------- caches + +#[test] +fn young_transition_cache_target_is_rewritten_through_the_log() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::object::scan_transition_cache_roots_mut); + + let keys = unsafe { young_keys_array() } as usize; + js_shadow_slot_set(0, ptr_bits(keys)); + // A predecessor that resolves, or the copied-minor prune retires the entry + // (`shape_descriptor_by_id(0)` is `None`) before the assertion reads it. + let prev = + crate::object::shapes::shape_descriptor_ensure(std::ptr::null(), 0, 1).expect("shape id"); + crate::object::test_seed_transition_cache_root_for_shape(prev, keys); + + let _ = gc_collect_minor(); + + let keys_after = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!(keys_after, keys); + assert_eq!( + crate::object::test_transition_cache_root(), + keys_after, + "the cached target must be rewritten to the evacuated keys array" + ); + let row = walk("object.transition_cache"); + assert!(row.partial); + assert!(row.visited >= 1, "{row:?}"); + assert!( + row.visited < row.table_len, + "a 16k-slot table must not be walked whole: {row:?}" + ); +} + +/// The PRODUCTION transition-cache writer arms on EITHER address, and the +/// second clause — a young interned KEY under an old target — was covered by +/// no test: all three `#[cfg(test)]` seeds either ignored the key or +/// classified it unconditionally, so `transition_cache_insert`'s own +/// `(len_marker == 0 && addr_is_minor_relevant(kid))` arm could be deleted +/// while the suite stayed green. This drives the real writer. +#[test] +fn young_transition_key_under_an_old_target_arms_the_log_through_the_writer() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::object::scan_transition_cache_roots_mut); + crate::object::test_clear_transition_cache_root(); + + // Target OLD: the first clause of the predicate is false for it. + let old_target = unsafe { + let arr = crate::arena::arena_alloc_gc_old( + std::mem::size_of::(), + std::mem::align_of::(), + GC_TYPE_ARRAY, + ) as *mut crate::array::ArrayHeader; + (*arr).length = 0; + (*arr).capacity = 0; + arr as usize + }; + assert!(!crate::arena::pointer_in_nursery(old_target)); + + // Key YOUNG, and long enough that `transition_key_id` keeps it a POINTER + // (`len_marker == 0`) rather than packing it as a length. + let key = crate::string::js_string_from_bytes(b"young-transition-key".as_ptr(), 20); + assert!(crate::arena::pointer_in_nursery(key as usize)); + + let before = young_log::last_walk("object.transition_cache"); + crate::object::test_transition_cache_insert(0, key, old_target, 0, 0); + + // The log must now name the slot the writer published into. Reading it + // through a minor is the same observation the scanner makes. + let _ = gc_collect_minor(); + let row = walk("object.transition_cache"); + assert!(row.partial, "{row:?}"); + assert!( + row.visited >= 1, + "the young KEY must have armed the log: {row:?} (before: {before:?})" + ); +} + +#[test] +fn young_shape_cache_entry_is_moved_through_the_log() { + let _guard = CopyingNurseryTestGuard::new(0); + gc_register_mutable_root_scanner(crate::object::scan_shape_cache_roots_mut); + + // Reachable ONLY through the cache (which roots it). Seeded through the + // PRODUCTION writer (`shape_cache_insert`), not a test seam: a seam that + // arms the log itself makes this test pass with the writer's own arm site + // deleted, which is how #9755 shipped an unenforced rule 1. + let keys = unsafe { young_keys_array() }; + let shape_id = 0x9754_0001; + crate::object::test_shape_cache_insert(shape_id, keys); + + let _ = gc_collect_minor(); + + let (inline, overflow) = crate::object::test_shape_cache_root(shape_id); + assert_ne!( + overflow, keys as usize, + "the overflow entry must have been evacuated" + ); + assert!(crate::arena::pointer_in_nursery(overflow)); + assert_eq!( + inline, overflow, + "inline and overflow must agree on the new address" + ); + let row = walk("object.shape_cache"); + assert!(row.partial); + assert!(row.visited >= 1, "{row:?}"); +} + +// --------------------------------------------------------------------------- +// `shapes.indices` arming (#9756 restructures this table; nothing exercised +// its four arm sites, so a missed `note` there was a collected live object +// that this suite would not have caught). +// --------------------------------------------------------------------------- + +/// Keys count above `KEYS_INDEX_THRESHOLD` (32), so the index is built at all. +const INDEXED_KEYS: u32 = 40; + +/// A YOUNG keys array of `INDEXED_KEYS` young string keys, in the dense +/// NaN-boxed layout `keys_array_dense_slots` reads. +unsafe fn young_indexed_keys_array() -> (*mut crate::array::ArrayHeader, Vec>) { + let arr = crate::array::js_array_alloc_with_length(INDEXED_KEYS); + let slots = (arr as *mut u8).add(std::mem::size_of::()) as *mut f64; + let mut names = Vec::new(); + for i in 0..INDEXED_KEYS { + let name = format!("young_key_{i:04}"); + let s = crate::string::js_string_from_bytes(name.as_ptr(), name.len() as u32); + *slots.add(i as usize) = f64::from_bits(string_bits(s as usize)); + names.push(name.into_bytes()); + } + (*arr).length = INDEXED_KEYS; + (arr, names) +} + +unsafe fn build_index_for(keys: *mut crate::array::ArrayHeader, names: &[Vec]) { + // `build = true` is the arm site: it inserts the `indices` entry. + crate::object::shapes::test_build_slot_index(keys, &names[0], INDEXED_KEYS); +} + +/// S17 — `shape_slot_lookup_verdict`'s build arm publishes an `indices` entry +/// keyed by a YOUNG keys address. +#[test] +fn building_a_slot_index_on_a_young_keys_array_arms_the_log() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::object::shapes::scan_shape_table_rekey_mut); + crate::object::shapes::test_clear_shape_table(); + + let (keys, names) = unsafe { young_indexed_keys_array() }; + js_shadow_slot_set(0, ptr_bits(keys as usize)); + assert!(crate::arena::pointer_in_nursery(keys as usize)); + unsafe { build_index_for(keys, &names) }; + assert!(crate::object::shapes::test_shape_index_len(keys as usize) > 0); + + // Rule 2 re-derives the relevant set from `indices` during the walk and + // panics if the log does not name this address. + let _ = gc_collect_minor(); + + let moved = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!( + moved, keys as usize, + "the keys array must have been evacuated" + ); + assert!( + crate::object::shapes::test_shape_index_len(moved) > 0, + "the index must follow the keys array to its new address" + ); +} + +/// S18 — `shape_keys_grown` re-keys the index onto the grown array's address. +#[test] +fn growing_an_indexed_keys_array_arms_the_log_for_the_new_address() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::object::shapes::scan_shape_table_rekey_mut); + crate::object::shapes::test_clear_shape_table(); + + let (old_keys, names) = unsafe { young_indexed_keys_array() }; + unsafe { build_index_for(old_keys, &names) }; + let (new_keys, _) = unsafe { young_indexed_keys_array() }; + js_shadow_slot_set(0, ptr_bits(new_keys as usize)); + + crate::object::shapes::shape_keys_grown(old_keys as usize, new_keys); + assert!(crate::object::shapes::test_shape_index_len(new_keys as usize) > 0); + + let _ = gc_collect_minor(); + let moved = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!(moved, new_keys as usize); + assert!(crate::object::shapes::test_shape_index_len(moved) > 0); +} + +/// S19 — `shape_index_migrate_after_delete` moves a complete index onto the +/// compacted array's address. +#[test] +fn migrating_an_index_after_a_delete_arms_the_log_for_the_new_address() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::object::shapes::scan_shape_table_rekey_mut); + crate::object::shapes::test_clear_shape_table(); + + let (old_keys, names) = unsafe { young_indexed_keys_array() }; + unsafe { build_index_for(old_keys, &names) }; + let (new_keys, _) = unsafe { young_indexed_keys_array() }; + js_shadow_slot_set(0, ptr_bits(new_keys as usize)); + + let migrated = crate::object::shapes::test_shape_index_migrate_after_delete( + old_keys as usize, + new_keys as usize, + /* removed_slot = */ 0, + INDEXED_KEYS, + /* old_keys_shared = */ false, + ); + assert!(migrated, "a complete index must migrate"); + + let _ = gc_collect_minor(); + let moved = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!(moved, new_keys as usize); + assert!(crate::object::shapes::test_shape_index_len(moved) > 0); +} + +/// S16 — `family_push_front`, the external-id install path, publishes a family +/// under a YOUNG keys address. +#[test] +fn installing_an_external_shape_id_arms_the_family_log() { + let _guard = CopyingNurseryTestGuard::new(1); + gc_register_mutable_root_scanner(crate::object::shapes::scan_shape_table_rekey_mut); + crate::object::shapes::test_clear_shape_table(); + + let keys = unsafe { young_keys_array() }; + js_shadow_slot_set(0, ptr_bits(keys as usize)); + let id = crate::object::shapes::test_unused_external_shape_id(); + assert!( + crate::object::shapes::test_install_external_shape_id(id, keys, 0, 0), + "the external id must install" + ); + + let _ = gc_collect_minor(); + let moved = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!(moved, keys as usize); + assert!( + !crate::object::shapes::test_shape_ids_for_keys(moved).is_empty(), + "the family must have followed the keys array" + ); +} diff --git a/crates/perry-runtime/src/gc/verify.rs b/crates/perry-runtime/src/gc/verify.rs index e5367d25bb..0332bcf3b8 100644 --- a/crates/perry-runtime/src/gc/verify.rs +++ b/crates/perry-runtime/src/gc/verify.rs @@ -212,8 +212,28 @@ pub(super) unsafe fn remember_evacuated_old_copy_young_slots( /// construction. Pages whose every slot now points old (the common case /// after evacuation rewrites) are still dropped, so the remembered set keeps /// shrinking as before. -pub(super) fn restore_surviving_dirty_coverage(snapshot: &RememberedDirtySnapshot) { +/// +/// #9754: `covered` names the objects the cycle's own dirty scan visited +/// COMPLETELY (every pointer slot on a dirty page and inside the body — +/// `scan_dirty_object_slots`). For those, `visit_slot_with_parent` already +/// re-remembered every slot whose post-visit child still needs tracking with +/// the same predicate this walk applies, so re-walking them can only re-insert +/// pages the sticky restore just inserted. They are skipped; the walk is then +/// proportional to the objects the dirty scan could NOT fully cover +/// (multi-page arrays, owners of out-of-body buffers) instead of to every slot +/// on every dirty page. Under `debug_assertions` the skipped objects are +/// walked anyway and any page the walk would have ADDED is a panic — the +/// machine check of the equivalence argument above. +pub(super) fn restore_surviving_dirty_coverage( + snapshot: &RememberedDirtySnapshot, + covered: &crate::fast_hash::PtrHashSet, + cycle_label: &str, +) { let mut sticky = StickyRememberedSet::default(); + let mut walked = 0usize; + let mut skipped = 0usize; + #[cfg(debug_assertions)] + let mut skipped_sticky = StickyRememberedSet::default(); // Mirror scan_remembered_dirty_slots_copying's scan_header guards: the // external dirty entries can carry headers the harness seeded // synthetically, and a dead entry may point at reclaimed memory — never @@ -249,6 +269,13 @@ pub(super) fn restore_surviving_dirty_coverage(snapshot: &RememberedDirtySnapsho }; if !snapshot.dirty_old_pages.is_empty() { crate::arena::old_arena_walk_objects_on_pages(&snapshot.dirty_old_pages, |hp| { + if covered.contains(&(hp as usize)) { + skipped += 1; + #[cfg(debug_assertions)] + debug_visit_covered_parent(hp as *mut GcHeader, &mut skipped_sticky); + return; + } + walked += 1; visit_parent(hp as *mut GcHeader); }); } @@ -257,6 +284,13 @@ pub(super) fn restore_surviving_dirty_coverage(snapshot: &RememberedDirtySnapsho if !seen_external.insert(header_addr) { continue; } + if covered.contains(&header_addr) { + skipped += 1; + #[cfg(debug_assertions)] + debug_visit_covered_parent(header_addr as *mut GcHeader, &mut skipped_sticky); + continue; + } + walked += 1; // External entries may be stale (or, in the GC unit tests, // synthetic). Establish that the address is dereference-safe // WITHOUT touching it: old/longlived arena pages are always @@ -275,7 +309,60 @@ pub(super) fn restore_surviving_dirty_coverage(snapshot: &RememberedDirtySnapsho visit_parent(header_addr as *mut GcHeader); } } - sticky.restore(); + let added = sticky.restore_counted(); + #[cfg(debug_assertions)] + { + let would_add = skipped_sticky.count_not_yet_dirty(); + assert_eq!( + would_add, 0, + "restore_surviving_dirty_coverage: {would_add} page(s) of {skipped} \ + dirty-scan-covered object(s) are not remembered — the dirty scan's \ + per-slot re-remembering disagrees with the coverage walk for an \ + object `scan_dirty_object_slots` reported complete" + ); + } + if crate::gc::gc_diag_enabled() { + eprintln!( + "[gc-restore-coverage] {cycle_label} dirty_pages={} objects_walked={walked} objects_skipped={skipped} pages_added={added}", + snapshot.dirty_pages.len() + ); + } +} + +/// Debug twin of the restore's `visit_parent` for a skipped object: re-derive +/// what the full walk would have remembered so the caller can assert it adds +/// nothing beyond what the dirty scan already restored. +#[cfg(debug_assertions)] +fn debug_visit_covered_parent(header: *mut GcHeader, sticky: &mut StickyRememberedSet) { + unsafe { + if header.is_null() { + return; + } + let arena_parent = plausible_gc_header(header, true); + let malloc_parent = !arena_parent && plausible_gc_header(header, false); + if !arena_parent && !malloc_parent { + return; + } + if (*header).gc_flags & GC_FLAG_FORWARDED != 0 { + return; + } + let user = (header as *mut u8).add(GC_HEADER_SIZE) as usize; + if arena_parent + && !matches!( + crate::arena::classify_heap_generation(user), + crate::arena::HeapGeneration::Old + ) + { + return; + } + visit_gc_rewrite_slots(header, |slot| { + if crate::weakref::is_weak_target_trace_slot(header, slot.slot) { + return; + } + slot.record_layout_read(); + remember_evacuated_old_to_young_slot(sticky, header, slot.slot); + }); + } } pub(super) fn rebuild_evacuated_old_to_young_remembered_set( diff --git a/crates/perry-runtime/src/gc/young_log.rs b/crates/perry-runtime/src/gc/young_log.rs new file mode 100644 index 0000000000..368d146149 --- /dev/null +++ b/crates/perry-runtime/src/gc/young_log.rs @@ -0,0 +1,300 @@ +//! Young-entry logs: per-side-table remembered sets for the copying minor +//! and the budgeted minor-only trace. +//! +//! # The problem this solves +//! +//! Every registered mutable-root scanner walks its whole table on every +//! collection. On the compiled claude-code TUI that is ~35k shape families, +//! ~120k descriptors and ~13k closure-prop owners per walk, three walks per +//! copying minor and two per budgeted minor (initial root scan + final +//! remark), 41+ minors per streamed reply — and every one of those walks found +//! `slots=0`: 34–56 ms per minor of scanner time discovering that nothing in +//! the table pointed at the nursery (`[gc-scanner-profile]`, 2026-09-04). +//! +//! The previous attempt (2026-09-01, reverted) skipped a walk when the +//! VISITOR was inert. That is unsound for the three expensive tables: they +//! ROOT (`visit_nanbox_u64_slot` on accessor get/set, `visit_nanbox_f64_slot` +//! on closure prop values, `visit_usize_slot` on carried keys arrays), and +//! rooting marks in Mark/Copy, so skipping drops live objects. +//! +//! # What a minor can act on +//! +//! A minor-scoped pass — the copying minor's preflight/mark/rewrite passes and +//! the budgeted cycle's `GcCollectionKind::Minor` root scan — neither moves +//! nor sweeps an OLD-generation object. `CopyingNurseryCollector::mark_addr` +//! returns an old address unchanged; `rewrite_raw_addr` follows forwarding +//! records that only a moved (young) object can carry; a minor-only trace's +//! marks on old headers are consumed by nothing (the minor sweep frees young +//! objects only, `PostTraceProbe::owner_is_dead` refuses old owners, and the +//! full path's remembered-set rebuild does not run for minors). So an entry +//! whose key AND every value are old is a provable no-op for every +//! minor-scoped visit. The set of addresses a minor CAN act on is the +//! complement: nursery, longlived (traced through, never swept, not barriered — +//! `barrier_parent_needs_remembering` is false for it) and malloc-GC objects +//! (swept by minors). [`addr_is_minor_relevant`] is that predicate. +//! +//! # The log +//! +//! [`YoungLog`] holds the KEYS of entries that may hold a minor-relevant +//! pointer. It is a log, not an index: duplicates and stale keys are allowed +//! (a stale key is a lookup miss and is dropped; a duplicate is visited twice, +//! idempotently). A minor-scoped scanner walks only the logged keys, visits +//! each entry exactly as the full walk would, and re-logs the entry iff it is +//! still relevant after the visit (a to-space survivor stays young; a promoted +//! object is old and drops out). A full-scope scanner walks the whole table as +//! before and REBUILDS the log from what it found. +//! +//! # The three rules (from `perry-young-gc-fixed-cost.md`) +//! +//! 1. **Arm before publish.** Every writer notes the key BEFORE the entry +//! becomes findable. A note is one page-map probe (`classify_heap_generation`, +//! hot-TLS cached) per pointer; an old key with old values notes nothing. +//! 2. **Machine-check the writer set.** Under `debug_assertions` a minor-scoped +//! walk first re-derives the relevant set from the authoritative table and +//! panics on any relevant entry the log does not name +//! ([`debug_assert_logged`]). The `gc/tests` copying-minor fixtures run this +//! on every collection, so deleting one `note` site is a red test, not a +//! silent leak. +//! 3. **A skip needs a counter.** [`note_walk`] records, per walk, how many +//! entries the log named, how many were visited, how many stayed relevant +//! and how big the table was; `[gc-young-log]` prints them per collection +//! under `PERRY_GC_DIAG=1`, and the tests read them back. +//! +//! Thread model: the shape and descriptor tables are agent-local +//! (`state()`), so their logs live beside them. The closure side tables are +//! process-global mutexed maps; their log is thread-local on purpose — an +//! entry's addresses belong to the heap of the thread that inserted it, and +//! only that thread's minors can act on them, so a foreign thread's walk must +//! neither visit nor drop them. + +use std::cell::RefCell; + +use super::GC_HEADER_SIZE; +use crate::arena::HeapGeneration; +use crate::value::{BIGINT_TAG, POINTER_MASK, POINTER_TAG, STRING_TAG, TAG_MASK}; + +/// Keys of side-table entries that may hold a pointer a minor can act on. +pub(crate) struct YoungLog { + keys: Vec, + /// A recycled buffer, so the log does not grow from ZERO CAPACITY every + /// collection. `take_sorted` hands `keys` out and installs this in its + /// place, so the notes made while the caller walks the batch land in a Vec + /// that already has room; `extend` and `take_spare` round the buffers + /// back. Without it every cycle re-grew a 20k-entry Vec from empty — the + /// same allocate-from-scratch shape these logs exist to remove from the + /// scanners, reintroduced one level down. + spare: Vec, +} + +impl YoungLog { + pub(crate) const fn new() -> Self { + Self { + keys: Vec::new(), + spare: Vec::new(), + } + } + + /// Record `key` as possibly minor-relevant. MUST run before the entry it + /// describes becomes findable (rule 1). Adjacent duplicates — a hot + /// `fn.x = …` loop — are collapsed; other duplicates are harmless. + #[inline] + pub(crate) fn note(&mut self, key: K) { + if self.keys.last() != Some(&key) { + self.keys.push(key); + } + } + + /// Take the logged keys, sorted and deduplicated, leaving the log empty. + /// Notes made while the caller walks the batch (owner-move hooks fire + /// from inside a visit) land in the emptied log and are picked up by the + /// caller's next `take_sorted` round. + pub(crate) fn take_sorted(&mut self) -> Vec { + // Swap the recycled buffer in rather than leaving a zero-capacity Vec + // behind: the notes made while the caller walks the batch land here. + let mut keys = std::mem::replace(&mut self.keys, std::mem::take(&mut self.spare)); + self.keys.clear(); + keys.sort_unstable(); + keys.dedup(); + keys + } + + /// A buffer for a walk's `kept` list, reusing the log's spare capacity. + /// Hand it back through [`extend`](Self::extend). + pub(crate) fn take_spare(&mut self) -> Vec { + let mut buf = std::mem::take(&mut self.spare); + buf.clear(); + buf + } + + /// Keep the larger of the two drained buffers for the next cycle. + fn stash_spare(&mut self, mut buf: Vec) { + buf.clear(); + if buf.capacity() > self.spare.capacity() { + self.spare = buf; + } + } + + /// Re-log the keys a walk found still relevant. + pub(crate) fn extend(&mut self, kept: Vec) { + if self.keys.is_empty() { + let drained = std::mem::replace(&mut self.keys, kept); + self.stash_spare(drained); + } else { + self.keys.extend_from_slice(&kept); + self.stash_spare(kept); + } + } + + /// Test-only: the table resets (`test_clear_*`) clear their log with them. + #[cfg(test)] + pub(crate) fn clear(&mut self) { + self.keys.clear(); + self.spare.clear(); + } + + /// Rule 2: the log must name every key in `relevant`. `relevant` is the + /// set the caller re-derived from the authoritative table under + /// `debug_assertions`; a miss is a writer that publishes without noting. + #[cfg(debug_assertions)] + pub(crate) fn debug_assert_logged(&self, table: &'static str, relevant: &[K]) + where + K: std::fmt::Debug, + { + if relevant.is_empty() { + return; + } + let mut logged = self.keys.clone(); + logged.sort_unstable(); + logged.dedup(); + for key in relevant { + assert!( + logged.binary_search(key).is_ok(), + "young log for {table} does not name {key:?}, which holds a \ + minor-relevant pointer: a writer of that table publishes \ + without `note`-ing the key first (see gc/young_log.rs rule 1)" + ); + } + } +} + +/// Can a minor-scoped pass act on the object at `addr`? +/// +/// `false` is authoritative for the old generation only: an old object is +/// neither moved nor swept by any minor, and it never becomes young again. +/// Everything a minor moves, marks-through or sweeps answers `true` — +/// nursery (eden + both survivor halves), longlived, and malloc-GC objects. +/// A non-heap word (handle id, foreign-thread address, integer) answers +/// `false` through the exact malloc-registry probe, never a header sniff. +#[inline] +pub(crate) fn addr_is_minor_relevant(addr: usize) -> bool { + if addr == 0 { + return false; + } + match crate::arena::classify_heap_generation(addr) { + HeapGeneration::Old => false, + HeapGeneration::Nursery | HeapGeneration::Longlived => true, + HeapGeneration::Unknown => { + addr > GC_HEADER_SIZE + && super::malloc::gc_malloc_header_is_tracked( + (addr - GC_HEADER_SIZE) as *const super::GcHeader, + ) + } + } +} + +/// [`addr_is_minor_relevant`] for a NaN-boxed value: only the three +/// pointer-carrying tags decode to an address; numbers, booleans, short +/// strings and `undefined` are never relevant. +#[inline] +pub(crate) fn bits_are_minor_relevant(bits: u64) -> bool { + let tag = bits & TAG_MASK; + if tag == POINTER_TAG || tag == STRING_TAG || tag == BIGINT_TAG { + addr_is_minor_relevant((bits & POINTER_MASK) as usize) + } else { + false + } +} + +/// One scanner walk's accounting (rule 3). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct YoungLogWalk { + /// The walk was minor-scoped and visited only the logged keys. + pub(crate) partial: bool, + /// Keys the log named when the walk started (after dedup). + pub(crate) logged: u64, + /// Entries actually visited (a full walk visits the whole table). + pub(crate) visited: u64, + /// Entries still relevant after the visit, i.e. the log size afterwards. + pub(crate) kept: u64, + /// Table size at walk time — `table_len - visited` is the work skipped. + pub(crate) table_len: u64, +} + +crate::perry_thread_local! { + /// Per-table rows since the last `report_and_reset`, keyed by table name. + static WALK_ROWS: RefCell> = + const { RefCell::new(Vec::new()) }; + /// The most recent walk per table, for the tests (never reset). + static LAST_WALK: RefCell> = + const { RefCell::new(Vec::new()) }; +} + +/// Record one walk. Cheap (one push per scanner per pass), so it is not +/// gated; printing is. +pub(crate) fn note_walk(table: &'static str, walk: YoungLogWalk) { + LAST_WALK.with(|rows| { + let mut rows = rows.borrow_mut(); + if let Some(row) = rows.iter_mut().find(|(name, _)| *name == table) { + row.1 = walk; + } else { + rows.push((table, walk)); + } + }); + if !super::gc_diag_enabled() { + return; + } + WALK_ROWS.with(|rows| { + let mut rows = rows.borrow_mut(); + if let Some(row) = rows.iter_mut().find(|(name, _, _)| *name == table) { + row.1.partial &= walk.partial; + row.1.logged += walk.logged; + row.1.visited += walk.visited; + row.1.kept += walk.kept; + row.1.table_len = row.1.table_len.max(walk.table_len); + row.2 += 1; + } else { + rows.push((table, walk, 1)); + } + }); +} + +/// The most recent walk recorded for `table` on this thread. +#[cfg(test)] +pub(crate) fn last_walk(table: &'static str) -> Option { + LAST_WALK.with(|rows| { + rows.borrow() + .iter() + .find(|(name, _)| *name == table) + .map(|(_, walk)| *walk) + }) +} + +/// Print the rows accumulated since the last report and clear them. Called +/// beside `scanner_profile::report_and_reset` so the two read together. +pub(super) fn report_and_reset(cycle_label: &str) { + if !super::gc_diag_enabled() { + return; + } + let rows = WALK_ROWS.with(|rows| std::mem::take(&mut *rows.borrow_mut())); + for (table, walk, passes) in rows { + eprintln!( + "[gc-young-log] {cycle_label} table={table} mode={} passes={passes} logged={} visited={} kept={} table_len={} skipped={}", + if walk.partial { "young" } else { "full" }, + walk.logged, + walk.visited, + walk.kept, + walk.table_len, + (walk.table_len * u64::from(passes)).saturating_sub(walk.visited), + ); + } +} diff --git a/crates/perry-runtime/src/hot_diag.rs b/crates/perry-runtime/src/hot_diag.rs index 48216ced2c..faa49fba65 100644 --- a/crates/perry-runtime/src/hot_diag.rs +++ b/crates/perry-runtime/src/hot_diag.rs @@ -405,6 +405,26 @@ struct SiteStat { key: String, misses: u64, by_reason: [u32; IC_MISS_REASONS], + /// Primes at this site whose token equals the one the site's MRU entry + /// ALREADY held. The cache was written with this shape, the next read of + /// the same shape came back to the miss handler anyway, and the handler + /// wrote the identical value again: the prime is not sticking. See + /// [`IcDiag::prime_same_token`]. + prime_same_token: u64, + /// Primes whose token differs from the MRU entry's — the site really is + /// seeing more than one receiver shape (polymorphism / megamorphism). + prime_new_token: u64, + /// Primes taken while the site's `PIC_WAY_STATE` was < 0, i.e. the ways + /// are latched off because the rotation was wider than they hold. + prime_while_megamorphic: u64, + /// Primes taken while the site had no way populated yet (state == 0). + prime_while_fresh: u64, + /// Primes taken while the site's ways were populated and live (state > 0). + prime_while_armed: u64, + /// Primes whose token was ALREADY sitting in one of the site's ways. The + /// cache held the right answer in a way and the read came back to the miss + /// handler regardless. See [`IcDiag::prime_in_ways`]. + prime_in_ways: u64, } #[derive(Default)] @@ -415,14 +435,96 @@ pub struct IcDiag { pub misses: u64, by_reason: [u64; IC_MISS_REASONS], sites: HashMap, + /// THE SPLIT. A site classified `own_inline_primed` misses, finds the + /// property as an own inline slot, and primes the cache — and then misses + /// again. Two mutually exclusive explanations, and the fix differs: + /// + /// * `prime_new_token` — the receiver's shape really did change between + /// the two reads. The site is polymorphic and the ways are (or should + /// be) doing their job; a fix would widen or re-tier them. + /// * `prime_same_token` — the site re-primed the shape it already had. + /// The cache holds the right answer and the emitted hit path did not + /// use it, or something invalidated it in between. That is a + /// priming/invalidation or IC-layout bug, not polymorphism. + /// + /// Measured on the claude-code TUI, shape identity is NOT the explanation + /// for the bulk of the misses (`{value, done}` iterator results share one + /// keys array since #7564 and their read sites still miss ~178k times per + /// turn), which is what this counter exists to settle. + pub prime_same_token: u64, + pub prime_new_token: u64, + pub prime_while_megamorphic: u64, + pub prime_while_fresh: u64, + pub prime_while_armed: u64, + /// The decisive counter for the `new_token` half. `prime_same_token` only + /// compares against the MRU entry (word 0), so a site rotating k <= 5 + /// shapes reports `new_token` on every prime even when the ways are doing + /// exactly what they were built for. This counts the primes whose token was + /// found in one of the four ways at prime time: the polymorphic cache + /// ALREADY held that shape's slot, and the emitted hit path still fell + /// through to the miss handler. + /// + /// So the three-way split of every prime is: + /// * `same_token` — re-primed the MRU shape (priming/invalidation), + /// * `new_token` + `in_ways` — the ways held it and were not consulted + /// (emitted gate / IC layout, i.e. codegen), + /// * `new_token` + not `in_ways` — a shape neither the MRU entry nor the + /// ways had (genuine polymorphism, or a first sighting). + pub prime_in_ways: u64, } crate::perry_thread_local! { static IC_DIAG: RefCell = RefCell::new(IcDiag::default()); } -/// Record one IC miss. `site` is the per-site cache slot address (stable for -/// the process lifetime), `key` the property-name string bytes. +/// Record one `pic_prime_get`, splitting it by whether the token the site is +/// being primed with is one it already held — in the MRU entry (`same`) or in +/// one of the ways (`in_ways`). See [`IcDiag::prime_same_token`] and +/// [`IcDiag::prime_in_ways`]. +/// +/// Diagnostic only: called from `pic_prime_get` behind [`ic_on`], and every +/// value it reads (`prev_tok`, `token`, `state`, the ways) is one the caller +/// already has in a register or in the cache line it has just touched. +pub fn ic_note_prime(site: usize, prev_tok: i64, token: i64, state: i64, in_ways: bool) { + IC_DIAG.with(|d| { + let mut d = d.borrow_mut(); + if d.started.is_none() { + d.started = Some(Instant::now()); + d.last_dump = d.started; + } + let same = prev_tok != 0 && prev_tok == token; + if same { + d.prime_same_token += 1; + } else { + d.prime_new_token += 1; + } + if in_ways { + d.prime_in_ways += 1; + } + match state.cmp(&0) { + std::cmp::Ordering::Less => d.prime_while_megamorphic += 1, + std::cmp::Ordering::Equal => d.prime_while_fresh += 1, + std::cmp::Ordering::Greater => d.prime_while_armed += 1, + } + let s = d.sites.entry(site).or_default(); + if same { + s.prime_same_token += 1; + } else { + s.prime_new_token += 1; + } + if in_ways { + s.prime_in_ways += 1; + } + match state.cmp(&0) { + std::cmp::Ordering::Less => s.prime_while_megamorphic += 1, + std::cmp::Ordering::Equal => s.prime_while_fresh += 1, + std::cmp::Ordering::Greater => s.prime_while_armed += 1, + } + }); +} + +/// Record one IC miss. `site` is the per-site cache address (stable for the +/// process lifetime), `key` the property-name string bytes. pub fn ic_note(site: usize, key: &[u8], reason: IcMissReason) { IC_DIAG.with(|d| { let mut d = d.borrow_mut(); @@ -470,9 +572,33 @@ impl IcDiag { } } out.push('\n'); + // THE SPLIT: of every prime, how many re-primed the token the site + // already held (the cache was right and was not used) versus a token + // it had not seen (real polymorphism)? + let primes = self.prime_same_token + self.prime_new_token; + if primes != 0 { + let pct = |n: u64| 100.0 * n as f64 / primes as f64; + let _ = writeln!( + out, + " primes={primes} same_token={} ({:.1} %) new_token={} ({:.1} %) \ + in_ways={} ({:.1} %) | way_state: fresh={} armed={} megamorphic={}", + self.prime_same_token, + pct(self.prime_same_token), + self.prime_new_token, + pct(self.prime_new_token), + self.prime_in_ways, + pct(self.prime_in_ways), + self.prime_while_fresh, + self.prime_while_armed, + self.prime_while_megamorphic + ); + } let mut rows: Vec<&SiteStat> = self.sites.values().collect(); rows.sort_by_key(|s| std::cmp::Reverse(s.misses)); - let _ = writeln!(out, " misses key reasons"); + let _ = writeln!( + out, + " misses same/new/inways fresh/armed/mega key reasons" + ); for s in rows.iter().take(40) { let mut reasons = String::new(); let mut idx: Vec = (0..IC_MISS_REASONS) @@ -482,7 +608,18 @@ impl IcDiag { for i in idx.iter().take(3) { let _ = write!(reasons, " {}={}", IC_REASON_NAMES[*i], s.by_reason[*i]); } - let _ = writeln!(out, " {:6} {:<24}{reasons}", s.misses, s.key); + let _ = writeln!( + out, + " {:6} {:>8}/{}/{:<8} {:>7}/{}/{:<8} {:<24}{reasons}", + s.misses, + s.prime_same_token, + s.prime_new_token, + s.prime_in_ways, + s.prime_while_fresh, + s.prime_while_armed, + s.prime_while_megamorphic, + s.key + ); } out } diff --git a/crates/perry-runtime/src/object/descriptor_state.rs b/crates/perry-runtime/src/object/descriptor_state.rs index 909002a9aa..109ef1e75e 100644 --- a/crates/perry-runtime/src/object/descriptor_state.rs +++ b/crates/perry-runtime/src/object/descriptor_state.rs @@ -107,6 +107,11 @@ pub(crate) struct DescriptorTables { pub(crate) attr_keys_by_owner: RefCell>>, /// Accessor twin of [`Self::attr_keys_by_owner`]. pub(crate) accessor_keys_by_owner: RefCell>>, + /// #9754: owners whose entries may hold a pointer a minor can act on — + /// a young owner, or an accessor whose getter/setter closure is young. + /// A minor-scoped `scan_descriptor_roots_mut` visits only these; see + /// `gc/young_log.rs`. + pub(crate) young_owners: RefCell>, } impl DescriptorTables { @@ -118,10 +123,35 @@ impl DescriptorTables { property_attrs_in_use: Cell::new(false), attr_keys_by_owner: RefCell::new(new_fast_key_hash_map()), accessor_keys_by_owner: RefCell::new(new_fast_key_hash_map()), + young_owners: RefCell::new(crate::gc::young_log::YoungLog::new()), } } } +const DESCRIPTOR_YOUNG_LOG_NAME: &str = "object.descriptors"; + +mod young; +use young::{relevant_descriptor_owners, scan_descriptor_roots_young}; + +/// Rule 1 of `gc/young_log.rs`: log `owner` BEFORE its descriptor is +/// published when the owner, or the accessor closure being stored, can +/// matter to a minor. Data descriptors carry no pointer, so `acc` is `None` +/// for them and only the owner decides. +#[inline] +fn note_young_descriptor_owner( + st: &crate::state::RuntimeState, + owner: usize, + acc: Option<&AccessorDescriptor>, +) { + use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; + if addr_is_minor_relevant(owner) + || acc + .is_some_and(|acc| bits_are_minor_relevant(acc.get) || bits_are_minor_relevant(acc.set)) + { + st.descriptors.young_owners.borrow_mut().note(owner); + } +} + /// Record `key` as owned by `owner` in an owner index. Idempotent: a /// `defineProperty` that overwrites an existing descriptor must not push a /// duplicate, or the key would be reported twice by `Object.keys`. @@ -774,6 +804,7 @@ pub(crate) fn set_property_attrs(obj: usize, key: String, attrs: PropertyAttrs) GLOBAL_DESCRIPTORS_IN_USE.store(true, Ordering::Relaxed); disable_inline_guards_for_descriptor_target(obj, &key); note_meta_descriptor_key(obj, &key, false); + note_young_descriptor_owner(st, obj, None); owner_index_add(&st.descriptors.attr_keys_by_owner, obj, &key); st.descriptors .property_descriptors @@ -996,6 +1027,7 @@ pub(crate) fn set_accessor_descriptor(obj: usize, key: String, acc: AccessorDesc disable_inline_guards_for_descriptor_target(obj, &key); note_accessor_descriptor_key(&key); note_meta_descriptor_key(obj, &key, true); + note_young_descriptor_owner(st, obj, Some(&acc)); owner_index_add(&st.descriptors.accessor_keys_by_owner, obj, &key); st.descriptors .accessor_descriptors @@ -1064,6 +1096,7 @@ pub(crate) fn install_fresh_accessor_property( GLOBAL_DESCRIPTORS_IN_USE.store(true, Ordering::Relaxed); disable_inline_guards_for_descriptor_target(obj, &key); note_accessor_descriptor_key(&key); + note_young_descriptor_owner(st, obj, Some(&acc)); match note_meta_descriptor_key_both(obj, &key) { Some((accessor_bit_was_set, attr_bit_was_set)) => { if accessor_bit_was_set { @@ -1181,6 +1214,7 @@ pub(crate) fn set_builtin_accessor_descriptor( note_meta_descriptor_key(obj, &key, true); note_meta_descriptor_key(obj, &key, false); let st = state(); + note_young_descriptor_owner(st, obj, Some(&acc)); owner_index_add(&st.descriptors.accessor_keys_by_owner, obj, &key); owner_index_add(&st.descriptors.attr_keys_by_owner, obj, &key); st.descriptors @@ -1214,6 +1248,7 @@ pub(crate) fn set_builtin_property_attrs(obj: usize, key: String, attrs: Propert // #6759 Phase C2: see `set_builtin_accessor_descriptor`. note_meta_descriptor_key(obj, &key, false); let st = state(); + note_young_descriptor_owner(st, obj, None); owner_index_add(&st.descriptors.attr_keys_by_owner, obj, &key); st.descriptors .property_descriptors @@ -1313,6 +1348,51 @@ pub(crate) fn prune_dead_descriptor_owner_entries(is_dead_owner: &dyn Fn(usize) } } +/// [`prune_dead_descriptor_owner_entries`] for a MINOR (#9754): only a young +/// owner can be dead, and a young owner is always in the young log (noted at +/// insert, re-logged by every minor-scoped walk while it stays young), so the +/// log is the complete candidate set. +pub(crate) fn prune_dead_descriptor_owner_entries_young(is_dead_owner: &dyn Fn(usize) -> bool) { + let st = state(); + let candidates = st.descriptors.young_owners.borrow_mut().take_sorted(); + let mut kept = Vec::with_capacity(candidates.len()); + for owner in candidates { + if is_dead_owner(owner) { + remove_descriptor_owner_entries(st, owner); + } else { + kept.push(owner); + } + } + st.descriptors.young_owners.borrow_mut().extend(kept); +} + +/// Drop every entry `owner` holds in both tables and both indexes, through +/// the owner index (O(owner's keys), not O(table)). +fn remove_descriptor_owner_entries(st: &crate::state::RuntimeState, owner: usize) { + if let Some(keys) = st + .descriptors + .attr_keys_by_owner + .borrow_mut() + .remove(&owner) + { + let mut attrs = st.descriptors.property_descriptors.borrow_mut(); + for key in keys { + attrs.remove(&(owner, key)); + } + } + if let Some(keys) = st + .descriptors + .accessor_keys_by_owner + .borrow_mut() + .remove(&owner) + { + let mut accessors = st.descriptors.accessor_descriptors.borrow_mut(); + for key in keys { + accessors.remove(&(owner, key)); + } + } +} + /// #6710: drop every property-attr + accessor descriptor owned by `obj`. /// /// The generic descriptor tables are keyed by owner address; for a native @@ -1357,6 +1437,10 @@ pub(crate) fn transfer_descriptor_owner(old_owner: usize, new_owner: usize) { return; } let st = state(); + // The moved entries keep their accessor values, so the new owner is + // logged unconditionally; the next minor-scoped walk drops it if nothing + // in it is relevant any more. + st.descriptors.young_owners.borrow_mut().note(new_owner); // The owner index names exactly this owner's keys, so neither table is // walked in full any more. Array growth calls this on every reallocation. { @@ -1452,6 +1536,14 @@ fn rewrite_descriptor_owner( /// reused address). pub(crate) fn scan_descriptor_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { let st = state(); + // #9754: a minor-scoped pass visits only the young-logged owners; the + // full walk below rebuilds the log from what it finds. + if visitor.young_scope() { + scan_descriptor_roots_young(visitor, st); + return; + } + let table_len = st.descriptors.attr_keys_by_owner.borrow().len() as u64 + + st.descriptors.accessor_keys_by_owner.borrow().len() as u64; { // Probe DISTINCT OWNERS via the index, not every `(owner, key)` pair. // This runs on every GC cycle, and since the moving young-gen scavenge @@ -1537,6 +1629,86 @@ pub(crate) fn scan_descriptor_roots_mut(visitor: &mut crate::gc::RuntimeRootVisi } } } + + // A full walk is authoritative: rebuild the young log from the tables. + let kept = relevant_descriptor_owners(st); + let kept_len = kept.len() as u64; + { + let mut log = st.descriptors.young_owners.borrow_mut(); + let _ = log.take_sorted(); + log.extend(kept); + } + crate::gc::young_log::note_walk( + DESCRIPTOR_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: false, + logged: table_len, + visited: table_len, + kept: kept_len, + table_len, + }, + ); +} + +/// Visit one owner's descriptors. Returns the post-visit owner address and +/// whether the entry can still matter to a minor. +fn scan_descriptor_owner( + visitor: &mut crate::gc::RuntimeRootVisitor<'_>, + st: &crate::state::RuntimeState, + owner: usize, +) -> (usize, bool) { + use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; + let new_owner = rewrite_descriptor_owner(visitor, owner); + let mut relevant = false; + let accessor_keys = st + .descriptors + .accessor_keys_by_owner + .borrow() + .get(&owner) + .cloned() + .unwrap_or_default(); + if !accessor_keys.is_empty() { + let mut accessors = st.descriptors.accessor_descriptors.borrow_mut(); + for key in &accessor_keys { + if let Some(acc) = accessors.get_mut(&(owner, key.clone())) { + if acc.get != 0 { + visitor.visit_nanbox_u64_slot(&mut acc.get); + } + if acc.set != 0 { + visitor.visit_nanbox_u64_slot(&mut acc.set); + } + relevant |= bits_are_minor_relevant(acc.get) || bits_are_minor_relevant(acc.set); + } + } + if new_owner != owner { + for key in accessor_keys { + if let Some(acc) = accessors.remove(&(owner, key.clone())) { + accessors.insert((new_owner, key), acc); + } + } + } + } + if new_owner != owner { + let attr_keys = st + .descriptors + .attr_keys_by_owner + .borrow() + .get(&owner) + .cloned() + .unwrap_or_default(); + if !attr_keys.is_empty() { + let mut attrs = st.descriptors.property_descriptors.borrow_mut(); + for key in attr_keys { + if let Some(value) = attrs.remove(&(owner, key.clone())) { + attrs.insert((new_owner, key), value); + } + } + } + owner_index_transfer(&st.descriptors.attr_keys_by_owner, owner, new_owner); + owner_index_transfer(&st.descriptors.accessor_keys_by_owner, owner, new_owner); + } + relevant |= addr_is_minor_relevant(new_owner); + (new_owner, relevant) } /// The owner index (`attr_keys_by_owner` / `accessor_keys_by_owner`) exists diff --git a/crates/perry-runtime/src/object/descriptor_state/young.rs b/crates/perry-runtime/src/object/descriptor_state/young.rs new file mode 100644 index 0000000000..0133fc7549 --- /dev/null +++ b/crates/perry-runtime/src/object/descriptor_state/young.rs @@ -0,0 +1,85 @@ +//! The minor-scoped half of the string-keyed descriptor root scan. +//! +//! `scan_descriptor_roots_mut` walks every owner in `attr_keys_by_owner` and +//! `accessor_keys_by_owner`; on a minor that is ~23k owners to discover that +//! none of them points at the nursery. The young-entry log +//! (`gc/young_log.rs`) narrows a minor-scoped pass to the owners a minor can +//! act on, and this module holds that pass plus the re-derivation of the +//! relevant set that rule 2 checks it against under `debug_assertions`. + +use super::*; + +/// Every owner whose entry can matter to a minor, re-derived from the +/// authoritative tables: a non-old owner, or an accessor whose getter or +/// setter is non-old. +pub(super) fn relevant_descriptor_owners(st: &crate::state::RuntimeState) -> Vec { + use crate::gc::young_log::{addr_is_minor_relevant, bits_are_minor_relevant}; + let mut relevant = Vec::new(); + for &owner in st.descriptors.attr_keys_by_owner.borrow().keys() { + if addr_is_minor_relevant(owner) { + relevant.push(owner); + } + } + for &owner in st.descriptors.accessor_keys_by_owner.borrow().keys() { + if addr_is_minor_relevant(owner) { + relevant.push(owner); + } + } + for ((owner, _), acc) in st.descriptors.accessor_descriptors.borrow().iter() { + if bits_are_minor_relevant(acc.get) || bits_are_minor_relevant(acc.set) { + relevant.push(*owner); + } + } + relevant.sort_unstable(); + relevant.dedup(); + relevant +} + +/// The minor-scoped walk (#9754): only the young-logged owners, each visited +/// exactly as the full walk visits it — accessor get/set rooted in every +/// phase, owner re-keyed across both tables and both indexes in the rewrite +/// phase — and re-logged iff still relevant afterwards. +pub(super) fn scan_descriptor_roots_young( + visitor: &mut crate::gc::RuntimeRootVisitor<'_>, + st: &crate::state::RuntimeState, +) { + let table_len = st.descriptors.attr_keys_by_owner.borrow().len() as u64 + + st.descriptors.accessor_keys_by_owner.borrow().len() as u64; + #[cfg(debug_assertions)] + { + let relevant = relevant_descriptor_owners(st); + st.descriptors + .young_owners + .borrow() + .debug_assert_logged(DESCRIPTOR_YOUNG_LOG_NAME, &relevant); + } + let mut logged = 0u64; + let mut visited = 0u64; + let mut kept = st.descriptors.young_owners.borrow_mut().take_spare(); + loop { + let batch = st.descriptors.young_owners.borrow_mut().take_sorted(); + if batch.is_empty() { + break; + } + logged += batch.len() as u64; + for owner in batch { + visited += 1; + let (new_owner, relevant) = scan_descriptor_owner(visitor, st, owner); + if relevant { + kept.push(new_owner); + } + } + } + let kept_len = kept.len() as u64; + st.descriptors.young_owners.borrow_mut().extend(kept); + crate::gc::young_log::note_walk( + DESCRIPTOR_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: true, + logged, + visited, + kept: kept_len, + table_len, + }, + ); +} diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index 088f9d70d1..37247daf70 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -256,9 +256,10 @@ impl FieldLookupCaches { pub use accessors::js_object_get_field; pub(crate) use accessors::{ accessor_receiver_override_begin, accessor_receiver_override_end, - array_prototype_property_value, builtin_reflection_accessor_read, class_getter_this, - invoke_accessor_getter, invoke_accessor_setter, is_typed_array_prototype, - object_field_at_with_live, ordinary_object_prototype_property_value, own_data_field_by_name, + accessor_receiver_override_take, array_prototype_property_value, + builtin_reflection_accessor_read, class_getter_this, invoke_accessor_getter, + invoke_accessor_setter, is_typed_array_prototype, object_field_at_with_live, + ordinary_object_prototype_property_value, own_data_field_by_name, primitive_builtin_prototype_property, primitive_object_prototype_accessor, string_index_value, }; pub(crate) use class_object_props::class_object_prototype_value; diff --git a/crates/perry-runtime/src/object/field_get_set/accessors.rs b/crates/perry-runtime/src/object/field_get_set/accessors.rs index 8c477cdb50..fbc65b3cb4 100644 --- a/crates/perry-runtime/src/object/field_get_set/accessors.rs +++ b/crates/perry-runtime/src/object/field_get_set/accessors.rs @@ -408,6 +408,12 @@ pub(crate) fn accessor_receiver_override_begin(receiver: f64) -> Option { }) } +/// Consume the original receiver before entering user code through a Proxy, +/// just as invoke_accessor_getter does before entering a getter body. +pub(crate) fn accessor_receiver_override_take() -> Option { + ACCESSOR_RECEIVER_OVERRIDE.with(|c| c.take()) +} + pub(crate) fn accessor_receiver_override_end(prev: Option) { ACCESSOR_RECEIVER_OVERRIDE.with(|c| c.set(prev)); } diff --git a/crates/perry-runtime/src/object/field_get_set/has_property.rs b/crates/perry-runtime/src/object/field_get_set/has_property.rs index a81839c513..e107abafc7 100644 --- a/crates/perry-runtime/src/object/field_get_set/has_property.rs +++ b/crates/perry-runtime/src/object/field_get_set/has_property.rs @@ -874,14 +874,8 @@ pub extern "C" fn js_object_has_property(obj: f64, key: f64) -> f64 { // Issue #233: resolve a grow forwarding pointer so `index in arr` // / `arr.hasOwnProperty(i)` stay correct after `arr.length = N`. let arr = crate::array::clean_arr_ptr(obj_ptr as *const crate::array::ArrayHeader); - let length = (*arr).length; - // A Proxy installed as the array's `[[Prototype]]` - // (`Object.setPrototypeOf(arr, proxy)`) — `array_spec_has_index` - // only recognizes a *real array* custom prototype, so a Proxy - // hop is silently treated as absent. Recover it here so the - // idx/string-key misses below can fall back to the proxy's - // `[[HasProperty]]` instead of a bare `false` (ECMA-262 10.1.7.1 - // step 5). + // Named keys still need Proxy dispatch below. Indexed keys use + // array_spec_has_index, which owns the complete prototype walk. let proxy_proto = super::super::prototype_chain::object_static_prototype(obj_ptr as usize) .filter(|&b| (b >> 48) == 0x7FFD) @@ -908,38 +902,11 @@ pub extern "C" fn js_object_has_property(obj: f64, key: f64) -> f64 { None }; if let Some(idx) = idx { - let _ = length; - // Spec HasProperty: own (dense slot / sparse named prop / - // accessor descriptor) OR inherited — a custom array - // [[Prototype]], `Array.prototype[i]`, or an - // `Object.prototype` index (data or accessor; test262 - // sort/precise-comparefn-throws checks `'2' in array` - // against an Object.prototype accessor). - if crate::array::array_spec_has_index(arr, idx) { - return nanbox_true; - } - if crate::array::object_prototype_has_index_prop(idx) { - return nanbox_true; - } - if let Some(proxy) = proxy_proto { - let idx_str = idx.to_string(); - let key_ptr = crate::string::js_string_from_bytes( - idx_str.as_ptr(), - idx_str.len() as u32, - ); - let key_val = f64::from_bits( - crate::value::js_nanbox_string(key_ptr as i64).to_bits(), - ); - return if crate::value::js_is_truthy(crate::proxy::js_proxy_has( - proxy, key_val, - )) != 0 - { - nanbox_true - } else { - nanbox_false - }; - } - return nanbox_false; + return if crate::array::array_spec_has_index(arr, idx) { + nanbox_true + } else { + nanbox_false + }; } if key_val.is_any_string() { let key_str = crate::value::js_get_string_pointer_unified(key) @@ -952,17 +919,11 @@ pub extern "C" fn js_object_has_property(obj: f64, key: f64) -> f64 { return nanbox_true; } if let Some(idx) = super::super::canonical_array_index(key_name) { - // Same spec HasProperty protocol as the - // numeric-key arm above: own + inherited - // (custom array proto / Array.prototype / - // Object.prototype data-or-accessor index; - // test262 sort/precise-comparefn-throws does - // `'2' in array`). - if crate::array::array_spec_has_index(arr, idx) - || crate::array::object_prototype_has_index_prop(idx) - { - return nanbox_true; - } + return if crate::array::array_spec_has_index(arr, idx) { + nanbox_true + } else { + nanbox_false + }; } else if array_prototype_property_value(key_name, obj_ptr as usize) .is_some() { diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index e4c6020c66..5df83c2564 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -315,6 +315,21 @@ pub(crate) unsafe fn pic_prime_get(cache: *mut PicCache, token: i64, slot: i64) let c = &mut *cache; let prev_tok = c[0]; let prev_slot = c[1]; + // `PERRY_IC_DIAG`: the prime split. `prev_tok == token` means this site is + // being primed with the shape its MRU entry ALREADY held — the cache was + // written, the next read of that same shape came back here anyway, and we + // are about to write the identical value again. That is a priming or + // invalidation problem, not polymorphism, and it is a different fix from + // `prev_tok != token` (the receiver really did change shape). Read before + // the write below, because the write destroys the evidence. + if crate::hot_diag::ic_on() { + // Was `token` already sitting in a WAY? The MRU comparison alone cannot + // tell a site rotating k <= PIC_WAYS+1 shapes (the ways doing their job) + // from one whose cached answer the emitted gate never consulted. Read + // here, before the loop below evicts `token` from its way. + let in_ways = (0..PIC_WAYS).any(|w| c[PIC_WAY_BASE + w * 2] == token); + crate::hot_diag::ic_note_prime(cache as usize, prev_tok, token, c[PIC_WAY_STATE], in_ways); + } c[0] = token; c[1] = slot; // Megamorphic. A rotation wider than the ways hold never hits one, so the @@ -498,7 +513,22 @@ fn ic_diag_note( std::slice::from_raw_parts(crate::string::string_data(key), (*key).byte_len as usize) } }; - crate::hot_diag::ic_note(cache_slot as usize, bytes, reason); + // Key the site by the RESOLVED cache, not by the slot that points at it, so + // these rows merge with the ones `pic_prime_get` records (it only ever has + // the resolved cache). A site that has never primed has no cache yet; key + // it by the slot, which is stable and has no prime rows to merge with. + // SAFETY: `cache_slot` is the codegen-emitted per-site slot (or null on the + // earliest exits, which `pic_slot_peek` handles); peeking only reads the + // published pointer and never allocates. + let site = unsafe { + let cache = pic_slot_peek(cache_slot); + if cache.is_null() { + cache_slot as usize + } else { + cache as usize + } + }; + crate::hot_diag::ic_note(site, bytes, reason); } #[no_mangle] @@ -977,6 +1007,115 @@ pub extern "C" fn js_object_get_field_ic_miss( /// - `site_id`: the typed-feedback site id /// - `cache_slot`: the per-site [`PicCacheSlot`] (resolved and primed by /// `..._ic_miss`) +/// `PERRY_IC_OUTLINE_FASTPATH=0` sends every outlined read back to the miss +/// handler, so the same binary can be measured with and without the hit path +/// one environment variable apart. Measurement only — nothing in the runtime +/// branches on it for behaviour, and the two settings are observationally +/// identical. +#[inline] +fn outlined_mru_hit_enabled() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + crate::gc::env_default_on_from_value( + std::env::var("PERRY_IC_OUTLINE_FASTPATH").ok().as_deref(), + ) + }) +} + +/// The inline-cache HIT the full-outline path never had. +/// +/// # Why this exists +/// +/// `js_object_get_field_ic` (#5391 path 3) replaces the inline generic-get +/// diamond with one call in oversized modules. The diamond's monomorphic +/// fast-load was traded away with it, and nothing replaced it: the helper +/// observed feedback and then called `js_object_get_field_ic_miss` +/// **unconditionally, on every read**, which primed the site's cache and +/// returned. The cache was written by every read and consulted by nobody. +/// +/// That is not a small trade on a minified bundle, because the threshold that +/// turns full-outlining on (4,000 callables) is met by the *whole module*, so +/// EVERY generic property read in such a program takes it. Measured on the +/// compiled claude-code TUI, one 400-character reply: 2,663,424 entries to the +/// miss handler over 12,326 sites, of which 2,122,626 primed, and **95.2 % of +/// those primes wrote the token the site's MRU entry already held** +/// (`PERRY_IC_DIAG`'s prime split). The four hottest sites — `.done`, +/// `.ambiguousAsWide`, `.value`, `.segment`, ~195k reads each — each recorded +/// exactly ONE new-token prime and ~195k same-token primes, with +/// `PIC_WAY_STATE` still 0. Perfectly monomorphic sites, a cache holding the +/// right answer, and the full miss ladder walked every time. +/// +/// So this is not a new cache or a new policy: it is the *existing* per-site +/// cache being read on the path that writes it. +/// +/// # The guards are the emitted diamond's, one for one +/// +/// Receiver is a real heap pointer (`>= HANDLE_BAND_MAX`), a `GC_TYPE_OBJECT` +/// with `OBJ_FLAG_HAS_DESCRIPTORS` clear, its shape stamp is non-zero and +/// equal to the cached token, and the cached slot carries no +/// `IC_SLOT_OVERFLOW_BIT`. Those are exactly the predicates +/// `lower_generic_property_get` emits before `pic.hit`, evaluated in the same +/// order, and the raw header loads are the same ones it emits — the caller has +/// already established the pointer tag, which is what licenses them there and +/// here. A `TAG_HOLE` in the slot is a deleted field and misses, as it does +/// there. +/// +/// Word 2 (the Array-subclass named-prefix token) and the polymorphic ways are +/// deliberately NOT served here: they are 2.5 % of primes between them and +/// each needs its own proof. They keep falling through to the handler. +/// +/// # Safety +/// `obj_handle` is the receiver with the NaN-box tag already masked off, and +/// the caller has established that the tag was `POINTER`/`STRING`. `cache_slot` +/// is the codegen-emitted per-site slot or null. +#[inline] +unsafe fn pic_outlined_mru_hit( + obj_handle: *const ObjectHeader, + cache_slot: *mut PicCacheSlot, +) -> Option { + if !outlined_mru_hit_enabled() { + return None; + } + let addr = obj_handle as usize; + if !crate::value::addr_class::is_above_handle_band(addr) { + return None; + } + // The site has never primed: there is nothing to hit, and resolving the + // slot is the miss handler's job. + let cache = pic_slot_peek(cache_slot); + if cache.is_null() { + return None; + } + let header = &*((addr - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader); + if header.obj_type != crate::gc::GC_TYPE_OBJECT + || header._reserved & crate::gc::OBJ_FLAG_HAS_DESCRIPTORS != 0 + { + return None; + } + // `object_shape_stamp` answers 0 for a receiver whose `parent_class_id` is + // not a ShapeId, which is what keeps a keyless receiver out of an empty + // cache slot (#809). + let stamp = crate::object::shapes::object_shape_stamp(obj_handle); + if stamp == 0 { + return None; + } + let c = &*cache; + if c[0] != (stamp as u64 | crate::object::shapes::PIC_ID_TOKEN_BIT) as i64 { + return None; + } + let slot = c[1]; + if (slot as u64) & u64::from(crate::proxy::IC_SLOT_OVERFLOW_BIT) != 0 { + return None; + } + let field = *((obj_handle as *const u8) + .add(std::mem::size_of::() + slot as usize * 8) + as *const f64); + if field.to_bits() == crate::value::TAG_HOLE { + return None; + } + Some(field) +} + #[no_mangle] pub extern "C" fn js_object_get_field_ic( obj_bits: i64, @@ -1016,6 +1155,13 @@ pub extern "C" fn js_object_get_field_ic( // is primed for any future inline sites sharing this global). if (tag & 0xFFFD) == 0x7FFD { crate::typed_feedback::js_typed_feedback_observe_property_get(site_id, obj_handle, key); + // The monomorphic hit the emitted diamond does inline. Everything it + // declines still reaches the handler below, so this only ever removes + // work. See `pic_outlined_mru_hit`. + if let Some(value) = unsafe { pic_outlined_mru_hit(obj_handle, cache_slot) } { + crate::typed_feedback::js_typed_feedback_record_guard_pass(site_id); + return value; + } return js_object_get_field_ic_miss(obj_handle, key, cache_slot); } // Invalid (non-pointer) receiver. `undefined`/`null` throw a TypeError (#462 — diff --git a/crates/perry-runtime/src/object/mod.rs b/crates/perry-runtime/src/object/mod.rs index f598712f9b..c7b48fc87d 100644 --- a/crates/perry-runtime/src/object/mod.rs +++ b/crates/perry-runtime/src/object/mod.rs @@ -102,12 +102,26 @@ pub(crate) mod has_own_helpers; mod instanceof; mod live_slots; mod null_stub; +mod side_table_roots; pub(crate) use live_slots::set_object_live_slot_count; pub use live_slots::{ js_object_live_slot_count, object_live_slot_count, perry_object_header_abi_revision, }; pub use null_stub::{js_unresolved_default_call, js_unresolved_namespace_stub}; pub(crate) use null_stub::{NullObjectBytes, NULL_OBJECT_BYTES}; +#[cfg(test)] +pub(crate) use side_table_roots::test_transition_cache_insert; +pub(crate) use side_table_roots::{ + prune_dead_transition_cache_entries, prune_dead_transition_cache_entries_young, +}; +pub use side_table_roots::{ + scan_shape_cache_roots, scan_shape_cache_roots_mut, scan_transition_cache_roots, + scan_transition_cache_roots_mut, +}; +#[cfg(test)] +pub(crate) use side_table_roots::{ + test_seed_transition_cache_entry, test_transition_cache_occupancy, +}; pub(crate) mod iterator_prototypes; pub(crate) mod map_set_subclass; mod namespace_create; @@ -256,9 +270,10 @@ pub(crate) use descriptor_state::{ json_object_getter_value, mark_all_keys, object_has_descriptors, object_proto_may_intercept_key, owner_has_property_descriptors, owner_may_have_descriptor_entries, plain_data_write_may_intercept, - prune_dead_descriptor_owner_entries, reflect_getter_closure_bits, set_accessor_descriptor, - set_builtin_accessor_descriptor, set_builtin_property_attrs, set_property_attrs, - transfer_descriptor_owner, AccessorDescriptor, DescriptorTables, PropertyAttrs, + prune_dead_descriptor_owner_entries, prune_dead_descriptor_owner_entries_young, + reflect_getter_closure_bits, set_accessor_descriptor, set_builtin_accessor_descriptor, + set_builtin_property_attrs, set_property_attrs, transfer_descriptor_owner, AccessorDescriptor, + DescriptorTables, PropertyAttrs, }; pub(crate) use field_get_set::FieldLookupCaches; pub(crate) use field_get_set::{ @@ -647,6 +662,21 @@ fn shape_cache_get_with_id(shape_id: u32) -> (*mut ArrayHeader, u32) { .unwrap_or((std::ptr::null_mut(), 0)) } +/// Rule 1 of `gc/young_log.rs` for the shape cache: log `shape_id` BEFORE the +/// entry naming `keys_array` becomes findable. +/// +/// Every writer of the cache — the production `shape_cache_insert` and the +/// `#[cfg(test)]` seed seam — arms through this one function. A seam that +/// re-implements the predicate is the failure mode this exists to prevent: +/// the tests then validate an arming rule that is not the one that ships, and +/// deleting the production arm site stays green. +#[inline] +pub(super) fn arm_shape_cache_young(shape_id: u32, keys_array: *mut ArrayHeader) { + if crate::gc::young_log::addr_is_minor_relevant(keys_array as usize) { + SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().note(shape_id)); + } +} + /// Insert a keys_array into the cache. Updates the inline slot /// (evicting any prior entry there) and also writes to the overflow /// map so misses on the inline cache still find the value. @@ -676,6 +706,9 @@ fn shape_cache_insert(shape_id: u32, keys_array: *mut ArrayHeader) { }; let st = crate::state::state(); let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); + // #9754 rule 1: log the id BEFORE the entry is published when the keys + // array can matter to a minor. + arm_shape_cache_young(shape_id, keys_array); unsafe { // GC_STORE_AUDIT(ROOT): shape_inline_cache entries are scanned by scan_shape_cache_roots_mut. let entry = &mut (*st.object_hot.shape_inline_cache.get())[slot]; @@ -783,6 +816,29 @@ const TRANSITION_CACHE_SIZE: usize = 16384; /// against this value even when no Rust path consults it directly. #[allow(dead_code)] const TRANSITION_CACHE_MASK: usize = TRANSITION_CACHE_SIZE - 1; +crate::perry_thread_local! { + /// #9754: transition-cache slots whose `key_ptr` / `next_keys` may still be + /// acted on by a minor (see `gc/young_log.rs`); a minor-scoped + /// `scan_transition_cache_roots_mut` visits only these. + static TRANSITION_CACHE_YOUNG: RefCell> = + const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; + /// #9754: shape-cache ids (inline slot and overflow key alike) whose keys + /// array may still be acted on by a minor. + static SHAPE_CACHE_YOUNG: RefCell> = + const { RefCell::new(crate::gc::young_log::YoungLog::new()) }; +} + +const TRANSITION_CACHE_YOUNG_LOG_NAME: &str = "object.transition_cache"; +const SHAPE_CACHE_YOUNG_LOG_NAME: &str = "object.shape_cache"; + +/// Is a transition-cache entry still something a minor can act on? +#[inline] +fn transition_entry_is_minor_relevant(entry: &TransitionEntry) -> bool { + use crate::gc::young_log::addr_is_minor_relevant; + entry.next_keys != 0 + && (addr_is_minor_relevant(entry.next_keys) + || ((entry.slot_idx >> 24) == 0 && addr_is_minor_relevant(entry.key_ptr))) +} // Per-thread transition cache (`ObjectHotTables::transition_cache`). Was a // process-wide `static mut`, but with `perry/thread` user code allocating @@ -798,6 +854,7 @@ const TRANSITION_CACHE_MASK: usize = TRANSITION_CACHE_SIZE - 1; // thread-locals (confirmed on a real Series 7: shrinking OR boxing removes // the corruption). `vec!` builds directly on the heap (no 320KB stack // temporary). + #[inline] fn with_transition_cache( f: impl FnOnce(*mut [TransitionEntry; TRANSITION_CACHE_SIZE]) -> R, @@ -1016,6 +1073,28 @@ unsafe fn transition_cache_stamp_shape_shared(next_keys: usize) -> bool { true } +/// Rule 1 of `gc/young_log.rs` for the transition cache: log `slot` BEFORE the +/// entry becomes findable. +/// +/// `kid` is only an address when `len_marker == 0`; with a length marker set +/// it is a packed length, not a pointer, so classifying it would be a category +/// error. Both writers — `transition_cache_insert` and the `#[cfg(test)]` seed +/// seam — arm through here, so that distinction cannot be dropped in one and +/// kept in the other (it was: the seam classified `key_ptr` unconditionally). +#[inline] +pub(super) fn arm_transition_cache_young( + slot: usize, + next_keys: usize, + kid: usize, + len_marker: u32, +) { + if crate::gc::young_log::addr_is_minor_relevant(next_keys) + || (len_marker == 0 && crate::gc::young_log::addr_is_minor_relevant(kid)) + { + TRANSITION_CACHE_YOUNG.with(|log| log.borrow_mut().note(slot as u32)); + } +} + fn transition_cache_insert( array_tail_owner: *const ObjectHeader, prev_shape_id: u32, @@ -1044,6 +1123,9 @@ fn transition_cache_insert( } } } + // #9754 rule 1: log the slot BEFORE the entry is published when either + // address can matter to a minor. + arm_transition_cache_young(slot, next_keys, kid, len_marker); with_transition_cache(|t| unsafe { // GC_STORE_AUDIT(ROOT): TRANSITION_CACHE_GLOBAL entries are scanned by scan_transition_cache_roots_mut. let entry = &mut (*t)[slot]; @@ -1073,169 +1155,6 @@ fn transition_cache_insert( // stay lazy to avoid cloning every growing prefix. } -/// GC root scanner for the transition cache. Same contract as -/// `scan_shape_cache_roots` — without this the mark phase would free -/// cached target arrays that no live object currently holds directly, -/// and the next cache-hit store would dereference freed memory. -/// -/// #855: walk the static via `&raw const` + raw pointer indexing to -/// avoid the `static_mut_refs` lint (hard error in Rust 2024). The -/// cache is thread-local-by-discipline (perry user code is single- -/// threaded), so the unsafe deref is sound. -pub fn scan_transition_cache_roots(mark: &mut dyn FnMut(f64)) { - let mut visitor = crate::gc::RuntimeRootVisitor::for_copy(mark); - scan_transition_cache_roots_mut(&mut visitor); -} - -pub fn scan_transition_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - with_transition_cache(|table| unsafe { - for i in 0..TRANSITION_CACHE_SIZE { - let entry = &mut (*table)[i]; - if entry.next_keys != 0 { - let mut invalidate = false; - // Content-namespace ids (len marker != 0) are string BYTES, - // not addresses — the visitor must not rewrite them. - if (entry.slot_idx >> 24) == 0 { - invalidate |= visitor.visit_metadata_usize_slot(&mut entry.key_ptr); - } - // #6759 phase 3: `next_keys` is WEAK, not a strong root. - // - // `visit_usize_slot` MARKS. With 16384 slots this cache was - // therefore keeping up to 16384 keys arrays — and, through - // them, their shape descriptors — alive whether or not any live - // object still had that shape. That is a direct contributor to - // the shape table growing without bound between full - // collections (measured: 786k descriptors on a workload holding - // under 400 live objects). - // - // A transition entry is a pure cache: it answers "adding key k - // to shape S yields shape T". If nothing has shape T any more, - // the answer is worthless, so pinning T's keys array to keep it - // answerable is backwards. `key_ptr` was already weak for the - // same reason; this makes the pair consistent. - // - // Rewrite-only keeps a surviving target's address correct; - // `prune_dead_transition_cache_entries` drops the entry when the - // target did not survive. - visitor.visit_metadata_usize_slot(&mut entry.next_keys); - if invalidate { - *entry = TransitionEntry { - key_ptr: 0, - next_keys: 0, - prev_shape_id: 0, - target_shape_id: 0, - slot_idx: 0, - target_len: 0, - }; - } - } - } - }); - array_tail_transition::scan_roots_mut(visitor); -} - -/// #8192: death pruning for the transition cache. -/// -/// The interned `key_ptr` is metadata-only and therefore weak; `next_keys` is -/// a strong root. The predecessor and target ShapeIds are stable non-pointer -/// metadata, so moving collection neither rewrites nor invalidates them. -/// -/// The entry is a pure cache, so the repair is to drop it. `next_keys == 0` is -/// the empty-slot sentinel. -/// -/// `gc::dead_owner::DEAD_KEY_PRUNES` runs `prune_dead_shape_keys` before this -/// function. A predecessor whose keys edge died therefore has no descriptor -/// by the time we visit the cache. Both ShapeIds must still resolve: the -/// predecessor is weak, while the strongly rooted target keys normally keep -/// their descriptor live. Checking both here makes that target invariant a -/// release-mode post-GC proof without adding a hash-table lookup to every hot -/// transition stamp. -#[cold] -pub(crate) fn prune_dead_transition_cache_entries(is_dead_owner: &dyn Fn(usize) -> bool) { - with_transition_cache(|table| unsafe { - for i in 0..TRANSITION_CACHE_SIZE { - let entry = &mut (*table)[i]; - if entry.next_keys == 0 { - continue; - } - let dead = ((entry.slot_idx >> 24) == 0 - && entry.key_ptr != 0 - && is_dead_owner(entry.key_ptr)) - // #6759 phase 3: `next_keys` stopped being a strong root, so a - // dead target is now possible and must be reaped here — this is - // the half that makes weakening it safe. - || is_dead_owner(entry.next_keys) - || shapes::shape_descriptor_by_id(entry.prev_shape_id).is_none() - || (entry.target_shape_id != 0 - && shapes::shape_descriptor_by_id(entry.target_shape_id).is_none()); - if dead { - *entry = TransitionEntry { - key_ptr: 0, - next_keys: 0, - prev_shape_id: 0, - target_shape_id: 0, - slot_idx: 0, - target_len: 0, - }; - } - } - }); - array_tail_transition::prune_invalid_entries(); -} - -#[cfg(test)] -pub(crate) fn test_transition_cache_occupancy() -> usize { - with_transition_cache(|table| unsafe { - (0..TRANSITION_CACHE_SIZE) - .filter(|&i| (*table)[i].next_keys != 0) - .count() - }) -} - -#[cfg(test)] -pub(crate) fn test_seed_transition_cache_entry( - prev_shape_id: u32, - key_ptr: usize, - next_keys: usize, -) { - let slot = transition_cache_slot(prev_shape_id, key_ptr); - with_transition_cache(|table| unsafe { - (*table)[slot] = TransitionEntry { - key_ptr, - next_keys, - prev_shape_id, - target_shape_id: 0, - slot_idx: 0, - target_len: 1, - }; - }); -} - -/// GC root scanner: mark all cached shape keys arrays so they're not freed. -/// The inline cache + overflow map both hold the raw `*mut ArrayHeader` -/// pointers; without this scanner, GC would free those arrays, leaving -/// every object with that shape holding a dangling `keys_array` pointer. -pub fn scan_shape_cache_roots(mark: &mut dyn FnMut(f64)) { - let mut visitor = crate::gc::RuntimeRootVisitor::for_copy(mark); - scan_shape_cache_roots_mut(&mut visitor); -} - -pub fn scan_shape_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - let st = crate::state::state(); - { - let entries = unsafe { &mut *st.object_hot.shape_inline_cache.get() }; - for entry in entries.iter_mut() { - visitor.visit_raw_mut_ptr_slot(&mut entry.keys_array); - } - } - { - let mut cache = st.object_hot.shape_cache_overflow.borrow_mut(); - for (arr_ptr, _runtime_shape_id) in cache.values_mut() { - visitor.visit_raw_mut_ptr_slot(arr_ptr); - } - } -} - /// GC root scanner: mark all JSValues stored in OVERFLOW_FIELDS. /// OVERFLOW_FIELDS stores extra properties for objects that exceed their pre-allocated inline /// slot count. The u64 JSValue bits may contain NaN-boxed pointers to heap objects (strings, @@ -1403,10 +1322,19 @@ pub fn scan_object_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<' } } +/// Drive the PRODUCTION shape-cache writer from a test. Deliberately nothing +/// but a call: a seam with logic of its own can drift from the writer it +/// stands in for, which is exactly what let a deleted arm site stay green. +#[cfg(test)] +pub(crate) fn test_shape_cache_insert(shape_id: u32, keys_array: *mut ArrayHeader) { + shape_cache_insert(shape_id, keys_array); +} + #[cfg(test)] pub(crate) fn test_seed_shape_cache_root(shape_id: u32, keys_array: *mut ArrayHeader) { let st = crate::state::state(); let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); + arm_shape_cache_young(shape_id, keys_array); unsafe { // GC_STORE_AUDIT(ROOT): test seed mirrors shape_inline_cache roots scanned by scan_shape_cache_roots_mut. let entry = &mut (*st.object_hot.shape_inline_cache.get())[slot]; diff --git a/crates/perry-runtime/src/object/native_call_method.rs b/crates/perry-runtime/src/object/native_call_method.rs index 05d32b1a39..e5044e8eb5 100644 --- a/crates/perry-runtime/src/object/native_call_method.rs +++ b/crates/perry-runtime/src/object/native_call_method.rs @@ -652,7 +652,9 @@ pub unsafe extern "C-unwind" fn js_native_call_method_value( let key_jsval = JSValue::from_bits(key.to_bits()); let is_symbol_key = crate::symbol::js_is_symbol(key) != 0; - if is_symbol_key { + // Well-known symbol calls must use the current property value below; + // direct registry dispatch bypasses own/prototype replacements (#9788). + if is_symbol_key && !crate::symbol::is_well_known_symbol(crate::symbol::sym_key_from_f64(key)) { let sym_key = crate::symbol::sym_key_from_f64(key); if sym_key != 0 { let bits = object.to_bits(); @@ -886,7 +888,7 @@ pub unsafe extern "C-unwind" fn js_native_call_method_value( // (whose slot is already the receiver), effect's Tag-class symbol *statics* // (plain data values), and any closure that doesn't read `this` are all left // untouched — keeping the #1758/#36/#321 closure-proto-chain paths intact. - let field = if is_symbol_key && crate::symbol::own_symbol_property(object, key).is_none() { + let field = if is_symbol_key && !crate::symbol::has_own_symbol_property(object, key) { f64::from_bits(crate::closure::clone_closure_rebind_this( field.to_bits(), object, diff --git a/crates/perry-runtime/src/object/object_ops/define_property.rs b/crates/perry-runtime/src/object/object_ops/define_property.rs index 9511262f78..fc3d1e4bc6 100644 --- a/crates/perry-runtime/src/object/object_ops/define_property.rs +++ b/crates/perry-runtime/src/object/object_ops/define_property.rs @@ -1302,7 +1302,7 @@ pub extern "C" fn js_object_define_property( desc_ptr as *const ObjectHeader, value_key, ); - crate::symbol::js_object_set_symbol_property( + crate::symbol::define_symbol_data_property( obj_value, key_value, f64::from_bits(value_field.bits()), diff --git a/crates/perry-runtime/src/object/prototype_chain.rs b/crates/perry-runtime/src/object/prototype_chain.rs index ba3b1d3ca9..29a9ce8084 100644 --- a/crates/perry-runtime/src/object/prototype_chain.rs +++ b/crates/perry-runtime/src/object/prototype_chain.rs @@ -571,16 +571,9 @@ pub(crate) fn resolve_inherited_field_from_prototype( return None; } let key_val = f64::from_bits(crate::value::js_nanbox_string(key as i64).to_bits()); - let receiver = - f64::from_bits(crate::value::js_nanbox_pointer(obj_ptr as i64).to_bits()); - let scope = crate::gc::RuntimeHandleScope::new(); - let previous_this = super::js_implicit_this_set(receiver); - let previous_this_handle = scope.root_nanbox_f64(previous_this); - let v = crate::proxy::js_proxy_get(proto_val, key_val); - super::js_implicit_this_set(previous_this_handle.get_nanbox_f64()); - if v.to_bits() == crate::value::TAG_UNDEFINED { - return None; - } + let receiver = super::field_get_set::accessor_receiver_override_take() + .unwrap_or_else(|| crate::value::js_nanbox_pointer(obj_ptr as i64)); + let v = crate::proxy::proxy_get_with_receiver(proto_val, key_val, receiver); return Some(crate::value::JSValue::from_bits(v.to_bits())); } } diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 8d26b9c024..f61cf84098 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -256,11 +256,30 @@ struct ShapeTableInner { /// /// Single-word key, so `PtrHasher` (#8125). families: crate::fast_hash::PtrHashMap, + /// #9754: keys-array addresses a minor can act on — the families and + /// slot indices whose keys array is not (yet) old. A minor-scoped + /// `scan_shape_table_rekey_mut` visits only these; see `gc/young_log.rs`. + young_keys: crate::gc::young_log::YoungLog, } +const SHAPE_YOUNG_LOG_NAME: &str = "shapes.families+indices"; + impl ShapeTableInner { + /// Rule 1 of `gc/young_log.rs`: log a keys address BEFORE a family or a + /// slot index is published under it, when the keys array is not old. + /// Every family insert funnels through `family_push_back` / + /// `family_append_fresh` / `family_push_front`; the slot-index inserts + /// call this themselves. + #[inline] + fn note_young_keys(&mut self, keys: u64) { + if crate::gc::young_log::addr_is_minor_relevant(keys as usize) { + self.young_keys.note(keys); + } + } + #[inline] fn family_push_back(&mut self, keys: u64, id: u32) { + self.note_young_keys(keys); self.families.entry(keys).or_default().push_back(id); } @@ -270,11 +289,13 @@ impl ShapeTableInner { /// linear in the number of descriptors this keys array has ever had. #[inline] fn family_append_fresh(&mut self, keys: u64, id: u32) { + self.note_young_keys(keys); self.families.entry(keys).or_default().append_unchecked(id); } #[inline] fn family_push_front(&mut self, keys: u64, id: u32) { + self.note_young_keys(keys); self.families.entry(keys).or_default().push_front(id); } @@ -340,6 +361,7 @@ impl ShapeTable { indices: crate::fast_hash::new_ptr_hash_map(), by_facts: crate::fast_hash::new_ptr_hash_map(), families: crate::fast_hash::new_ptr_hash_map(), + young_keys: crate::gc::young_log::YoungLog::new(), }), } } @@ -1674,6 +1696,7 @@ pub(crate) unsafe fn shape_slot_lookup_verdict( if !build { return KeysIndexVerdict::Unindexed; } + inner.note_young_keys(keys_id as u64); inner.indices.entry(keys_id).or_insert(ShapeIndex { indexed_len: 0, slots: crate::fast_hash::new_ptr_hash_map(), @@ -1767,6 +1790,7 @@ pub(crate) fn shape_keys_grown(old_keys: usize, new_keys: *const ArrayHeader) { } let mut inner = crate::state::state().shapes.inner.borrow_mut(); if let Some(shape) = inner.indices.remove(&old_keys) { + inner.note_young_keys(new_id as u64); inner.indices.insert(new_id, shape); } } @@ -1860,6 +1884,34 @@ pub(crate) fn prune_dead_shape_keys(is_dead_owner: &dyn Fn(usize) -> bool) { } } +/// [`prune_dead_shape_keys`] for a MINOR (#9754): only a young keys array can +/// die, and a young keys address is always in the young log (noted at +/// insert, re-logged by every minor-scoped walk while it stays young), so the +/// log is the complete candidate set. +pub(crate) fn prune_dead_shape_keys_young(is_dead_owner: &dyn Fn(usize) -> bool) { + let table = &crate::state::state().shapes; + let mut inner = table.inner.borrow_mut(); + let candidates = inner.young_keys.take_sorted(); + let mut kept = Vec::with_capacity(candidates.len()); + for keys in candidates { + let addr = keys as usize; + if !is_dead_owner(addr) && !shape_keys_address_is_recycled(addr) { + kept.push(keys); + continue; + } + inner.indices.remove(&addr); + let ids: Vec = inner + .families + .get(&keys) + .map(|ids| ids.as_slice().to_vec()) + .unwrap_or_default(); + for id in ids { + remove_descriptor_indexed_under(&mut inner, id, keys); + } + } + inner.young_keys.extend(kept); +} + /// Metadata-only forwarding repair for the weak descriptor table and /// pointer-keyed slot indices. Mark/copy mode does not root anything; live /// object scans provide descriptor reachability, and post-copy rewrite follows @@ -1877,6 +1929,13 @@ pub(crate) fn scan_shape_table_rekey_mut(visitor: &mut crate::gc::RuntimeRootVis let table = &crate::state::state().shapes; let mut inner = table.inner.borrow_mut(); let rewrite_phase = visitor.is_metadata_rewrite_phase(); + // #9754: a minor-scoped pass visits only the young-logged keys addresses; + // the full walk below rebuilds the log from what it finds. + if visitor.young_scope() { + scan_shape_table_young(visitor, table, &mut inner, rewrite_phase); + return; + } + let table_len = (inner.families.len() + inner.indices.len()) as u64; let mut moved_families: Vec<(u64, u64)> = Vec::new(); let mut dead_descriptor_ids: Vec<(u32, u64)> = Vec::new(); // The shared slab view is scoped to the probe loop: retirement below @@ -1947,59 +2006,244 @@ pub(crate) fn scan_shape_table_rekey_mut(visitor: &mut crate::gc::RuntimeRootVis remove_descriptor_indexed_under(&mut inner, id, indexed); } for (old, new) in moved_families { - let Some(ids) = inner.families.remove(&old) else { - continue; - }; - if new == 0 { - continue; - } - for &id in ids.as_slice() { - let Some(record) = table.slab().get(id) else { - continue; - }; - // The accelerator was keyed with the OLD address; the other five - // facts never change under the collector. - if record.has(RECORD_FLAG_FACTS_INDEXED) { - inner.facts_remove(record.facts_key_with_keys(old), id); - inner.facts_push_back(record.facts_key_with_keys(new), id); + move_shape_family(table, &mut inner, old, new); + } + + if rewrite_phase && !inner.indices.is_empty() { + let moved: Vec<(usize, usize)> = inner + .indices + .keys() + .filter_map(|&keys_id| { + let mut addr = keys_id; + visitor.visit_metadata_usize_slot(&mut addr); + (addr != keys_id).then_some((keys_id, addr)) + }) + .collect(); + for (old, new) in moved { + if let Some(shape) = inner.indices.remove(&old) { + inner.indices.insert(new, shape); } - inner.family_push_back(new, id); + } + // Drop indices entries whose keys-array address was recycled: the + // forwarding record at the old address points to a DIFFERENT object + // (not a keys array), so `visit_metadata_usize_slot` either rekeyed + // it to the wrong address (caught above by the type mismatch on the + // new address) or returned false because the forwarding walk could + // not classify the address. Either way the keys array is dead; remove + // the stale entry so property lookups don't resolve the wrong shape. + let recycled: Vec = inner + .indices + .keys() + .filter(|&&keys_id| shape_keys_address_is_recycled(keys_id)) + .copied() + .collect(); + for old in recycled { + inner.indices.remove(&old); } } - if !rewrite_phase || inner.indices.is_empty() { + // A full walk is authoritative: rebuild the young log from the tables. + let kept = relevant_shape_keys(&inner); + let kept_len = kept.len() as u64; + let _ = inner.young_keys.take_sorted(); + inner.young_keys.extend(kept); + crate::gc::young_log::note_walk( + SHAPE_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: false, + logged: table_len, + visited: table_len, + kept: kept_len, + table_len, + }, + ); +} + +/// Re-index a family that the collector moved from `old` to `new` (`new == +/// 0`: every member retired, drop it). +fn move_shape_family(table: &ShapeTable, inner: &mut ShapeTableInner, old: u64, new: u64) { + let Some(ids) = inner.families.remove(&old) else { + return; + }; + if new == 0 { return; } - let moved: Vec<(usize, usize)> = inner - .indices - .keys() - .filter_map(|&keys_id| { - let mut addr = keys_id; - visitor.visit_metadata_usize_slot(&mut addr); - (addr != keys_id).then_some((keys_id, addr)) - }) - .collect(); - for (old, new) in moved { - if let Some(shape) = inner.indices.remove(&old) { - inner.indices.insert(new, shape); + for &id in ids.as_slice() { + let Some(record) = table.slab().get(id) else { + continue; + }; + // The accelerator was keyed with the OLD address; the other five + // facts never change under the collector. + if record.has(RECORD_FLAG_FACTS_INDEXED) { + inner.facts_remove(record.facts_key_with_keys(old), id); + inner.facts_push_back(record.facts_key_with_keys(new), id); } + inner.family_push_back(new, id); } - // Drop indices entries whose keys-array address was recycled: the - // forwarding record at the old address points to a DIFFERENT object - // (not a keys array), so `visit_metadata_usize_slot` either rekeyed - // it to the wrong address (caught above by the type mismatch on the - // new address) or returned false because the forwarding walk could - // not classify the address. Either way the keys array is dead; remove - // the stale entry so property lookups don't resolve the wrong shape. - let recycled: Vec = inner - .indices +} + +/// Every keys address a minor can act on, re-derived from the authoritative +/// tables (families and slot indices whose keys array is not old). +fn relevant_shape_keys(inner: &ShapeTableInner) -> Vec { + use crate::gc::young_log::addr_is_minor_relevant; + let mut relevant: Vec = inner + .families .keys() - .filter(|&&keys_id| shape_keys_address_is_recycled(keys_id)) .copied() + .filter(|&keys| keys != 0 && addr_is_minor_relevant(keys as usize)) .collect(); - for old in recycled { - inner.indices.remove(&old); + relevant.extend( + inner + .indices + .keys() + .copied() + .filter(|&keys| addr_is_minor_relevant(keys)) + .map(|keys| keys as u64), + ); + relevant.sort_unstable(); + relevant.dedup(); + relevant +} + +/// The minor-scoped walk (#9754): only the young-logged keys addresses, each +/// visited exactly as the full walk visits it — the family's carrier gate, +/// the record rewrite, the recycled-address retirement, the slot-index +/// re-key — and re-logged iff the keys array is still not old afterwards. +fn scan_shape_table_young( + visitor: &mut crate::gc::RuntimeRootVisitor<'_>, + table: &ShapeTable, + inner: &mut ShapeTableInner, + rewrite_phase: bool, +) { + let table_len = (inner.families.len() + inner.indices.len()) as u64; + #[cfg(debug_assertions)] + { + let relevant = relevant_shape_keys(inner); + inner + .young_keys + .debug_assert_logged(SHAPE_YOUNG_LOG_NAME, &relevant); + } + let mut logged = 0u64; + let mut visited = 0u64; + let mut kept = inner.young_keys.take_spare(); + loop { + let batch = inner.young_keys.take_sorted(); + if batch.is_empty() { + break; + } + logged += batch.len() as u64; + for keys in batch { + if keys == 0 { + continue; + } + visited += 1; + let (post, relevant) = + scan_shape_keys_address(visitor, table, inner, rewrite_phase, keys); + if relevant { + kept.push(post); + } + // The family moves in the MARK pass (a carrier's `visit_usize_slot` + // copies the keys array) while the slot index is re-keyed only in + // the REWRITE pass, so between the two the index still sits under + // the from-space address: keep that key logged as well. + if post != keys && inner.indices.contains_key(&(keys as usize)) { + kept.push(keys); + } + } } + let kept_len = kept.len() as u64; + inner.young_keys.extend(kept); + crate::gc::young_log::note_walk( + SHAPE_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: true, + logged, + visited, + kept: kept_len, + table_len, + }, + ); +} + +/// Visit one keys address — its family and its slot index — with the same +/// per-entry body as the full walk. Returns the post-visit address and +/// whether the keys array can still matter to a minor. +fn scan_shape_keys_address( + visitor: &mut crate::gc::RuntimeRootVisitor<'_>, + table: &ShapeTable, + inner: &mut ShapeTableInner, + rewrite_phase: bool, + indexed: u64, +) -> (u64, bool) { + let mut post = indexed; + let ids: Vec = inner + .families + .get(&indexed) + .map(|ids| ids.as_slice().to_vec()) + .unwrap_or_default(); + if !ids.is_empty() { + let slab = table.slab(); + let mut descriptor: Option = None; + for &id in &ids { + if let Some(lifted) = slab.lift(id) { + if lifted.old_carrier || lifted.cache_carrier { + descriptor = Some(lifted); + break; + } + descriptor.get_or_insert(lifted); + } + } + match descriptor { + None => { + // Every id retired under a stale address; the family is empty. + move_shape_family(table, inner, indexed, 0); + } + Some(descriptor) => { + let mut addr = indexed as usize; + let moved = if descriptor.old_carrier || descriptor.cache_carrier { + visitor.visit_usize_slot(&mut addr) + } else { + visitor.visit_metadata_usize_slot(&mut addr) + }; + if rewrite_phase && shape_keys_address_is_recycled(addr) { + for id in ids { + remove_descriptor_indexed_under(inner, id, indexed); + } + } else { + if moved { + for &id in &ids { + if let Some(record) = slab.record_ptr(id) { + // SAFETY: live slab record, single-threaded agent; + // the store is idempotent (see the full walk). + unsafe { (*record).keys = addr as u64 }; + } + } + } + if addr as u64 != indexed { + move_shape_family(table, inner, indexed, addr as u64); + post = addr as u64; + } + } + } + } + } + if rewrite_phase && inner.indices.contains_key(&(indexed as usize)) { + let mut addr = indexed as usize; + visitor.visit_metadata_usize_slot(&mut addr); + if addr != indexed as usize { + if let Some(shape) = inner.indices.remove(&(indexed as usize)) { + inner.indices.insert(addr, shape); + } + post = addr as u64; + } + if shape_keys_address_is_recycled(addr) { + inner.indices.remove(&addr); + } + } + ( + post, + crate::gc::young_log::addr_is_minor_relevant(post as usize), + ) } // #8112 sabotage switch. Suppressing the descriptor edge proves the fixture's diff --git a/crates/perry-runtime/src/object/shapes_slot_list.rs b/crates/perry-runtime/src/object/shapes_slot_list.rs index b93ddd35e3..e6f8b7b1e9 100644 --- a/crates/perry-runtime/src/object/shapes_slot_list.rs +++ b/crates/perry-runtime/src/object/shapes_slot_list.rs @@ -170,6 +170,7 @@ pub(crate) fn shape_index_migrate_after_delete( !list.is_empty() }); index.indexed_len = old_key_count - 1; + inner.note_young_keys(new_keys_id as u64); inner.indices.insert(new_keys_id, index); true } diff --git a/crates/perry-runtime/src/object/shapes_test_support.rs b/crates/perry-runtime/src/object/shapes_test_support.rs index 311bb982e5..29977c46eb 100644 --- a/crates/perry-runtime/src/object/shapes_test_support.rs +++ b/crates/perry-runtime/src/object/shapes_test_support.rs @@ -83,6 +83,7 @@ pub(crate) fn test_clear_shape_table() { inner.indices.clear(); inner.by_facts.clear(); inner.families.clear(); + inner.young_keys.clear(); // SAFETY: test-only reset with no slab reference held. unsafe { table.slab_mut().clear() }; drop(inner); @@ -157,3 +158,78 @@ pub(crate) fn test_seed_shape_entry(keys_id: usize) { pub(crate) fn test_shape_id_for_keys(keys_id: usize) -> Option { test_shape_ids_for_keys(keys_id).first().copied() } + +/// Number of indexed slots recorded for `keys_id`, or 0 when the address has +/// no `indices` entry. Used by the `shapes.indices` arming tests. +#[cfg(test)] +pub(crate) fn test_shape_index_len(keys_id: usize) -> u32 { + let inner = crate::state::state().shapes.inner.borrow(); + inner + .indices + .get(&keys_id) + .map(|ix| ix.indexed_len) + .unwrap_or(0) +} + +/// A process-global shape id no descriptor in this agent has claimed, for the +/// `install_external_shape_id` path. +#[cfg(test)] +pub(crate) fn test_unused_external_shape_id() -> u32 { + let table = &crate::state::state().shapes; + let mut id = super::SHAPE_ID_END - 1; + while table.slab().record_ptr(id).is_some() { + id -= 1; + } + id +} + +/// Build the slot index for `keys` through the PRODUCTION path +/// (`shape_slot_lookup` with `build = true`), which is the `indices` arm site. +/// Nothing but a call, so it cannot drift from the writer it stands in for. +/// +/// # Safety +/// `keys` must be a live keys array of `key_count` dense string slots. +#[cfg(test)] +pub(crate) unsafe fn test_build_slot_index( + keys: *const super::ArrayHeader, + probe: &[u8], + key_count: u32, +) { + let h = crate::object::keys_lookup::key_bytes_hash(probe.as_ptr(), probe.len()); + let _ = super::shape_slot_lookup(keys, probe, h, key_count, true); +} + +/// `shapes_slot_list::shape_index_migrate_after_delete`, reachable from the +/// `gc::tests` suites (the module is private to `shapes`). +#[cfg(test)] +pub(crate) fn test_shape_index_migrate_after_delete( + old_keys_id: usize, + new_keys_id: usize, + removed_slot: u32, + old_key_count: u32, + old_keys_shared: bool, +) -> bool { + super::shapes_slot_list::shape_index_migrate_after_delete( + old_keys_id, + new_keys_id, + removed_slot, + old_key_count, + old_keys_shared, + ) +} + +/// `shapes_slot_list::install_external_shape_id`, same reason. +#[cfg(test)] +pub(crate) fn test_install_external_shape_id( + id: u32, + keys: *const super::ArrayHeader, + logical_key_count: u32, + live_inline_slot_count: u32, +) -> bool { + super::shapes_slot_list::install_external_shape_id( + id, + keys, + logical_key_count, + live_inline_slot_count, + ) +} diff --git a/crates/perry-runtime/src/object/side_table_roots.rs b/crates/perry-runtime/src/object/side_table_roots.rs new file mode 100644 index 0000000000..fdaa3262c3 --- /dev/null +++ b/crates/perry-runtime/src/object/side_table_roots.rs @@ -0,0 +1,368 @@ +//! GC root scanning and dead-owner pruning for the two hot shape caches. +//! +//! The transition cache (`(predecessor ShapeId, key) -> successor`) and the +//! shape cache (`static shape id -> canonical keys array`) both hold raw +//! `*mut ArrayHeader` pointers that no live object need hold directly, so the +//! collector has to visit them or the next cache hit dereferences freed +//! memory. Both carry a young-entry log (`gc/young_log.rs`), so each scanner +//! and each prune exists in a full-walk and a minor-scoped form; keeping the +//! four pairs together — and out of `object/mod.rs`, which is at the +//! file-size gate — is what this module is for. +//! +//! Everything here reaches its tables through `super`: the caches, their +//! logs and their per-entry helpers stay private to `object`. + +use super::*; + +/// GC root scanner for the transition cache. Same contract as +/// `scan_shape_cache_roots` — without this the mark phase would free +/// cached target arrays that no live object currently holds directly, +/// and the next cache-hit store would dereference freed memory. +/// +/// #855: walk the static via `&raw const` + raw pointer indexing to +/// avoid the `static_mut_refs` lint (hard error in Rust 2024). The +/// cache is thread-local-by-discipline (perry user code is single- +/// threaded), so the unsafe deref is sound. +pub fn scan_transition_cache_roots(mark: &mut dyn FnMut(f64)) { + let mut visitor = crate::gc::RuntimeRootVisitor::for_copy(mark); + scan_transition_cache_roots_mut(&mut visitor); +} + +pub fn scan_transition_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + // #9754: a minor-scoped pass visits only the young-logged slots; a full + // pass walks the table and rebuilds the log. Both share + // `scan_transition_cache_slot`. + if visitor.young_scope() { + let mut logged = 0u64; + let mut visited = 0u64; + let mut kept = TRANSITION_CACHE_YOUNG.with(|log| log.borrow_mut().take_spare()); + #[cfg(debug_assertions)] + with_transition_cache(|table| unsafe { + let relevant: Vec = (0..TRANSITION_CACHE_SIZE) + .filter(|&i| transition_entry_is_minor_relevant(&(*table)[i])) + .map(|i| i as u32) + .collect(); + TRANSITION_CACHE_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(TRANSITION_CACHE_YOUNG_LOG_NAME, &relevant) + }); + }); + let batch = TRANSITION_CACHE_YOUNG.with(|log| log.borrow_mut().take_sorted()); + logged += batch.len() as u64; + with_transition_cache(|table| unsafe { + for slot in batch { + visited += 1; + if scan_transition_cache_slot(visitor, table, slot as usize) { + kept.push(slot); + } + } + }); + let kept_len = kept.len() as u64; + TRANSITION_CACHE_YOUNG.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + TRANSITION_CACHE_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: true, + logged, + visited, + kept: kept_len, + table_len: TRANSITION_CACHE_SIZE as u64, + }, + ); + array_tail_transition::scan_roots_mut(visitor); + return; + } + let _ = TRANSITION_CACHE_YOUNG.with(|log| log.borrow_mut().take_sorted()); + let mut kept = Vec::new(); + with_transition_cache(|table| unsafe { + for i in 0..TRANSITION_CACHE_SIZE { + if scan_transition_cache_slot(visitor, table, i) { + kept.push(i as u32); + } + } + }); + let kept_len = kept.len() as u64; + TRANSITION_CACHE_YOUNG.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + TRANSITION_CACHE_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: false, + logged: TRANSITION_CACHE_SIZE as u64, + visited: TRANSITION_CACHE_SIZE as u64, + kept: kept_len, + table_len: TRANSITION_CACHE_SIZE as u64, + }, + ); + array_tail_transition::scan_roots_mut(visitor); +} + +/// Visit one transition-cache slot. Returns whether the entry can still +/// matter to a minor afterwards. +/// +/// # Safety +/// `table` must be this thread's transition cache. +unsafe fn scan_transition_cache_slot( + visitor: &mut crate::gc::RuntimeRootVisitor<'_>, + table: *mut [TransitionEntry; TRANSITION_CACHE_SIZE], + i: usize, +) -> bool { + let entry = &mut (*table)[i]; + if entry.next_keys == 0 { + return false; + } + let mut invalidate = false; + // Content-namespace ids (len marker != 0) are string BYTES, + // not addresses — the visitor must not rewrite them. + if (entry.slot_idx >> 24) == 0 { + invalidate |= visitor.visit_metadata_usize_slot(&mut entry.key_ptr); + } + // #6759 phase 3: `next_keys` is WEAK, not a strong root. + // + // `visit_usize_slot` MARKS. With 16384 slots this cache was + // therefore keeping up to 16384 keys arrays — and, through + // them, their shape descriptors — alive whether or not any live + // object still had that shape. That is a direct contributor to + // the shape table growing without bound between full + // collections (measured: 786k descriptors on a workload holding + // under 400 live objects). + // + // A transition entry is a pure cache: it answers "adding key k + // to shape S yields shape T". If nothing has shape T any more, + // the answer is worthless, so pinning T's keys array to keep it + // answerable is backwards. `key_ptr` was already weak for the + // same reason; this makes the pair consistent. + // + // Rewrite-only keeps a surviving target's address correct; + // `prune_dead_transition_cache_entries` drops the entry when the + // target did not survive. + visitor.visit_metadata_usize_slot(&mut entry.next_keys); + if invalidate { + *entry = TransitionEntry { + key_ptr: 0, + next_keys: 0, + prev_shape_id: 0, + target_shape_id: 0, + slot_idx: 0, + target_len: 0, + }; + return false; + } + transition_entry_is_minor_relevant(entry) +} + +/// [`prune_dead_transition_cache_entries`] for a MINOR (#9754): only a slot +/// in the young log can name a young — hence possibly dead — address. +#[cold] +pub(crate) fn prune_dead_transition_cache_entries_young(is_dead_owner: &dyn Fn(usize) -> bool) { + let candidates = TRANSITION_CACHE_YOUNG.with(|log| log.borrow_mut().take_sorted()); + let mut kept = Vec::with_capacity(candidates.len()); + with_transition_cache(|table| unsafe { + for slot in candidates { + let entry = &mut (*table)[slot as usize]; + if entry.next_keys == 0 { + continue; + } + if transition_entry_is_dead(entry, is_dead_owner) { + *entry = TransitionEntry { + key_ptr: 0, + next_keys: 0, + prev_shape_id: 0, + target_shape_id: 0, + slot_idx: 0, + target_len: 0, + }; + } else { + kept.push(slot); + } + } + }); + TRANSITION_CACHE_YOUNG.with(|log| log.borrow_mut().extend(kept)); + array_tail_transition::prune_invalid_entries(); +} + +/// The death test of `prune_dead_transition_cache_entries`, shared with the +/// young-only variant. +fn transition_entry_is_dead( + entry: &TransitionEntry, + is_dead_owner: &dyn Fn(usize) -> bool, +) -> bool { + ((entry.slot_idx >> 24) == 0 && entry.key_ptr != 0 && is_dead_owner(entry.key_ptr)) + // #6759 phase 3: `next_keys` stopped being a strong root, so a + // dead target is now possible and must be reaped here — this is + // the half that makes weakening it safe. + || is_dead_owner(entry.next_keys) + || shapes::shape_descriptor_by_id(entry.prev_shape_id).is_none() + || (entry.target_shape_id != 0 + && shapes::shape_descriptor_by_id(entry.target_shape_id).is_none()) +} + +/// #8192: death pruning for the transition cache. +/// +/// The interned `key_ptr` is metadata-only and therefore weak; `next_keys` is +/// a strong root. The predecessor and target ShapeIds are stable non-pointer +/// metadata, so moving collection neither rewrites nor invalidates them. +/// +/// The entry is a pure cache, so the repair is to drop it. `next_keys == 0` is +/// the empty-slot sentinel. +/// +/// `gc::dead_owner::DEAD_KEY_PRUNES` runs `prune_dead_shape_keys` before this +/// function. A predecessor whose keys edge died therefore has no descriptor +/// by the time we visit the cache. Both ShapeIds must still resolve: the +/// predecessor is weak, while the strongly rooted target keys normally keep +/// their descriptor live. Checking both here makes that target invariant a +/// release-mode post-GC proof without adding a hash-table lookup to every hot +/// transition stamp. +#[cold] +pub(crate) fn prune_dead_transition_cache_entries(is_dead_owner: &dyn Fn(usize) -> bool) { + with_transition_cache(|table| unsafe { + for i in 0..TRANSITION_CACHE_SIZE { + let entry = &mut (*table)[i]; + if entry.next_keys == 0 { + continue; + } + if transition_entry_is_dead(entry, is_dead_owner) { + *entry = TransitionEntry { + key_ptr: 0, + next_keys: 0, + prev_shape_id: 0, + target_shape_id: 0, + slot_idx: 0, + target_len: 0, + }; + } + } + }); + array_tail_transition::prune_invalid_entries(); +} + +#[cfg(test)] +pub(crate) fn test_transition_cache_occupancy() -> usize { + with_transition_cache(|table| unsafe { + (0..TRANSITION_CACHE_SIZE) + .filter(|&i| (*table)[i].next_keys != 0) + .count() + }) +} + +/// Drive the PRODUCTION transition-cache writer from a test. Nothing but a +/// call, so it cannot drift from the writer it stands in for. +#[cfg(test)] +pub(crate) fn test_transition_cache_insert( + prev_shape_id: u32, + interned_key: *const crate::StringHeader, + next_keys: usize, + slot_idx: u32, + target_shape_id: u32, +) { + super::transition_cache_insert( + std::ptr::null(), + prev_shape_id, + interned_key, + next_keys, + slot_idx, + target_shape_id, + ); +} + +#[cfg(test)] +pub(crate) fn test_seed_transition_cache_entry( + prev_shape_id: u32, + key_ptr: usize, + next_keys: usize, +) { + let slot = transition_cache_slot(prev_shape_id, key_ptr); + // The seed writes `slot_idx: 0`, i.e. no length marker, so `key_ptr` is an + // address here — which is the `len_marker == 0` arm of the shared predicate. + arm_transition_cache_young(slot, next_keys, key_ptr, 0); + with_transition_cache(|table| unsafe { + (*table)[slot] = TransitionEntry { + key_ptr, + next_keys, + prev_shape_id, + target_shape_id: 0, + slot_idx: 0, + target_len: 1, + }; + }); +} + +/// GC root scanner: mark all cached shape keys arrays so they're not freed. +/// The inline cache + overflow map both hold the raw `*mut ArrayHeader` +/// pointers; without this scanner, GC would free those arrays, leaving +/// every object with that shape holding a dangling `keys_array` pointer. +pub fn scan_shape_cache_roots(mark: &mut dyn FnMut(f64)) { + let mut visitor = crate::gc::RuntimeRootVisitor::for_copy(mark); + scan_shape_cache_roots_mut(&mut visitor); +} + +pub fn scan_shape_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + use crate::gc::young_log::addr_is_minor_relevant; + let st = crate::state::state(); + // The inline array is 256 fixed slots: always walked. The overflow map + // holds every shape id ever cached; #9754: a minor-scoped pass visits + // only the young-logged ids there, a full pass rebuilds the log. + let entries = unsafe { &mut *st.object_hot.shape_inline_cache.get() }; + for entry in entries.iter_mut() { + visitor.visit_raw_mut_ptr_slot(&mut entry.keys_array); + } + let mut cache = st.object_hot.shape_cache_overflow.borrow_mut(); + let table_len = cache.len() as u64; + if visitor.young_scope() { + #[cfg(debug_assertions)] + { + let relevant: Vec = cache + .iter() + .filter(|(_, (arr_ptr, _))| addr_is_minor_relevant(*arr_ptr as usize)) + .map(|(&id, _)| id) + .collect(); + SHAPE_CACHE_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(SHAPE_CACHE_YOUNG_LOG_NAME, &relevant) + }); + } + let batch = SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().take_sorted()); + let logged = batch.len() as u64; + let mut kept = SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().take_spare()); + for id in batch { + if let Some((arr_ptr, _)) = cache.get_mut(&id) { + visitor.visit_raw_mut_ptr_slot(arr_ptr); + if addr_is_minor_relevant(*arr_ptr as usize) { + kept.push(id); + } + } + } + let kept_len = kept.len() as u64; + SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + SHAPE_CACHE_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: true, + logged, + visited: logged, + kept: kept_len, + table_len, + }, + ); + return; + } + let _ = SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().take_sorted()); + let mut kept = Vec::new(); + for (&id, (arr_ptr, _runtime_shape_id)) in cache.iter_mut() { + visitor.visit_raw_mut_ptr_slot(arr_ptr); + if addr_is_minor_relevant(*arr_ptr as usize) { + kept.push(id); + } + } + let kept_len = kept.len() as u64; + SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + SHAPE_CACHE_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: false, + logged: table_len, + visited: table_len, + kept: kept_len, + table_len, + }, + ); +} diff --git a/crates/perry-runtime/src/object/test_root_accessors.rs b/crates/perry-runtime/src/object/test_root_accessors.rs index 3b76daf0a0..094138aa55 100644 --- a/crates/perry-runtime/src/object/test_root_accessors.rs +++ b/crates/perry-runtime/src/object/test_root_accessors.rs @@ -22,6 +22,11 @@ pub(crate) fn test_shape_cache_root(shape_id: u32) -> (usize, usize) { #[cfg(test)] pub(crate) fn test_seed_transition_cache_root(next_keys: usize) { + // Slot 0, no key and no length marker: the `len_marker == 0` arm of the + // shared predicate with `kid == 0`, which classifies as not-relevant. + // Arming through the shared helper is what keeps every writer of this + // table on one rule (this seam used to carry a third variant of it). + super::arm_transition_cache_young(0, next_keys, 0, 0); with_transition_cache(|t| unsafe { // GC_STORE_AUDIT(ROOT): test seed mirrors TRANSITION_CACHE_GLOBAL roots scanned by scan_transition_cache_roots_mut. let entry = &mut (*t)[0]; @@ -34,6 +39,17 @@ pub(crate) fn test_seed_transition_cache_root(next_keys: usize) { }); } +/// `test_seed_transition_cache_root` under a predecessor ShapeId that resolves, +/// so `prune_dead_transition_cache_entries` (which retires an entry whose +/// predecessor has no descriptor) keeps the entry across a collection. +#[cfg(test)] +pub(crate) fn test_seed_transition_cache_root_for_shape(prev_shape_id: u32, next_keys: usize) { + test_seed_transition_cache_root(next_keys); + with_transition_cache(|t| unsafe { + (*t)[0].prev_shape_id = prev_shape_id; + }); +} + #[cfg(test)] pub(crate) fn test_transition_cache_root() -> usize { with_transition_cache(|t| unsafe { (*t)[0].next_keys }) @@ -41,6 +57,8 @@ pub(crate) fn test_transition_cache_root() -> usize { #[cfg(test)] pub(crate) fn test_clear_transition_cache_root() { + super::TRANSITION_CACHE_YOUNG.with(|log| log.borrow_mut().clear()); + super::SHAPE_CACHE_YOUNG.with(|log| log.borrow_mut().clear()); with_transition_cache(|t| unsafe { for i in 0..TRANSITION_CACHE_SIZE { // GC_STORE_AUDIT(ROOT): test clear writes non-pointer sentinels into scanned TRANSITION_CACHE_GLOBAL roots. diff --git a/crates/perry-runtime/src/proxy.rs b/crates/perry-runtime/src/proxy.rs index 66cc23fc2b..849c0c32d4 100644 --- a/crates/perry-runtime/src/proxy.rs +++ b/crates/perry-runtime/src/proxy.rs @@ -25,6 +25,9 @@ use crate::closure::{js_closure_call0, js_closure_call1, js_closure_call2, js_cl mod apply_construct; pub use apply_construct::{call_proxy_value_with_this, js_proxy_apply, js_proxy_construct}; pub(crate) use apply_construct::{is_callable_function, is_constructor_function}; +mod get; +pub use get::js_proxy_get; +pub(crate) use get::proxy_get_with_receiver; mod has_delete; pub(crate) use has_delete::reflect_ordinary_delete_property_key; pub use has_delete::{js_proxy_delete, js_proxy_has}; @@ -912,109 +915,6 @@ fn call_with_this_and_args(f: f64, this_arg: f64, args: &[f64]) -> f64 { result } -/// Detect the runtime's "null object" sentinel returned by -/// `js_native_call_method` when a method lookup falls off the end. -/// `proxy[key]` — if handler.get exists, call it with (target, key); -/// otherwise fetch the field from the target directly via the generic path. -#[no_mangle] -pub extern "C" fn js_proxy_get(proxy_boxed: f64, key: f64) -> f64 { - let _proxy_pin = pin_proxy_for_native_call(proxy_boxed); - let id = match lookup(proxy_boxed) { - Some(id) => id, - None => return f64::from_bits(TAG_UNDEFINED), - }; - // `[[Get]] ( P, Receiver )` receives an already-computed property key P, but - // codegen calls this helper with the raw index value for a computed read on - // a statically-known proxy (`proxy[10]` lowers to - // `js_proxy_get(proxy, 10.0)`). Apply `ToPropertyKey` so a numeric index is - // seen by the trap as the canonical string key (`10` -> `"10"`) and the - // forward-to-target path below stringifies consistently. Symbols and - // strings pass through unchanged. Without this the get trap received a raw - // number and key-equality checks (`key === "10"`) silently failed (test262 - // Proxy/get/trap-is-{null,undefined}-target-is-proxy `proxy[10]`). A key - // that is already a string (the overwhelmingly common `proxy.foo` case) or - // a symbol is left untouched, so this only pays `ToPropertyKey` for the - // numeric / object-index forms. - let key = { - let tag = key.to_bits() & 0xFFFF_0000_0000_0000; - let is_string_key = - tag == crate::value::STRING_TAG || tag == crate::value::SHORT_STRING_TAG; - if is_string_key || unsafe { crate::symbol::js_is_symbol(key) } != 0 { - key - } else { - unsafe { crate::object::js_to_property_key(key) } - } - }; - let (target, handler, revoked) = PROXIES.with(|p| { - p.borrow() - .get(id as usize) - .and_then(|o| o.as_ref()) - .map(|e| (e.target, e.handler, e.revoked)) - .unwrap_or(( - f64::from_bits(TAG_UNDEFINED), - f64::from_bits(TAG_UNDEFINED), - false, - )) - }); - if revoked { - return revoked_return(); - } - let trap = handler_trap(handler, "get"); - if is_callable(trap) { - let scope = crate::gc::RuntimeHandleScope::new(); - let target_h = scope.root_nanbox_f64(target); - let key_h = scope.root_nanbox_f64(key); - let result = call_trap( - handler, - trap, - &[ - target_h.get_nanbox_f64(), - key_h.get_nanbox_f64(), - proxy_boxed, - ], - ); - let result_h = scope.root_nanbox_f64(result); - invariants::enforce_get_invariant( - target_h.get_nanbox_f64(), - key_h.get_nanbox_f64(), - result_h.get_nanbox_f64(), - ); - return result_h.get_nanbox_f64(); - } - // No get trap — forward to the target's `[[Get]]`. A proxy target must - // recurse through proxy dispatch rather than `target_get`, which would deref - // the fake pointer. - if lookup(target).is_some() { - return js_proxy_get(target, key); - } - // `p.apply` / `p.call` / `p.bind` VALUE reads on a callable-wrapping - // proxy resolve to Function.prototype's methods with the PROXY as the - // receiver — reify a bound method so a later invocation dispatches - // `js_native_call_method(proxy, "call", …)` and routes through the - // proxy's [[Call]] (apply trap). Reading off the target instead would - // bypass the trap. (Test262 proxy-toString reads `.apply` as a value; - // Function.prototype.toString on the reified method is the - // NativeFunction form.) - if crate::object::value_is_callable(target) { - if let Some(name) = key_to_rust_string(key) { - let method: Option<&'static [u8]> = match name.as_str() { - "apply" => Some(b"apply"), - "call" => Some(b"call"), - "bind" => Some(b"bind"), - _ => None, - }; - if let Some(m) = method { - // Only when the target has no OWN override of the slot. - let t_ptr = extract_pointer(target.to_bits()) as usize; - if !crate::closure::closure_has_own_dynamic_prop(t_ptr, &name) { - return unsafe { crate::closure::reify_function_method_value(proxy_boxed, m) }; - } - } - } - } - target_get(target, key) -} - /// Resolve the ultimate target when a Proxy wraps a class constructor. Used /// by method-call dispatch to bind a static method's visible `this` to the /// Proxy receiver while retaining the target class as its lexical owner. @@ -1165,18 +1065,6 @@ fn target_get_property_key(target: f64, property_key: f64) -> f64 { crate::object::js_object_get_field_by_name_f64(obj_ptr, key_ptr) } -fn target_get(target: f64, key: f64) -> f64 { - let scope = crate::gc::RuntimeHandleScope::new(); - let target_handle = scope.root_nanbox_f64(target); - let key_handle = scope.root_nanbox_f64(key); - let property_key_handle = scope - .root_nanbox_f64(unsafe { crate::object::js_to_property_key(key_handle.get_nanbox_f64()) }); - target_get_property_key( - target_handle.get_nanbox_f64(), - property_key_handle.get_nanbox_f64(), - ) -} - /// `Reflect.set` with an explicit receiver: OrdinarySet(target, P, V, /// receiver), boolean result NaN-boxed. pub(crate) fn reflect_ordinary_set_with_receiver( diff --git a/crates/perry-runtime/src/proxy/get.rs b/crates/perry-runtime/src/proxy/get.rs new file mode 100644 index 0000000000..08c1ab2e3c --- /dev/null +++ b/crates/perry-runtime/src/proxy/get.rs @@ -0,0 +1,126 @@ +//! Proxy [[Get]] carries the original Receiver through traps and target hops. +use super::*; + +/// `proxy[key]` uses the proxy itself as Receiver. Prototype and Reflect reads +/// use the explicit-receiver entry below. +#[no_mangle] +pub extern "C" fn js_proxy_get(proxy_boxed: f64, key: f64) -> f64 { + proxy_get_with_receiver(proxy_boxed, key, proxy_boxed) +} + +pub(crate) fn proxy_get_with_receiver(proxy_boxed: f64, key: f64, receiver: f64) -> f64 { + let _proxy_pin = pin_proxy_for_native_call(proxy_boxed); + let scope = crate::gc::RuntimeHandleScope::new(); + let receiver = scope.root_nanbox_f64(receiver); + let id = match lookup(proxy_boxed) { + Some(id) => id, + None => return f64::from_bits(TAG_UNDEFINED), + }; + // `[[Get]] ( P, Receiver )` receives an already-computed property key P, but + // codegen calls this helper with the raw index value for a computed read on + // a statically-known proxy (`proxy[10]` lowers to + // `js_proxy_get(proxy, 10.0)`). Apply `ToPropertyKey` so a numeric index is + // seen by the trap as the canonical string key (`10` -> `"10"`) and the + // forward-to-target path below stringifies consistently. Symbols and + // strings pass through unchanged. Without this the get trap received a raw + // number and key-equality checks (`key === "10"`) silently failed (test262 + // Proxy/get/trap-is-{null,undefined}-target-is-proxy `proxy[10]`). A key + // that is already a string (the overwhelmingly common `proxy.foo` case) or + // a symbol is left untouched, so this only pays `ToPropertyKey` for the + // numeric / object-index forms. + let key = { + let tag = key.to_bits() & 0xFFFF_0000_0000_0000; + let is_string_key = + tag == crate::value::STRING_TAG || tag == crate::value::SHORT_STRING_TAG; + if is_string_key || unsafe { crate::symbol::js_is_symbol(key) } != 0 { + key + } else { + unsafe { crate::object::js_to_property_key(key) } + } + }; + let (target, handler, revoked) = PROXIES.with(|p| { + p.borrow() + .get(id as usize) + .and_then(|o| o.as_ref()) + .map(|e| (e.target, e.handler, e.revoked)) + .unwrap_or(( + f64::from_bits(TAG_UNDEFINED), + f64::from_bits(TAG_UNDEFINED), + false, + )) + }); + if revoked { + return revoked_return(); + } + // Looking up handler.get can itself run a getter and move the heap. + let target_h = scope.root_nanbox_f64(target); + let handler_h = scope.root_nanbox_f64(handler); + let key_h = scope.root_nanbox_f64(key); + let trap = handler_trap(handler_h.get_nanbox_f64(), "get"); + if is_callable(trap) { + let result = call_trap( + handler_h.get_nanbox_f64(), + trap, + &[ + target_h.get_nanbox_f64(), + key_h.get_nanbox_f64(), + receiver.get_nanbox_f64(), + ], + ); + let result_h = scope.root_nanbox_f64(result); + invariants::enforce_get_invariant( + target_h.get_nanbox_f64(), + key_h.get_nanbox_f64(), + result_h.get_nanbox_f64(), + ); + return result_h.get_nanbox_f64(); + } + // No get trap — forward to the target's `[[Get]]`. A proxy target must + // recurse through proxy dispatch rather than ordinary target dispatch, which would deref + // the fake pointer. + let target = target_h.get_nanbox_f64(); + let key = key_h.get_nanbox_f64(); + if lookup(target).is_some() { + return proxy_get_with_receiver(target, key, receiver.get_nanbox_f64()); + } + // `p.apply` / `p.call` / `p.bind` VALUE reads on a callable-wrapping + // proxy resolve to Function.prototype's methods with the PROXY as the + // receiver — reify a bound method so a later invocation dispatches + // `js_native_call_method(proxy, "call", …)` and routes through the + // proxy's [[Call]] (apply trap). Reading off the target instead would + // bypass the trap. (Test262 proxy-toString reads `.apply` as a value; + // Function.prototype.toString on the reified method is the + // NativeFunction form.) + if crate::object::value_is_callable(target) { + if let Some(name) = key_to_rust_string(key) { + let method: Option<&'static [u8]> = match name.as_str() { + "apply" => Some(b"apply"), + "call" => Some(b"call"), + "bind" => Some(b"bind"), + _ => None, + }; + if let Some(m) = method { + // Only when the target has no OWN override of the slot. + let t_ptr = extract_pointer(target.to_bits()) as usize; + if !crate::closure::closure_has_own_dynamic_prop(t_ptr, &name) { + return unsafe { crate::closure::reify_function_method_value(proxy_boxed, m) }; + } + } + } + } + // Ordinary target getters and further prototype hops must keep Receiver. + // Clear/restore the override around the operation; getters and Proxy hops + // consume it before entering user code so nested reads bind independently. + let previous_this = scope.root_nanbox_f64(crate::object::js_implicit_this_set( + receiver.get_nanbox_f64(), + )); + let previous_override = + crate::object::accessor_receiver_override_begin(receiver.get_nanbox_f64()) + .map(|value| scope.root_nanbox_f64(value)); + let result = target_get_property_key(target_h.get_nanbox_f64(), key_h.get_nanbox_f64()); + crate::object::accessor_receiver_override_end( + previous_override.map(|value| value.get_nanbox_f64()), + ); + crate::object::js_implicit_this_set(previous_this.get_nanbox_f64()); + result +} diff --git a/crates/perry-runtime/src/proxy/reflect.rs b/crates/perry-runtime/src/proxy/reflect.rs index b653b195f4..cecf2299cd 100644 --- a/crates/perry-runtime/src/proxy/reflect.rs +++ b/crates/perry-runtime/src/proxy/reflect.rs @@ -10,9 +10,7 @@ use super::{ /// /// - throws `TypeError` for a non-object target, /// - uses `receiver` as the `this` binding for accessor getters, -/// - dispatches proxy `get` traps (forwarding `(target, key)` to the existing -/// proxy path; the three-argument trap receiver is out of scope - Perry's -/// proxy traps are two-argument). +/// - dispatches proxy `get` traps with `(target, key, receiver)`. /// /// `receiver` is the optional third argument; codegen passes `target` when the /// call site omits it (matching the spec default), and `undefined` is treated @@ -30,9 +28,6 @@ pub extern "C" fn js_reflect_get(target: f64, key: f64, receiver: f64) -> f64 { .root_nanbox_f64(unsafe { crate::object::js_to_property_key(key_handle.get_nanbox_f64()) }); let target = target_handle.get_nanbox_f64(); let property_key = property_key_handle.get_nanbox_f64(); - if lookup(target).is_some() { - return js_proxy_get(target, property_key); - } // Default receiver to target when undefined. let receiver = receiver_handle.get_nanbox_f64(); let recv = if receiver.to_bits() == TAG_UNDEFINED { @@ -40,6 +35,9 @@ pub extern "C" fn js_reflect_get(target: f64, key: f64, receiver: f64) -> f64 { } else { receiver }; + if lookup(target).is_some() { + return super::proxy_get_with_receiver(target, property_key, recv); + } // #2766: if `key` resolves to an accessor *getter* on `target`, rebind its // `this` to the receiver and invoke it - object-literal getters capture // `this` in a reserved closure slot (not `IMPLICIT_THIS`), so plain diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 714a4f81c6..b6787e3c51 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -6,6 +6,11 @@ #[cfg(feature = "regex-engine")] use regex::Regex; use std::cell::RefCell; +// Every use of `HashMap` in this file is inside a `#[cfg(feature = "regex-engine")]` +// block, so an unconditional import is an unused-import error under the +// `warnings` job's `-D warnings` when `perry`'s own binaries pull the runtime +// in without that feature. +#[cfg(feature = "regex-engine")] use std::collections::HashMap; use std::ptr; use std::sync::Arc; @@ -37,6 +42,8 @@ mod exec_array; #[cfg(feature = "regex-engine")] mod flags; #[cfg(feature = "regex-engine")] +mod global_guards; +#[cfg(feature = "regex-engine")] mod global_scan; #[cfg(feature = "regex-engine")] mod grammar; @@ -70,6 +77,8 @@ use exec_array::{ #[cfg(feature = "regex-engine")] use flags::validate_and_canonicalize_flags; #[cfg(feature = "regex-engine")] +use global_guards::{ensure_replace_all_regex_global, throw_match_all_non_global_regex}; +#[cfg(feature = "regex-engine")] use grammar::{ collapse_redos_guard_quantifiers, has_invalid_repeated_quantifier, has_unicode_forbidden_legacy_escape, has_unicode_forbidden_pattern, js_regex_to_rust, @@ -457,6 +466,16 @@ pub(crate) fn build_fancy_regex(pattern: &str) -> Result bool { if !fancy_ok { return false; } - Regex::new(r"[^\s\S]").unwrap() + Regex::new(NEVER_MATCH_PATTERN).unwrap() } }; if crate::hot_diag::regex_on() { @@ -587,7 +606,7 @@ fn get_or_compile_regex(pattern: &str, flags: &str) -> Arc { } // Both engines rejected it (validation normally throws before this // point) — keep the historical behavior: cache + return never-match. - let arc = Arc::new(Regex::new(r"[^\s\S]").unwrap()); + let arc = Arc::new(Regex::new(NEVER_MATCH_PATTERN).unwrap()); evict_regex_cache_if_full(&mut cache); cache.insert((pattern.to_string(), flags.to_string()), arc.clone()); arc @@ -767,31 +786,6 @@ pub(super) fn js_string_from_str(s: &str) -> *mut StringHeader { } #[cfg(feature = "regex-engine")] -fn throw_replace_all_non_global_regex() -> ! { - let message = b"String.prototype.replaceAll called with a non-global RegExp argument"; - let msg = crate::string::js_string_from_bytes(message.as_ptr(), message.len() as u32); - let err = crate::error::js_typeerror_new(msg); - crate::exception::js_throw(crate::value::js_nanbox_pointer(err as i64)) -} - -#[cfg(feature = "regex-engine")] -fn throw_match_all_non_global_regex() -> ! { - let message = b"String.prototype.matchAll called with a non-global RegExp argument"; - let msg = crate::string::js_string_from_bytes(message.as_ptr(), message.len() as u32); - let err = crate::error::js_typeerror_new(msg); - crate::exception::js_throw(crate::value::js_nanbox_pointer(err as i64)) -} - -#[cfg(feature = "regex-engine")] -#[inline] -fn ensure_replace_all_regex_global(re: *const RegExpHeader) { - unsafe { - if !(*re).global { - throw_replace_all_non_global_regex(); - } - } -} - /// Throw a `SyntaxError` with the given message and never return. #[cfg(feature = "regex-engine")] pub(super) fn throw_regexp_syntax_error(message: &str) -> ! { diff --git a/crates/perry-runtime/src/regex/global_guards.rs b/crates/perry-runtime/src/regex/global_guards.rs new file mode 100644 index 0000000000..76bb7eab84 --- /dev/null +++ b/crates/perry-runtime/src/regex/global_guards.rs @@ -0,0 +1,30 @@ +//! `replaceAll` / `matchAll` non-global receiver guards. +//! +//! Split out of `regex.rs` to keep that file under the 2000-line size gate. + +use super::RegExpHeader; + +pub(super) fn throw_replace_all_non_global_regex() -> ! { + let message = b"String.prototype.replaceAll called with a non-global RegExp argument"; + let msg = crate::string::js_string_from_bytes(message.as_ptr(), message.len() as u32); + let err = crate::error::js_typeerror_new(msg); + crate::exception::js_throw(crate::value::js_nanbox_pointer(err as i64)) +} + +#[cfg(feature = "regex-engine")] +pub(super) fn throw_match_all_non_global_regex() -> ! { + let message = b"String.prototype.matchAll called with a non-global RegExp argument"; + let msg = crate::string::js_string_from_bytes(message.as_ptr(), message.len() as u32); + let err = crate::error::js_typeerror_new(msg); + crate::exception::js_throw(crate::value::js_nanbox_pointer(err as i64)) +} + +#[cfg(feature = "regex-engine")] +#[inline] +pub(super) fn ensure_replace_all_regex_global(re: *const RegExpHeader) { + unsafe { + if !(*re).global { + throw_replace_all_non_global_regex(); + } + } +} diff --git a/crates/perry-runtime/src/regex/lazy.rs b/crates/perry-runtime/src/regex/lazy.rs index 438d8eca4b..236c7a80f4 100644 --- a/crates/perry-runtime/src/regex/lazy.rs +++ b/crates/perry-runtime/src/regex/lazy.rs @@ -255,6 +255,58 @@ fn build_and_install_programs(re: *const RegExpHeader) { .get(&(pattern.to_string(), flags.to_string())) .cloned() }); + // ── Repair before publishing ────────────────────────────────────────── + // + // A built header is treated as AUTHORITATIVE — `lookup_fancy_regex` / + // `lookup_repeat_matcher` read a null slot beside a non-null `regex_ptr` + // as "this pattern has no such program" — and `install_programs` below + // memoizes the triple against the pattern text, so whatever is assembled + // here becomes the answer for every later construction of the same + // literal. It therefore has to be complete, and the probes above cannot + // guarantee that on their own: the three caches are capped independently + // and each `clear()`s wholesale, while + // `compile_and_cache_regex_checked` returns early whenever `REGEX_CACHE` + // already holds the pattern — so it never re-runs the fancy or + // repeat-matcher build for a pattern whose `REGEX_CACHE` entry survived a + // clear of one of the others. + // + // Both gaps are silent WRONG ANSWERS, not slowdowns: a lookbehind literal + // whose fancy program is missing matches nothing at all, and a + // quantified-capture literal whose repeat matcher is missing reports the + // linear engine's capture assignment instead of ECMA-262's. Re-derive + // what is missing. Each check costs nothing when the caches are coherent, + // which is the normal case. + let mut fancy_arc = fancy_arc; + if fancy_arc.is_none() && std_arc.as_str() == super::NEVER_MATCH_PATTERN { + // The standard program matches nothing, so this pattern is only + // usable through the fancy engine — and it is not there. + let flag_prefixed = flag_prefixed_pattern(&pattern, &flags); + if let Ok(fre) = super::build_fancy_regex(&flag_prefixed) { + let arc = Arc::new(fre); + FANCY_CACHE.with(|fc| { + let mut fc = fc.borrow_mut(); + evict_regex_cache_if_full(&mut fc); + fc.insert((pattern.to_string(), flags.to_string()), arc.clone()); + }); + fancy_arc = Some(arc); + } + } + let mut repeat_arc = repeat_arc; + if repeat_arc.is_none() { + // `compile` is a byte scan that returns `None` immediately unless a + // capture group sits under a quantifier (or inside a negative + // lookaround), so this is free for the patterns that do not need it. + if let Some(matcher) = super::repeat_matcher::compile(&pattern, &flags) { + let arc = Arc::new(matcher); + REPEAT_MATCHER_CACHE.with(|cache| { + let mut cache = cache.borrow_mut(); + evict_regex_cache_if_full(&mut cache); + cache.insert((pattern.to_string(), flags.to_string()), arc.clone()); + }); + repeat_arc = Some(arc); + } + } + // Remember the built programs against the pattern text, so the next // construction of the same literal is born built (`js_regexp_new`). super::site_cache::install_programs( diff --git a/crates/perry-runtime/src/regex/tests.rs b/crates/perry-runtime/src/regex/tests.rs index e41c2abc61..d82dea78e9 100644 --- a/crates/perry-runtime/src/regex/tests.rs +++ b/crates/perry-runtime/src/regex/tests.rs @@ -1757,3 +1757,76 @@ fn global_test_advances_and_resets_last_index() { assert_eq!(js_regexp_get_last_index(repeat), 5.0); assert_eq!(js_regexp_test(repeat, r), 0); } + +/// A capacity event in one compiled-program cache must not leave a pattern +/// whose real program lives in ANOTHER of them permanently non-matching. +/// +/// `compile_and_cache_regex_checked` returns early when `REGEX_CACHE` already +/// holds the pattern, so it never re-runs the fancy build; for a lookbehind +/// pattern that `REGEX_CACHE` entry is the never-match placeholder and the +/// real program is the one in `FANCY_CACHE`. Clear `FANCY_CACHE` on its own — +/// which is exactly what its independent 512-entry overflow used to do — and +/// `get_or_compile_regex` hands back a program that matches nothing while +/// nothing rebuilds the fallback. Since `lookup_fancy_regex` now treats a +/// built header as authoritative and `site_cache::install_programs` memoizes +/// the triple against the pattern text, that is not one bad header: every +/// later construction of the same literal is born with it. +/// +/// The fix is that `lazy::build_and_install_programs` REPAIRS the header +/// before publishing it and before memoizing the triple: a standard program +/// that is the never-match placeholder with no fancy program beside it means +/// the fancy program is missing, so it is rebuilt. (Clearing the three caches +/// as a group was built first and dropped: it closes the route into the bad +/// state but cannot repair a header already in it, so this test still failed.) +#[test] +fn a_single_program_cache_clear_cannot_disarm_a_lookbehind_literal() { + let _lock = crate::gc::global_side_table_test_lock(); + let source = "(?<=foo)bar"; + let scope = crate::gc::RuntimeHandleScope::new(); + site_cache::test_reset(); + + let build = || { + let pattern = scope.root_string_ptr(make_string(source)); + let flags = scope.root_string_ptr(make_string("")); + pattern.with_mut_ptr::(|pattern| { + flags.with_mut_ptr::(|flags| js_regexp_new(pattern, flags)) + }) + }; + let subject = scope.root_string_ptr(make_string("foobar")); + + let warm = build(); + assert_eq!( + subject.with_const_ptr::(|s| js_regexp_test(warm, s)), + 1, + "a lookbehind pattern must match through the fancy fallback" + ); + + // The state a `FANCY_CACHE` overflow produces: its programs are gone, the + // never-match placeholder for this pattern survives in `REGEX_CACHE`. + FANCY_CACHE.with(|fc| fc.borrow_mut().clear()); + assert!( + REGEX_CACHE.with(|c| c + .borrow() + .contains_key(&(source.to_string(), String::new()))), + "the placeholder must survive, or this test exercises nothing" + ); + // A fresh literal site, so the construction cache cannot answer from the + // programs the first header built. + site_cache::test_reset(); + + let cold = build(); + unsafe { + lazy::ensure_regex_compiled(cold); + assert!( + !(*cold).fancy_ptr.is_null(), + "a built header must carry every program its pattern needs — a null \ + fancy_ptr here is memoized by site_cache::install_programs and makes \ + the breakage permanent for this literal" + ); + } + assert_eq!( + subject.with_const_ptr::(|s| js_regexp_test(cold, s)), + 1, + "the literal must still match after an unrelated cache reached capacity" + ); +} diff --git a/crates/perry-runtime/src/symbol/get.rs b/crates/perry-runtime/src/symbol/get.rs index a4a7d78148..d481930448 100644 --- a/crates/perry-runtime/src/symbol/get.rs +++ b/crates/perry-runtime/src/symbol/get.rs @@ -25,6 +25,41 @@ fn well_known_symbol_method_name(sym_key: usize) -> Option<&'static str> { None } +/// An iterator dispatch alias is a fallback for the method declared in source. +/// Check writes on the intervening prototypes first, but stop at the nearest +/// declaration so a subclass method still shadows a replaced base method. +unsafe fn class_iterator_prototype_override( + receiver: f64, + sym: f64, + mut class_id: u32, + method_owner: u32, +) -> Option { + for _ in 0..32 { + let declared = crate::object::class_decl_prototype_object(class_id); + let dynamic = crate::object::class_prototype_object(class_id); + for proto in [declared, dynamic] { + if proto.is_null() { + continue; + } + let proto_value = crate::value::js_nanbox_pointer(proto as i64); + if let Some(acc) = accessors::symbol_accessor_property(proto_value, sym) { + return Some(accessors::invoke_symbol_accessor_getter(acc.get, receiver)); + } + if let Some(value) = own_symbol_property(proto_value, sym) { + return Some(value); + } + } + if class_id == method_owner { + break; + } + match crate::object::get_parent_class_id(class_id) { + Some(parent) if parent != 0 && parent != class_id => class_id = parent, + _ => break, + } + } + None +} + /// Does `obj` carry an OWN symbol-keyed property under `sym`, **without /// invoking** an accessor for it? /// @@ -812,7 +847,14 @@ pub unsafe extern "C" fn js_object_get_symbol_property(obj_f64: f64, sym_f64: f6 // on `method_owner_class_id` first: `js_class_method_bind` // otherwise mints a bound closure for a non-existent method. if let Some(method_name) = well_known_symbol_method_name(sym_key) { - if crate::object::method_owner_class_id(class_id, method_name).is_some() { + if let Some(owner) = + crate::object::method_owner_class_id(class_id, method_name) + { + if let Some(value) = + class_iterator_prototype_override(obj_f64, sym_f64, class_id, owner) + { + return value; + } return crate::object::js_class_method_bind( obj_f64, method_name.as_ptr(), diff --git a/crates/perry-runtime/src/tls_hot.rs b/crates/perry-runtime/src/tls_hot.rs index cffdc9cbc5..217176590e 100644 --- a/crates/perry-runtime/src/tls_hot.rs +++ b/crates/perry-runtime/src/tls_hot.rs @@ -564,11 +564,16 @@ pub(crate) fn hot() -> &'static HotTls { /// Claimed once, on the first thread that resolves the declaration, and stable /// for the life of the process — so every thread finds the same declaration at /// the same index in its own cache. -pub struct SlotId(std::sync::atomic::AtomicU32); +pub struct SlotId(std::sync::atomic::AtomicU32, &'static str); impl SlotId { pub const fn new() -> Self { - Self(std::sync::atomic::AtomicU32::new(SLOT_UNASSIGNED)) + Self::named("") + } + + #[doc(hidden)] + pub const fn named(name: &'static str) -> Self { + Self(std::sync::atomic::AtomicU32::new(SLOT_UNASSIGNED), name) } /// The claimed index, or a sentinel `>= HOT_SLOT_CAPACITY`. @@ -592,20 +597,12 @@ impl SlotId { fn claim(&self) -> u32 { use std::sync::atomic::Ordering; maybe_install_stats_hook(); - let mut next = match CLAIM_LOCK.lock() { - Ok(next) => next, - Err(poisoned) => poisoned.into_inner(), - }; let current = self.0.load(Ordering::Relaxed); if current != SLOT_UNASSIGNED { return current; } - let idx = if (*next as usize) < HOT_SLOT_CAPACITY { - let idx = *next; - *next += 1; - idx - } else { - SLOT_OVERFLOW + let idx = unsafe { + js_tls_hot_claim_slot(self.1.as_ptr(), self.1.len(), self as *const Self as usize) }; self.0.store(idx, Ordering::Relaxed); idx @@ -618,21 +615,53 @@ impl Default for SlotId { } } -/// The next index [`SlotId::claim`] will hand out. Also the count of -/// declarations claimed so far, which is what -/// [`claimed_slots`] reports and what the capacity test asserts against. -static CLAIM_LOCK: std::sync::Mutex = std::sync::Mutex::new(0); +/// One slot per logical declaration across provider images. The C entry point +/// below owns this registry even when the runtime's Rust crate hashes differ. +/// Provider images must come from the same source/ABI, as for HotTls itself. +static CLAIM_LOCK: std::sync::Mutex, u32>> = + std::sync::Mutex::new(std::collections::BTreeMap::new()); -/// How many declarations have claimed a slot in this process. +/// The provider images share HotTls, so slot identities must be shared too. +/// Use a C entry point for preemption even when separate runtime builds have +/// different Rust crate hashes. Names identify declarations, never TLS values. /// -/// Instrumentation for the capacity assertion: overflow is silent by design -/// (the declaration keeps working, slowly), so something has to be able to see -/// how close the process is to the ceiling. -pub fn claimed_slots() -> u32 { - match CLAIM_LOCK.lock() { - Ok(next) => *next, - Err(poisoned) => *poisoned.into_inner(), +/// # Safety +/// For a nonempty name, `name` must point to `len` readable bytes. A name must +/// identify the same thread-local declaration (and value type) in every image. +/// For an anonymous declaration, `anonymous` must be its unique static address. +#[no_mangle] +#[inline(never)] // Calls must remain interposable across provider images. +pub unsafe extern "C" fn js_tls_hot_claim_slot( + name: *const u8, + len: usize, + anonymous: usize, +) -> u32 { + let key = if len == 0 { + format!("anonymous:{anonymous}").into_bytes() + } else { + std::slice::from_raw_parts(name, len).to_vec() + }; + let mut slots = CLAIM_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(&idx) = slots.get(&key) { + return idx; + } + if slots.len() >= HOT_SLOT_CAPACITY { + return SLOT_OVERFLOW; } + let idx = slots.len() as u32; + slots.insert(key, idx); + idx +} + +#[no_mangle] +#[inline(never)] +pub extern "C" fn js_tls_hot_claimed_slots() -> u32 { + CLAIM_LOCK.lock().unwrap_or_else(|p| p.into_inner()).len() as u32 +} + +/// How many declarations have claimed a slot in this process. +pub fn claimed_slots() -> u32 { + js_tls_hot_claimed_slots() } /// How many slots *this thread* has populated. @@ -866,7 +895,8 @@ impl HotKey { self.slot.raw() } - /// `value` is the address of this thread's `T`, published by this key. + /// `value` is this thread's `T`, published by this declaration in one of + /// the compatible provider images sharing the cache. /// /// # Safety /// `value` must have come from this key's slot or from its own `resolve`. @@ -881,19 +911,26 @@ impl HotKey { unsafe { &*(value as *const T) } } - /// Resolve through the real `thread_local!`, claim this declaration's slot - /// if it has none yet, and publish the address for this thread. + /// Claim the shared declaration slot, reuse any published storage, or + /// resolve through the real `thread_local!` and publish it for this thread. #[cold] #[inline(never)] fn resolve_and_cache(&'static self) -> Result<*mut u8, std::thread::AccessError> { - // Resolve first, and outside the claim lock: initialising the value can - // run arbitrary runtime code, including other `perry_thread_local!` - // first touches. - let value = (self.resolve)()?; + // Claim before resolving storage: another provider can already have + // published this declaration in the shared cache. Do not construct or + // overwrite a second copy. The claim lock is released before any TLS + // initializer runs, so nested first touches remain safe. let mut idx = self.slot.raw(); if idx == SLOT_UNASSIGNED { idx = self.slot.claim(); } + if (idx as usize) < HOT_SLOT_CAPACITY { + let cached = hot().slot(idx); + if !cached.is_null() { + return Ok(cached); + } + } + let value = (self.resolve)()?; if (idx as usize) < HOT_SLOT_CAPACITY { // Arm before publishing: after this store any thread-teardown of // the value un-publishes the slot it is about to invalidate. @@ -965,7 +1002,12 @@ macro_rules! __perry_thread_local_one { ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)+) => { $(#[$attr])* $vis static $name: $crate::tls_hot::HotKey<$t> = { - static SLOT: $crate::tls_hot::SlotId = $crate::tls_hot::SlotId::new(); + // Module/name alone collide for function-local declarations. + // Avoid file!(): Cargo can use relative vs absolute source paths + // for the same crate in workspace and standalone provider builds. + static SLOT: $crate::tls_hot::SlotId = $crate::tls_hot::SlotId::named(concat!( + module_path!(), "::", stringify!($name), "@", line!(), ":", column!() + )); // `GUARD` is 1 exactly when `$t` has drop glue, so the guard — // and with it the thread-local's destructor — exists exactly when // a cached address could otherwise outlive the value. @@ -997,6 +1039,9 @@ macro_rules! __perry_thread_local_storage { }; } +#[cfg(test)] +mod provider_tests; + #[cfg(test)] mod tests { /// Every cached address must equal the address of the `thread_local!` it diff --git a/crates/perry-runtime/src/tls_hot/provider_tests.rs b/crates/perry-runtime/src/tls_hot/provider_tests.rs new file mode 100644 index 0000000000..eea3fd83f9 --- /dev/null +++ b/crates/perry-runtime/src/tls_hot/provider_tests.rs @@ -0,0 +1,90 @@ +//! Provider declaration identity and shared-storage regressions (#9791). + +/// Model two separately compiled provider copies of one declaration. +/// Merely allocating noncolliding indices is insufficient: both handles +/// must use the same storage and only one initializer/destructor may run. +#[test] +fn provider_copies_share_storage_without_initializing_a_second_value() { + use std::sync::atomic::{AtomicUsize, Ordering}; + static INITIALIZED: AtomicUsize = AtomicUsize::new(0); + static DROPPED: AtomicUsize = AtomicUsize::new(0); + struct Probe(std::cell::Cell); + impl Probe { + fn new() -> Self { + INITIALIZED.fetch_add(1, Ordering::SeqCst); + Self(std::cell::Cell::new(0)) + } + } + impl Drop for Probe { + fn drop(&mut self) { + DROPPED.fetch_add(1, Ordering::SeqCst); + } + } + type Storage = super::HotCell; + thread_local! { + static FIRST_STORAGE: Storage = Storage::new(Probe::new()); + static SECOND_STORAGE: Storage = Storage::new(Probe::new()); + } + static FIRST_SLOT: super::SlotId = super::SlotId::named("provider-test::shared"); + static SECOND_SLOT: super::SlotId = super::SlotId::named("provider-test::shared"); + static FIRST: super::HotKey = super::HotKey::new( + &FIRST_SLOT, + || FIRST_STORAGE.try_with(|c| c.value_addr()), + |idx| { + let _ = FIRST_STORAGE.try_with(|c| c.arm_guard(idx)); + }, + ); + static SECOND: super::HotKey = super::HotKey::new( + &SECOND_SLOT, + || SECOND_STORAGE.try_with(|c| c.value_addr()), + |idx| { + let _ = SECOND_STORAGE.try_with(|c| c.arm_guard(idx)); + }, + ); + // Reverse which provider is touched first, and overlap the threads to + // exercise independent claim atomics and isolate each thread's value. + let barrier = std::sync::Arc::new(std::sync::Barrier::new(8)); + let workers: Vec<_> = (0..8) + .map(|i| { + let barrier = barrier.clone(); + std::thread::spawn(move || { + let (first, second) = if i % 2 == 0 { + (&FIRST, &SECOND) + } else { + (&SECOND, &FIRST) + }; + first.with(|p| p.0.set(i + 100)); + barrier.wait(); + assert_eq!(second.with(|p| p.0.get()), i + 100); + assert_eq!( + first.with(|p| p as *const Probe), + second.with(|p| p as *const Probe) + ); + }) + }) + .collect(); + for worker in workers { + worker.join().expect("provider probe thread panicked"); + } + assert_eq!(FIRST.slot_index(), SECOND.slot_index()); + assert!((FIRST.slot_index() as usize) < super::HOT_SLOT_CAPACITY); + assert_eq!(INITIALIZED.load(Ordering::SeqCst), 8); + assert_eq!(DROPPED.load(Ordering::SeqCst), 8); +} + +/// Function-local declarations can have the same module and identifier; +/// the macro's source coordinates must keep their storage independent. +#[test] +fn same_named_local_declarations_remain_distinct() { + fn first() -> u32 { + crate::perry_thread_local! { static LOCAL: std::cell::Cell = const { std::cell::Cell::new(11) }; } + assert_eq!(LOCAL.with(|p| p.get()), 11); + LOCAL.slot_index() + } + fn second() -> u32 { + crate::perry_thread_local! { static LOCAL: std::cell::Cell = const { std::cell::Cell::new(22) }; } + assert_eq!(LOCAL.with(|p| p.get()), 22); + LOCAL.slot_index() + } + assert_ne!(first(), second()); +} diff --git a/crates/perry/tests/issue_9371_large_presized_array.rs b/crates/perry/tests/issue_9371_large_presized_array.rs index 8b7dfefeba..bccc0affe1 100644 --- a/crates/perry/tests/issue_9371_large_presized_array.rs +++ b/crates/perry/tests/issue_9371_large_presized_array.rs @@ -4,12 +4,34 @@ //! values or falling into quadratic string-keyed property insertion. use std::path::PathBuf; -use std::process::Command; +use std::process::{Command, Output, Stdio}; +use std::time::{Duration, Instant}; fn perry_bin() -> PathBuf { PathBuf::from(env!("CARGO_BIN_EXE_perry")) } +fn run_with_timeout(mut command: Command) -> Output { + command.stdout(Stdio::piped()).stderr(Stdio::piped()); + let mut child = command.spawn().expect("run compiled fixture"); + let deadline = Instant::now() + Duration::from_secs(30); + loop { + if child.try_wait().expect("poll compiled fixture").is_some() { + return child.wait_with_output().expect("collect fixture output"); + } + if Instant::now() >= deadline { + child.kill().expect("kill timed out fixture"); + let output = child.wait_with_output().expect("collect timeout output"); + panic!( + "large array fixture exceeded 30 seconds\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + std::thread::sleep(Duration::from_millis(25)); + } +} + #[test] fn large_presized_arrays_fill_densely_and_preserve_every_value() { let dir = tempfile::tempdir().expect("tempdir"); @@ -17,7 +39,9 @@ fn large_presized_arrays_fill_densely_and_preserve_every_value() { let output = dir.path().join("main_bin"); std::fs::write( &entry, - r#" + concat!( + include_str!("../../../test-files/test_gap_9784_module_presized_array.ts"), + r#" declare function gc(): void; function fillAndVerify(slots: number, addExpando: boolean): string { @@ -51,6 +75,7 @@ huge[16] = 9; huge[100000000] = 11; console.log(huge.length, huge[0], huge[1] === undefined, huge[16], huge[100000000]); "#, + ), ) .expect("write fixture"); @@ -70,7 +95,11 @@ console.log(huge.length, huge[0], huge[1] === undefined, huge[16], huge[10000000 String::from_utf8_lossy(&compile.stderr) ); - let expected = "900000:0:0.25:899999.25:undefined\n\ + let expected = "1000000 0 999496507 0 999999\n\ + 1000001 0 496500 0 1000000\n\ + 1200000 0 999394967 0 1199999\n\ + literal 0 0 1000000\n\ + 900000:0:0.25:899999.25:undefined\n\ 1000001:0:0.25:1000000.25:undefined\n\ 1200000:0:0.25:1199999.25:kept\n\ cells 1000001 32640 0 255\n\ @@ -82,7 +111,7 @@ console.log(huge.length, huge[0], huge[1] === undefined, huge[16], huge[10000000 .env("PERRY_GC_FORCE_EVACUATE", "1") .env("PERRY_GC_VERIFY_EVACUATION", "1"); } - let run = command.output().expect("run compiled fixture"); + let run = run_with_timeout(command); assert!( run.status.success(), "compiled fixture failed with moving_gc={moving_gc}\nstatus: {:?}\nstdout:\n{}\nstderr:\n{}", diff --git a/scripts/gc_rekeyed_key_tables.json b/scripts/gc_rekeyed_key_tables.json index d096d3f65a..b31756956e 100644 --- a/scripts/gc_rekeyed_key_tables.json +++ b/scripts/gc_rekeyed_key_tables.json @@ -32,10 +32,10 @@ "why": "#8191: registered prune retains on !is_dead_owner over the wrapper ObjectHeader address (builtins/formatting/boxed_primitives.rs)." }, { - "site": "crates/perry-runtime/src/closure/dynamic_props.rs::scan_closure_dynamic_props_roots_mut", + "site": "crates/perry-runtime/src/closure/dynamic_props.rs::scan_closure_owner", "table": "CLOSURE_PROPS / CLOSURE_STATIC_PROTOTYPES / CLOSURE_DELETED_KEYS", "death": "dead_owner:prune_dead_closure_side_table_owners", - "why": "All three tables are retained on the GC_TYPE_CLOSURE-narrowed predicate in one prune (closure/dynamic_props.rs:206-214)." + "why": "All three tables are retained on the GC_TYPE_CLOSURE-narrowed predicate in one prune (closure/dynamic_props.rs:206-214). #9754: the per-owner body shared by the full walk and the young-log walk." }, { "site": "crates/perry-runtime/src/fs/mod.rs::scan_filehandle_object_fd_metadata_roots_mut", @@ -98,10 +98,10 @@ "why": "Cleared per dead object by gc_type_clear_dead_payload_side_tables (gc/types.rs:799) from the sweep (gc/oldgen.rs:700, 924, 1573), outside the dead_owner fan-out." }, { - "site": "crates/perry-runtime/src/object/mod.rs::scan_transition_cache_roots_mut", + "site": "crates/perry-runtime/src/object/side_table_roots.rs::scan_transition_cache_slot", "table": "TRANSITION_CACHE_GLOBAL", "death": "dead_owner:prune_dead_transition_cache_entries", - "why": "#8192: registered prune drops the whole cache entry when either weak half (prev_keys keys-array, key_ptr interned string) is dead; next_keys is a strong root and cannot be." + "why": "#8192: registered prune drops the whole cache entry when either weak half (prev_keys keys-array, key_ptr interned string) is dead; next_keys is a strong root and cannot be. #9754: the per-slot body shared by the full walk and the young-log walk." }, { "site": "crates/perry-runtime/src/object/native_module/callable_exports.rs::scan_builtin_closure_metadata_roots_mut", @@ -115,6 +115,12 @@ "death": "dead_owner:prune_dead_shape_keys", "why": "Registered prune drops descriptors whose keys_array is dead and retains inner.indices on the same predicate, then rebuilds the reverse indices (object/shapes.rs:966-985)." }, + { + "site": "crates/perry-runtime/src/object/shapes.rs::scan_shape_keys_address", + "table": "state().shapes.inner (descriptors + indices)", + "death": "dead_owner:prune_dead_shape_keys", + "why": "#9754: the young-log walk's per-keys-address body, the same family probe and slot-index re-key as scan_shape_table_rekey_mut; pruned by the same registered prune (and its young-log variant on minors)." + }, { "site": "crates/perry-runtime/src/perf_hooks.rs::scan_perf_entries_roots_mut", "table": "PERF_ENTRY_KEYS_ARRAY", diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index c942984b5e..b52c8629f5 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -270,7 +270,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -286,8 +286,8 @@ }, "sources": { "crates/perry-runtime/src/gc/census.rs": "8050a9d1ca15f783195ccfa5963089b60bc4a6c31d9ced5755537623e70e7e3c", - "crates/perry-runtime/src/gc/cycle.rs": "4acea623de941aac70d38a4c993a3cd23135152e51f24b11bacd3d68e2571208", - "crates/perry-runtime/src/gc/mod.rs": "ae00e84027b5b442c4a4723309b3e8f6911bc8f592c2ecd5678024e7c244ee26", + "crates/perry-runtime/src/gc/cycle.rs": "763d552271b8e983a796b4e9648cd8ee984a0602b2b56aeefdb8713c0049c31f", + "crates/perry-runtime/src/gc/mod.rs": "085c3dcde34a172aa2b96ee4500658abae77cd34ee7f0e2dfeee06ae5774a414", "crates/perry-runtime/src/gc/policy.rs": "319ed42f1a985c88f6362657a08518077283fe5216d6055fc82343b34dec50f9", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } @@ -481,6 +481,18 @@ "verdict": "not_a_gc_pointer", "why": "First-insertion order for CLASS_DYNAMIC_PROPS: HashMap> of owned Rust strings, needed because the value table is a HashMap while [[OwnPropertyKeys]] needs order. Holds no JSValues; the f64 values live in CLASS_DYNAMIC_PROPS, which is already a scanned root." }, + { + "file": "crates/perry-runtime/src/object/mod.rs", + "name": "SHAPE_CACHE_YOUNG", + "verdict": "not_a_gc_pointer", + "why": "#9754 remembered set: `YoungLog` of shape-cache IDs (inline slot and overflow key alike) whose keys array a minor may act on. Ids, not addresses; the arrays themselves are visited by the registered shape-cache scanner." + }, + { + "file": "crates/perry-runtime/src/object/mod.rs", + "name": "TRANSITION_CACHE_YOUNG", + "verdict": "not_a_gc_pointer", + "why": "#9754 remembered set: `YoungLog` = two `Vec` (live + recycled spare) holding transition-cache SLOT INDICES whose `key_ptr`/`next_keys` a minor may act on. Indices, not addresses \u2014 a `u32` cannot hold a 48-bit pointer. The pointer-bearing entries they index are visited by `scan_transition_cache_roots_mut`, which is registered." + }, { "file": "crates/perry-runtime/src/object/native_module.rs", "name": "TEST_BOUND_METHOD_MOVE", diff --git a/scripts/shape_descriptor_census_baseline.json b/scripts/shape_descriptor_census_baseline.json index 680aff574a..ce87ef8648 100644 --- a/scripts/shape_descriptor_census_baseline.json +++ b/scripts/shape_descriptor_census_baseline.json @@ -42,20 +42,22 @@ "crates/perry-runtime/src/object/field_get_set/field_ops.rs|keys_array|declaration|pub extern fn js_object_set_keys(obj: *mut ObjectHeader, keys_array: *mut ArrayHeader) {": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|access|crate::gc::runtime_store_root_raw_mut_ptr_slot(&mut entry.keys_array, keys_array);": 2, "crates/perry-runtime/src/object/mod.rs|keys_array|access|return (entry.keys_array, entry.runtime_shape_id);": 1, - "crates/perry-runtime/src/object/mod.rs|keys_array|access|visitor.visit_raw_mut_ptr_slot(&mut entry.keys_array);": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|fn shape_cache_insert(shape_id: u32, keys_array: *mut ArrayHeader) {": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|keys_array: *mut ArrayHeader,": 2, "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|keys_array: std::ptr::null_mut(),": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|pub(crate) fn test_seed_shape_cache_root(shape_id: u32, keys_array: *mut ArrayHeader) {": 1, + "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|pub(crate) fn test_shape_cache_insert(shape_id: u32, keys_array: *mut ArrayHeader) {": 1, + "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|pub(super) fn arm_shape_cache_young(shape_id: u32, keys_array: *mut ArrayHeader) {": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|unsafe fn set_object_keys_array(obj: *mut ObjectHeader, keys_array: *mut ArrayHeader) {": 1, "crates/perry-runtime/src/object/object_ops.rs|keys_array|declaration|pub(crate) use keys_array::{": 1, + "crates/perry-runtime/src/object/side_table_roots.rs|keys_array|access|visitor.visit_raw_mut_ptr_slot(&mut entry.keys_array);": 1, "crates/perry-runtime/src/object/test_root_accessors.rs|keys_array|access|let inline = unsafe { (*st.object_hot.shape_inline_cache.get())[slot].keys_array as usize };": 1 }, "summary": { "codegen_object_header_size_sites": 43, - "raw_member_files": 8, + "raw_member_files": 9, "raw_member_sites": { - "keys_array": 24 + "keys_array": 26 } } } diff --git a/test-files/test_gap_9676_stdin_unref_ref_keeps_reader.ts b/test-files/test_gap_9676_stdin_unref_ref_keeps_reader.ts index 2a11c08adc..dc280056f2 100644 --- a/test-files/test_gap_9676_stdin_unref_ref_keeps_reader.ts +++ b/test-files/test_gap_9676_stdin_unref_ref_keeps_reader.ts @@ -76,6 +76,10 @@ function runRole(name: string, onFirst: (s: any) => void, doChurn: boolean): voi console.log(name + " phase1: true"); onFirst(s); if (doChurn) console.log(name + " churn: " + (churn(300000) > 0)); + // The parent sends TWO only after the toggle and all churn complete. + // A fixed delay cannot prove this ordering and four sequential 2.5s + // waits alone exceed the parity suite's 10s per-process budget (#9783). + console.log(name + " ready: true"); } else if (phase === 1 && text.indexOf("TWO") >= 0) { clearInterval(ticker); finish(name + " phase2: true"); @@ -110,9 +114,24 @@ if (role === "unref-ref") { new Promise((resolve) => { const child = spawn(process.execPath, childArgs, { env: { ...process.env, [ROLE_ENV]: name }, - stdio: ["pipe", "inherit", "inherit"], + stdio: ["pipe", "pipe", "inherit"], }); let settled = false; + let output = ""; + let sentSecond = false; + child.stdout!.on("data", (chunk: any) => { + process.stdout.write(chunk); + output += String(chunk); + // stdout may split the readiness line across arbitrary chunks. + if (!sentSecond && output.includes(name + " ready: true\n")) { + sentSecond = true; + try { + child.stdin!.write("TWO\n"); + } catch { + /* child already gone */ + } + } + }); const watchdog = setTimeout(() => { if (settled) return; settled = true; @@ -120,7 +139,8 @@ if (role === "unref-ref") { child.kill("SIGKILL"); resolve(); }, WATCHDOG_MS); - child.on("exit", (code) => { + // Drain the piped stdout before printing the role's exit summary. + child.on("close", (code) => { if (settled) return; settled = true; clearTimeout(watchdog); @@ -134,14 +154,6 @@ if (role === "unref-ref") { /* child already gone */ } }, 120); - // Late enough that the churn role has finished collecting first. - setTimeout(() => { - try { - child.stdin!.write("TWO\n"); - } catch { - /* child already gone */ - } - }, 2500); }); (async () => { diff --git a/test-files/test_gap_9782_specialized_typedarray_last_use.ts b/test-files/test_gap_9782_specialized_typedarray_last_use.ts new file mode 100644 index 0000000000..ec3e8c1ad8 --- /dev/null +++ b/test-files/test_gap_9782_specialized_typedarray_last_use.ts @@ -0,0 +1,22 @@ +// A specialized callee hoists the typed-array data pointer. Its caller must +// retain the array even when the call is the binding's last source-level use. +// Also run with PERRY_GC_HEAP_LIMIT=8 PERRY_GEN_GC=0 to force full collections. +let sink: any[] = []; +function sumDuringChurn(buf: Int32Array, count: number): number { + let sum = 0; + for (let i = 0; i < count; i++) { + sink.push({ i, text: "churn-" + i, pair: [i, i + 1] }); + if (sink.length > 4096) sink = []; + sum = (sum + buf[i & 255]) | 0; + } + return sum; +} +function localOwner(): number { + const local = new Int32Array(256); + for (let i = 0; i < 256; i++) local[i] = i; + return sumDuringChurn(local, 320000); +} +const top = new Int32Array(256); +for (let i = 0; i < 256; i++) top[i] = i; +console.log("module-last-use", sumDuringChurn(top, 320000)); +console.log("local-last-use", localOwner()); diff --git a/test-files/test_gap_9784_module_presized_array.ts b/test-files/test_gap_9784_module_presized_array.ts new file mode 100644 index 0000000000..d77930ecc6 --- /dev/null +++ b/test-files/test_gap_9784_module_presized_array.ts @@ -0,0 +1,44 @@ +// #9784: logical array length must not prove backing-store capacity. + +const boundarySlots = 1000000; +const boundary: number[] = new Array(boundarySlots); +for (let i = 0; i < boundarySlots; i++) boundary[i] = i; +let boundaryWrong = 0; +let boundaryChecksum = 0; +for (let i = 0; i < boundarySlots; i++) { + if (boundary[i] !== i) boundaryWrong++; + boundaryChecksum = (boundaryChecksum + boundary[i]) % 1000000007; +} +console.log(boundarySlots, boundaryWrong, boundaryChecksum, boundary[0], boundary[boundarySlots - 1]); + +const aboveSlots = 1000001; +const above: number[] = new Array(aboveSlots); +for (let i = 0; i < aboveSlots; i++) above[i] = i; +let aboveWrong = 0; +let aboveChecksum = 0; +for (let i = 0; i < aboveSlots; i++) { + if (above[i] !== i) aboveWrong++; + aboveChecksum = (aboveChecksum + above[i]) % 1000000007; +} +console.log(aboveSlots, aboveWrong, aboveChecksum, above[0], above[aboveSlots - 1]); + +const largerSlots = 1200000; +const larger: number[] = new Array(largerSlots); +for (let i = 0; i < largerSlots; i++) larger[i] = i; +let largerWrong = 0; +let largerChecksum = 0; +for (let i = 0; i < largerSlots; i++) { + if (larger[i] !== i) largerWrong++; + largerChecksum = (largerChecksum + larger[i]) % 1000000007; +} +console.log(largerSlots, largerWrong, largerChecksum, larger[0], larger[largerSlots - 1]); + +// A literal allocation inside a function also supplies a static length proof. +function literalLocal(): void { + const values: number[] = new Array(1000001); + for (let i = 0; i < 1000001; i++) values[i] = i; + let wrong = 0; + for (let i = 0; i < 1000001; i++) if (values[i] !== i) wrong++; + console.log("literal", wrong, values[0], values[1000000]); +} +literalLocal(); diff --git a/test-files/test_gap_9785_array_prototype_chain_depth.ts b/test-files/test_gap_9785_array_prototype_chain_depth.ts new file mode 100644 index 0000000000..cf79a18ca1 --- /dev/null +++ b/test-files/test_gap_9785_array_prototype_chain_depth.ts @@ -0,0 +1,74 @@ +// #9785: indexed Get, HasProperty, and strict Set follow every custom array +// prototype link, stop at null, and preserve ordinary-array inheritance. + +function show(label: string, value: unknown): void { + console.log(label, JSON.stringify(value === undefined ? "undefined" : value)); +} + +// ── 1. a middle link that must be consulted ────────────────────────────────── +const mid: any = { 5: "from-mid", 7: "mid-seven" }; +const protoArr: any = []; +protoArr[3] = "from-protoArr"; +Object.setPrototypeOf(protoArr, mid); + +const arr: any = []; +arr[0] = "own-zero"; +Object.setPrototypeOf(arr, protoArr); + +show("depth.own", arr[0]); +show("depth.viaProtoArr", arr[3]); +show("depth.viaMid", arr[5]); // spec: "from-mid" — the skipped link +show("depth.viaMid7", arr[7]); // spec: "mid-seven" +show("depth.absent", arr[9]); +show("depth.hasMid", 5 in arr); +show("depth.hasAbsent", 9 in arr); + +// ── 2. a chain terminated with null must stop ──────────────────────────────── +(Array.prototype as any)[42] = "default-array-proto"; +(Object.prototype as any)[43] = "default-object-proto"; + +const cutProto: any = []; +cutProto[1] = "cut-one"; +Object.setPrototypeOf(cutProto, null); + +const cut: any = []; +Object.setPrototypeOf(cut, cutProto); + +show("cut.viaCutProto", cut[1]); +show("cut.arrayProtoLeak", cut[42]); // spec: undefined +show("cut.objectProtoLeak", cut[43]); // spec: undefined +show("cut.has42", 42 in cut); +show("cut.has43", 43 in cut); + +// A plain array still inherits both, which pins that the leak above is about +// chain termination and not about the indices being absent altogether. +const plain: any = []; +show("plain.arrayProto", plain[42]); +show("plain.objectProto", plain[43]); + +// ── 3. strict [[Set]] must consult the same chain the [[Get]] walks ────────── +// A non-writable inherited index makes a strict assignment throw; the owner +// search has to find it through the SAME depth the read uses. +const roProto: any = {}; +Object.defineProperty(roProto, "6", { value: "readonly", writable: false, enumerable: true }); +const midArr: any = []; +Object.setPrototypeOf(midArr, roProto); +const target: any = []; +Object.setPrototypeOf(target, midArr); + +let threw = "no-throw"; +try { + "use strict"; + const assign = new Function("o", '"use strict"; o[6] = "written";'); + assign(target); +} catch (error) { + threw = (error as Error).constructor.name; +} +show("strictSet.threw", threw); +show("strictSet.value", target[6]); +show("strictSet.own", Object.prototype.hasOwnProperty.call(target, "6")); + +// Clean up so the trailing summary is not polluted for other readers. +delete (Array.prototype as any)[42]; +delete (Object.prototype as any)[43]; +console.log("array-proto-depth-v1:done"); diff --git a/test-files/test_gap_9785_array_prototype_receivers.ts b/test-files/test_gap_9785_array_prototype_receivers.ts new file mode 100644 index 0000000000..0008e03183 --- /dev/null +++ b/test-files/test_gap_9785_array_prototype_receivers.ts @@ -0,0 +1,91 @@ +"use strict"; + +// Multi-hop array prototypes must preserve Receiver and stop at the first own +// property, including undefined values and writable shadows of readonly data. +const reads: string[] = []; +const far: any = []; +const near: any = []; +const receiver: any = ["receiver"]; +Object.defineProperty(far, "2", { + configurable: true, + get() { reads.push(`get:${this === receiver}`); return this[0]; }, +}); +Object.setPrototypeOf(near, far); +Object.setPrototypeOf(receiver, near); +console.log("receiver", receiver[2], Array.prototype.at.call(receiver, 2), reads.join("|")); +receiver.length = 3; +reads.length = 0; +console.log("join", Array.prototype.join.call(receiver, ","), reads.join("|")); + +const readonly: any = {}; +Object.defineProperty(readonly, "4", { value: "readonly", writable: false }); +const shadow: any = []; +shadow[4] = undefined; +Object.setPrototypeOf(shadow, readonly); +const child: any = []; +Object.setPrototypeOf(child, shadow); +console.log("shadow-before", child[4], 4 in child); +child[4] = "written"; +console.log("shadow-after", child[4], Object.hasOwn(child, 4), shadow[4]); + +// A grown prototype is still the same chain node, even when its old allocation +// became a forwarding stub after the child captured it. +const grown: any = []; +const grownChild: any = []; +Object.setPrototypeOf(grownChild, grown); +for (let i = 0; i < 100; i++) grown.push(i); +console.log("grown", grownChild[99], 99 in grownChild, 100 in grownChild); + +// Interleave arrays and ordinary objects before a Proxy to check that its +// traps still observe the original array and run once per internal operation. +const traps: string[] = []; +let original: any; +const proxy = new Proxy({}, { + get(_target, key, recv) { + if (key === "1") { traps.push(`get:${recv === original}`); return "one"; } + return undefined; + }, + has(_target, key) { + if (key === "1" || key === "7") traps.push(`has:${String(key)}`); + return key === "1"; + }, +}); +const objectHop = Object.create(proxy); +const arrayHop: any = []; +Object.setPrototypeOf(arrayHop, objectHop); +original = [0, , 2]; +Object.setPrototypeOf(original, arrayHop); +console.log("proxy-get", original[1], traps.join("|")); +traps.length = 0; +console.log("proxy-has", 1 in original, traps.join("|")); +traps.length = 0; +console.log("proxy-indexOf", Array.prototype.indexOf.call(original, "one"), traps.join("|")); +traps.length = 0; +console.log("proxy-missing", 7 in original, "7" in original, traps.join("|")); + +// An inherited Proxy get trap may itself perform an unrelated inherited read. +// The outer Receiver must not leak into that nested lookup. +const innerProto = { get marker() { return this.name; } }; +const inner = Object.create(innerProto); +inner.name = "inner"; +const nestedProxy = new Proxy({}, { + get(_target, key, recv) { + return `${recv === nested}:${inner.marker}`; + }, +}); +const nested: any = []; +Object.setPrototypeOf(nested, Object.create(nestedProxy)); +console.log("nested-trap", nested[1]); + +const targetGetter = { get 1() { return this.name; } }; +const noTrap = new Proxy(new Proxy(targetGetter, {}), {}); +const getterChild: any = []; +getterChild.name = "array"; +Object.setPrototypeOf(getterChild, noTrap); +console.log("proxy-getter", getterChild[1], Reflect.get(noTrap, "1", { name: "reflect" })); +let reflected: any; +const reflectProxy = new Proxy({}, { + get(_target, _key, recv) { return recv === reflected; }, +}); +reflected = {}; +console.log("reflect-receiver", Reflect.get(reflectProxy, "x", reflected)); diff --git a/test-files/test_gap_9786_array_proxy_prototype.ts b/test-files/test_gap_9786_array_proxy_prototype.ts new file mode 100644 index 0000000000..5e128f95e9 --- /dev/null +++ b/test-files/test_gap_9786_array_proxy_prototype.ts @@ -0,0 +1,60 @@ +// Array fast paths must walk the *actual* full prototype chain and must not +// treat Proxy prototypes as if no custom prototype existed. + +"use strict"; + +const setterLog: string[] = []; +const grand: any = {}; +Object.defineProperty(grand, "3", { + configurable: true, + get() { + return "from-grand"; + }, + set(this: any, value: any) { + setterLog.push(`${this === deep}:${value}`); + }, +}); +const middle: any[] = []; +Object.setPrototypeOf(middle, grand); +const deep: any[] = [0]; +Object.setPrototypeOf(deep, middle); +deep[3] = 17; +console.log(setterLog.join(","), Object.hasOwn(deep, 3), deep[3], deep.length); + +const proxyLog: string[] = []; +let proxied: any[]; +const proxyPrototype = new Proxy( + {}, + { + get(_target, key, receiver) { + if (key === "4") proxyLog.push(`get:${receiver === proxied}`); + return key === "4" ? "proxy-four" : Reflect.get(_target, key, receiver); + }, + has(_target, key) { + if (key === "4") proxyLog.push("has"); + return key === "4" || Reflect.has(_target, key); + }, + set(_target, key, value, receiver) { + proxyLog.push(`set:${String(key)}:${value}:${receiver === proxied}`); + return true; + }, + }, +); +proxied = [1]; +Object.setPrototypeOf(proxied, proxyPrototype); +proxied[4] = 29; +console.log(Object.hasOwn(proxied, 4), proxied[4], 4 in proxied, proxied.length); +console.log(proxyLog.join("|")); + +const holeGrand: any = { 1: "inherited-hole" }; +const holeMiddle: any[] = []; +Object.setPrototypeOf(holeMiddle, holeGrand); +const holey: any[] = [0, , 2]; +Object.setPrototypeOf(holey, holeMiddle); +const seen: string[] = []; +Array.prototype.forEach.call(holey, (v: any, i: number) => seen.push(`${i}:${v}`)); +console.log( + Array.prototype.join.call(holey, ","), + Array.prototype.indexOf.call(holey, "inherited-hole"), + seen.join("|"), +); diff --git a/test-files/test_gap_9788_iterator_protocol_mutation.ts b/test-files/test_gap_9788_iterator_protocol_mutation.ts new file mode 100644 index 0000000000..31e311210b --- /dev/null +++ b/test-files/test_gap_9788_iterator_protocol_mutation.ts @@ -0,0 +1,49 @@ +// Exercise declaration/expression vtables, own overrides, prototype mutation, +// IteratorClose, and symbol enumeration in one deterministic matrix. + +class DeclaredRange { + lo: number; + hi: number; + constructor(lo: number, hi: number) { + this.lo = lo; + this.hi = hi; + } + *[Symbol.iterator]() { + for (let i = this.lo; i <= this.hi; i++) yield i; + } +} + +const ExpressionRange = class { + *[Symbol.iterator]() { + yield "expr-a"; + yield "expr-b"; + } +}; + +console.log([...new DeclaredRange(2, 4)].join(",")); +console.log([...new ExpressionRange()].join(",")); +console.log( + Object.getOwnPropertySymbols(ExpressionRange.prototype).map(String).join(","), + Object.getOwnPropertyNames(ExpressionRange.prototype).join(","), +); + +const own: any = new DeclaredRange(1, 2); +own[Symbol.iterator] = function* () { + yield 99; +}; +console.log([...own].join(",")); + +(DeclaredRange.prototype as any)[Symbol.iterator] = function* () { + yield 70; + yield 71; +}; +console.log([...new DeclaredRange(1, 2)].join(",")); + +const iterator: any = new Map([[1, "a"], [2, "b"]]).entries(); +let closed = 0; +iterator.return = () => { + closed++; + return { done: true, value: undefined }; +}; +for (const _entry of iterator) break; +console.log("closed", closed); diff --git a/test-files/test_gap_9788_iterator_prototype_overrides.ts b/test-files/test_gap_9788_iterator_prototype_overrides.ts new file mode 100644 index 0000000000..160bf1f579 --- /dev/null +++ b/test-files/test_gap_9788_iterator_prototype_overrides.ts @@ -0,0 +1,60 @@ +// #9788: declaration/expression and module/function loop paths must all read +// the current Symbol.iterator, including prototype accessors and inheritance. +class Range { + name = "range"; + *[Symbol.iterator]() { yield 1; yield 2; } +} +function functionLoop() { + const result: unknown[] = []; + for (const value of new Range()) result.push(value); + return result.join(","); +} +console.log("before", functionLoop()); +Range.prototype[Symbol.iterator] = function* () { yield 7; yield 8; }; +const moduleValues: unknown[] = []; +for (const value of new Range()) moduleValues.push(value); +console.log("loops", moduleValues.join(","), functionLoop()); +console.log("call", new Range()[Symbol.iterator]().next().value); +console.log("array-from", Array.from(new Range()).join(",")); + +class Inherited extends Range {} +class Own extends Range { *[Symbol.iterator]() { yield 3; } } +console.log("inherit", [...new Inherited()].join(","), [...new Own()].join(",")); +Inherited.prototype[Symbol.iterator] = function* () { yield 4; }; +console.log("sub-override", [...new Inherited()].join(","), [...new Range()].join(",")); + +let accessorReceiver: any; +let gets = 0; +Object.defineProperty(Range.prototype, Symbol.iterator, { + configurable: true, + get() { + gets++; + accessorReceiver = this; + return function* () { yield this.name; }; + }, +}); +const instance = new Range(); +console.log("getter", [...instance].join(","), gets, accessorReceiver === instance); +Object.defineProperty(Range.prototype, Symbol.iterator, { value: undefined, configurable: true }); +try { console.log([...instance]); } catch (error) { console.log("undefined", error instanceof TypeError); } + +const Expression = class { *[Symbol.iterator]() { yield "old"; } }; +Expression.prototype[Symbol.iterator] = function* () { yield "new"; }; +console.log("expression", [...new Expression()].join(",")); +console.log("expression-names", Object.getOwnPropertyNames(Expression.prototype).join(",")); + +class Plain { + [Symbol.iterator]() { return [5, 6][Symbol.iterator](); } +} +console.log("non-generator", [...new Plain()].join(",")); +console.log("plain-names", Object.getOwnPropertyNames(Plain.prototype).join(",")); +class Literal { + "@@iterator"() { return "literal"; } +} +console.log("literal", Object.getOwnPropertyNames(Literal.prototype).join(","), new Literal()["@@iterator"]()); +const ownGetter: any = new Own(); +let ownGets = 0; +Object.defineProperty(ownGetter, Symbol.iterator, { + get() { ownGets++; return function* () { yield 9; }; }, +}); +console.log("own-getter-call", ownGetter[Symbol.iterator]().next().value, ownGets); diff --git a/tests/fixtures/issue_8075_provider_gc/stdlib-linker.sh b/tests/fixtures/issue_8075_provider_gc/stdlib-linker.sh index 0533ce1485..277b2b6544 100755 --- a/tests/fixtures/issue_8075_provider_gc/stdlib-linker.sh +++ b/tests/fixtures/issue_8075_provider_gc/stdlib-linker.sh @@ -18,9 +18,11 @@ stdlib_provider_exports=( js_headers_set js_readable_stream_get_reader_with_options js_readable_stream_new_from_source_object + js_readable_stream_new_with_strategy_and_source_type js_reader_read js_response_body js_response_body_init_ptr + js_response_body_init_reset js_response_get_headers js_response_new js_stdlib_init_dispatch diff --git a/tests/fixtures/issue_8075_provider_gc/stdlib-provider/src/lib.rs b/tests/fixtures/issue_8075_provider_gc/stdlib-provider/src/lib.rs index ee860364fe..3419235ba2 100644 --- a/tests/fixtures/issue_8075_provider_gc/stdlib-provider/src/lib.rs +++ b/tests/fixtures/issue_8075_provider_gc/stdlib-provider/src/lib.rs @@ -25,6 +25,7 @@ unsafe extern "C" fn pin_issue_8038_response_surface() { let _ = perry_stdlib::js_headers_set(0.0, std::ptr::null(), std::ptr::null()); let _ = perry_stdlib::js_headers_append(0.0, std::ptr::null(), std::ptr::null()); let _ = perry_stdlib::js_headers_get(0.0, std::ptr::null()); + let _ = perry_stdlib::js_response_body_init_reset(); let _ = perry_stdlib::js_response_body_init_ptr(0.0); let _ = perry_stdlib::js_response_new(std::ptr::null(), 0.0, std::ptr::null(), 0.0); let _ = perry_stdlib::js_response_get_headers(0.0); @@ -32,6 +33,8 @@ unsafe extern "C" fn pin_issue_8038_response_surface() { let _ = perry_stdlib::js_fetch_response_status_text(0.0); let _ = perry_stdlib::js_response_body(0.0); let _ = perry_stdlib::js_readable_stream_new_from_source_object(0.0, 0.0); + let _ = + perry_stdlib::js_readable_stream_new_with_strategy_and_source_type(0.0, 0.0, 0.0, 0.0, 0.0); let _ = perry_stdlib::js_readable_stream_get_reader_with_options(0.0, 0.0); let _ = perry_stdlib::js_reader_read(0.0); perry_stdlib::js_stdlib_init_dispatch(); diff --git a/tests/test_gc_ratchet.py b/tests/test_gc_ratchet.py index 76bcce51e9..0066a63970 100644 --- a/tests/test_gc_ratchet.py +++ b/tests/test_gc_ratchet.py @@ -237,6 +237,95 @@ def _hard(failures): return [failure for failure in failures if not failure.startswith("NOTE")] +class CurrentCounterDeterminismTests(unittest.TestCase): + def test_measure_keeps_disagreement_and_runs_the_remaining_probes(self): + stderr = ( + "#gcmetric heap_used_bytes=1000000\n" + "#gcmetric heap_total_bytes=20971520\n" + "#gcmetric rss_bytes=30000000\n" + ) + + def run(copied_bytes=1500000): + return { + "returncode": 0, "stdout": "ok\n", "wall_ms": 200, + "peak_rss_bytes": 31000000, + "stderr": stderr + "[gc-copy-minor] ran copied_objects=20000 " + f"copied_bytes={copied_bytes} promoted_objects=4000 " + "promoted_bytes=500000 freed_bytes=100000000\n[gc-step]\n", + } + + with tempfile.TemporaryDirectory() as tmp: + probes_dir = Path(tmp) + for name in ("01_probe", "02_other"): + (probes_dir / f"{name}.ts").write_text("// stub\n", encoding="utf-8") + with mock.patch( + "benchmarks.gc_ratchet.gc_ratchet.compile_probe", return_value=Path("stub") + ), mock.patch( + "benchmarks.gc_ratchet.gc_ratchet.run_once", + side_effect=[run()] * 4 + [run(1500002)] + [run()] * 5, + ) as runner: + result = measure( + perry=Path("stub"), probes_dir=probes_dir, repeats=3, node=None, warmup=0 + ) + self.assertEqual(runner.call_count, 10) + self.assertEqual(set(result["probes"]), {"01_probe", "02_other"}) + metric = result["probes"]["01_probe"]["metrics"]["copied_bytes"] + self.assertEqual(metric["samples"], [1500000, 1500002]) + self.assertEqual(metric["spread"], 2) + + def test_every_gated_counter_rejects_disagreement_inside_its_band(self): + for profile in PROFILES: + for metric in GC_METRICS: + with self.subTest(profile=profile, metric=metric): + current = _measurement(_pair()) + value = BASE_VALUES[metric] + dist = distribution([value - 1, value + 1]) + # A stale cached spread must not hide the disagreeing samples. + dist["spread"] = 0 + current["probes"]["01_probe"]["metrics"][metric] = dist + rows, failures = evaluate(_baseline(_pair()), current, profile=profile) + self.assertTrue(any( + metric in failure and "not deterministic" in failure + for failure in _hard(failures) + )) + self.assertEqual(len(rows), 2 * len(ALL_METRICS)) + row = next(r for r in rows if r.probe == "01_probe" and r.metric == metric) + self.assertEqual(row.status, "UNFIT (traced samples disagree)") + + def test_only_the_documented_cells_may_vary(self): + payload = _with_override(metric="copied_bytes") + payload["probe_overrides"]["01_probe"]["freed_bytes"] = _override_entry() + baseline = _baseline(_pair(), payload) + current = _measurement(_pair()) + for metric in ("copied_bytes", "freed_bytes"): + value = BASE_VALUES[metric] + current["probes"]["01_probe"]["metrics"][metric] = distribution([value, value + 1]) + for profile in PROFILES: + rows, failures = evaluate(baseline, current, profile=profile) + self.assertEqual(_hard(failures), []) + excluded = [r for r in rows if not r.gating and r.probe == "01_probe"] + self.assertTrue({"copied_bytes", "freed_bytes"} <= {r.metric for r in excluded}) + for probe, metric in (("01_probe", "copied_objects"), ("02_other", "copied_bytes")): + with self.subTest(probe=probe, metric=metric): + perturbed = copy.deepcopy(current) + value = BASE_VALUES[metric] + perturbed["probes"][probe]["metrics"][metric] = distribution([value, value + 1]) + _, failures = evaluate(baseline, perturbed, profile="shared_ci") + self.assertTrue(any( + probe in failure and metric in failure and "not deterministic" in failure + for failure in _hard(failures) + )) + + def test_array_growth_exclusions_match_the_recorded_samples(self): + receipt = json.loads((REPO_ROOT / "benchmarks/gc_ratchet/evidence/9790-array-growth-pacing.json").read_text(encoding="utf-8")) + entries = _shipped_tolerances()["probe_overrides"]["07_array_grow_evacuate"] + self.assertEqual(set(entries), {"copied_bytes", "freed_bytes"}) + for metric, entry in entries.items(): + values = [sample[metric] for sample in receipt["samples"]] + self.assertEqual(entry["evidence"]["observed_runs"], len(values)) + self.assertEqual(entry["evidence"]["observed_spread"], max(values) - min(values)) + + class ParsingTests(unittest.TestCase): def test_measurement_refuses_a_host_without_wait4_before_launching(self): with mock.patch.object(os, "wait4", None, create=True):