diff --git a/benchmarks/memory/README.md b/benchmarks/memory/README.md index cf871f3439..31d6d9bffb 100644 --- a/benchmarks/memory/README.md +++ b/benchmarks/memory/README.md @@ -26,9 +26,14 @@ so framework ports can be added without renames. benchmarks/memory// package.json Nx targets: build:, test:perf:, test:flame:, test:types bench-utils.ts memoryBenchOptions, seeded LCG (+ sequential request loop on the server side) + isolated-benchmark.ts registers churn benches through the shared child-process controller vitest..config.ts aggregates scenarios/*//vite.config.ts scenarios/// one isolated app per scenario + setup.ts + memory.bench.ts + memory.flame.ts + +benchmarks/memory/shared/ + isolated-process.ts parent-side child lifecycle and IPC protocol + isolated-process-child.ts fresh Node/V8 process that owns and runs one invocation ``` One app per scenario; apps and bench names are stable once landed (CodSpeed @@ -39,27 +44,59 @@ same workload through the Flame profiler. ## How the memory instrument executes a bench -- The bench function is warmed up, then **measured exactly once**, starting - after a forced GC. Under plain `vitest bench` the suites only smoke-test: - timing output is meaningless; real numbers come from CodSpeed. -- Under CodSpeed the bench fn runs several warmup invocations plus the - measured one **on the same mount**, so bench fns must be idempotent and - module-level counters/LCGs are used where ids must never repeat across - invocations. -- Plain `vitest bench` never runs suite hooks (`beforeAll`/`afterAll`) and +- The bench function is warmed up, then **measured exactly once**. Under plain + `vitest bench` the suites only smoke-test: timing output is meaningless; real + numbers come from CodSpeed. +- Churn benchmarks give every CodSpeed optimization invocation and the measured + invocation a fresh Node child process. The child imports the same production + build used by the Flame runner, executes the scenario sanity path, and runs + one full-sized warm-up outside the marker. A full loop is intentional: a + fresh process no longer inherits the V8 heap growth and runtime caches that + earlier benchmarks used to warm implicitly, and a token warm-up leaves those + one-time native allocations inside the measured timeline. Client warm-ups + use a disposable app that is torn down before the child creates the measured + app; server warm-ups use IDs that cannot overlap the measured request + sequence. + After the child reports that the workloads are loaded, the parent sends an + unmeasured `prime` command over the same IPC channel used for measurement. + The child settles pending work and forces two collections before + acknowledging it. This primes both IPC directions and the child command + queue before the parent benchmark function tells the child to execute the + real inner loop inside the CodSpeed marker. Teardown and process exit happen + after the marker. +- The fresh child deliberately replays the same seeded workload on every + invocation. Module-level counters still make every item within one inner loop + unique; they no longer carry state from CodSpeed warmups into measurement. +- Churn loops use deliberately large measured iteration counts so the regular + steady-state shape dominates the timeline and a per-iteration leak is + amplified. Their full-sized warm-up loops are unmeasured and use disjoint + inputs, so they establish the same steady state without consuming or hiding + the measured leak signal. +- Peak-footprint benchmarks remain direct: their existing lifecycle and pinned + inter-iteration collection points were already stable and do not benefit from + process isolation. +- Plain `vitest bench` never runs the Vitest suite hooks and only honors tinybench's `setup`/`teardown` options; the CodSpeed runner - does the exact opposite. Client benches therefore register **both** — in - any given mode exactly one pair runs. -- The process runs with V8 determinism flags (predictable GC schedule, - `--no-opt`). Never call `global.gc()` manually in **churn** scenarios — - their signal is accumulation across iterations, which a forced collection - masks. **Peak** scenarios do the opposite: they set - `pinGcBetweenIterations` on the request loop so a collection runs between - iterations. Their signal is the footprint of a single request, and without - pinned GC points the measured peak flips by a whole payload depending on - whether iteration i's garbage is collected before iteration i+1 allocates. - Because of `--no-opt`, allocation counts overstate production; numbers are - for regression tracking, not absolute claims. + does the exact opposite. Isolated benches therefore register **both** the + suite hooks and Tinybench options; in any given mode exactly one pair runs. +- Isolated children inherit the Vitest worker's V8 flags, including CodSpeed's + memory-analysis configuration and any scenario-specific flags. The controller + then supplies deterministic defaults for flags the worker did not already + set: `--expose-gc`, `--predictable`, `--no-opt`, `--no-flush-bytecode`, and + fixed initial/semi-space sizes. Disabling optimization prevents a workload + from crossing a JIT tier-up threshold inside the marker; retaining bytecode, + pre-sizing the heap, and exercising one full loop before measurement keep + compilation and heap-growth bursts out of the measured peak. The forced + pre-measurement collections remove only unreachable setup, sanity, and + warm-up garbage; reachable caches or leaks survive and remain part of the + measured baseline and subsequent accumulation. +- Server request loops also pin collections between iterations. This removes + floating response/render garbage whose collection timing otherwise shifts the + peak, while retained objects still accumulate because collection cannot + reclaim reachable memory. Peak scenarios additionally verify that the heap + returned to its established floor. CodSpeed's memory-analysis execution can + overstate production allocation counts; numbers are for regression tracking, + not absolute claims. - Keep each bench under **~1.5M allocations** (instrument overhead grows past 2M); this is the main constraint when tuning iteration counts. @@ -167,14 +204,14 @@ pnpm nx run @benchmarks/memory-client-navigation-churn-react:test:flame --output Flame writes reports under the scenario's ignored `.profiles//` directory, including `heap-profile-*.html` and `heap-profile-*.md`. The `memory.flame.ts` entrypoints run the same workload shape as `memory.bench.ts` -but manually start profiling after sanity/setup work and stop it after the -measured workload. Treat these profiles as diagnostic heap-sampling attribution; -they are not CodSpeed memory metrics such as peak memory, allocated bytes, or -allocation counts. The heap sampler is stopped before profile conversion and -Flame report generation, so Flame/pprof report-generation work should not appear -as part of the captured workload. Flame runs do not force GC before profiling; -doing so would perturb the workload and still would not make heap sampling -equivalent to CodSpeed memory metrics. +but manually start profiling after sanity, warm-up, and setup work and stop it +after the measured workload. Treat these profiles as diagnostic heap-sampling +attribution; they are not CodSpeed memory metrics such as peak memory, allocated +bytes, or allocation counts. The heap sampler is stopped before profile +conversion and Flame report generation, so Flame/pprof report-generation work +should not appear as part of the captured workload. Flame runs do not force GC +before profiling; doing so would perturb the workload and still would not make +heap sampling equivalent to CodSpeed memory metrics. Clean local Flame profile output with: diff --git a/benchmarks/memory/client/benchmark.ts b/benchmarks/memory/client/benchmark.ts index 8b877b0740..9f0d0024f0 100644 --- a/benchmarks/memory/client/benchmark.ts +++ b/benchmarks/memory/client/benchmark.ts @@ -3,5 +3,26 @@ export interface ClientMemoryWorkload { before?: () => Promise | void run: () => Promise | void sanity: () => Promise | void + warmup?: () => Promise | void after?: () => Promise | void } + +export async function warmClientMemoryWorkload(workload: ClientMemoryWorkload) { + if (!workload.warmup) { + return + } + + if (Boolean(workload.before) !== Boolean(workload.after)) { + throw new Error( + `Client memory workload ${workload.name} must define both before and after when it defines either hook`, + ) + } + + await workload.before?.() + + try { + await workload.warmup() + } finally { + await workload.after?.() + } +} diff --git a/benchmarks/memory/client/flame-runner.ts b/benchmarks/memory/client/flame-runner.ts index 04941fda67..cabb3665a8 100644 --- a/benchmarks/memory/client/flame-runner.ts +++ b/benchmarks/memory/client/flame-runner.ts @@ -1,10 +1,12 @@ import { profileFlameWorkload } from '../flame-control.ts' import { window } from './jsdom.ts' +import { warmClientMemoryWorkload } from './benchmark.ts' import type { ClientMemoryWorkload } from './benchmark.ts' export async function runClientFlameBenchmark(workload: ClientMemoryWorkload) { try { await workload.sanity() + await warmClientMemoryWorkload(workload) await workload.before?.() await profileFlameWorkload(workload.run, workload.name) } finally { diff --git a/benchmarks/memory/client/isolated-benchmark.ts b/benchmarks/memory/client/isolated-benchmark.ts new file mode 100644 index 0000000000..42e41b2a9b --- /dev/null +++ b/benchmarks/memory/client/isolated-benchmark.ts @@ -0,0 +1,41 @@ +import { afterEach, beforeEach, bench, describe } from 'vitest' +import { IsolatedMemoryProcess } from '../shared/isolated-process.ts' +import { memoryBenchOptions } from './bench-utils.ts' +import type { IsolatedMemoryBenchmarkKind } from '../shared/isolated-process.ts' + +type RegisterIsolatedClientMemoryBenchmarkOptions = { + name: string + setupUrl: URL +} + +const kind = 'client' satisfies IsolatedMemoryBenchmarkKind + +export function registerIsolatedClientMemoryBenchmark( + options: RegisterIsolatedClientMemoryBenchmarkOptions, +) { + const isolatedProcess = new IsolatedMemoryProcess({ + kind, + setupUrl: options.setupUrl, + workloadNames: [options.name], + }) + + const run = async () => { + try { + await isolatedProcess.run(0) + } catch (error) { + await isolatedProcess.stop().catch(() => {}) + throw error + } + } + + describe('memory', () => { + beforeEach(() => isolatedProcess.start()) + afterEach(() => isolatedProcess.stop()) + + bench(options.name, run, { + ...memoryBenchOptions, + setup: () => isolatedProcess.start(), + teardown: () => isolatedProcess.stop(), + }) + }) +} diff --git a/benchmarks/memory/client/package.json b/benchmarks/memory/client/package.json index d4c9f732c7..d5dbd94642 100644 --- a/benchmarks/memory/client/package.json +++ b/benchmarks/memory/client/package.json @@ -9,6 +9,7 @@ "#memory-client/benchmark": "./benchmark.ts", "#memory-client/bench-utils": "./bench-utils.ts", "#memory-client/flame-runner": "./flame-runner.ts", + "#memory-client/isolated-benchmark": "./isolated-benchmark.ts", "#memory-client/lifecycle": "./lifecycle.ts" }, "dependencies": { diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/react/memory.bench.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/react/memory.bench.ts index e645ab38f1..966e517226 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/react/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client interrupted-navigations (react)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts index 048a9cc833..72a85a5337 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client interrupted-navigations (react)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/shared.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/shared.ts index 80f234f0c8..03e7925558 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/shared.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/shared.ts @@ -47,9 +47,17 @@ type InterruptedNavigationRouter = { ) => () => void } -const interruptedNavigationIterations = 150 +const interruptedNavigationIterations = 300 +const interruptedNavigationWarmupIterations = interruptedNavigationIterations const interruptedNavigationPairs = createInterruptedNavigationPairs( interruptedNavigationIterations, + 13, + '', +) +const interruptedNavigationWarmupPairs = createInterruptedNavigationPairs( + interruptedNavigationWarmupIterations, + 0x1a2b3c, + 'warmup-', ) const uninitialized = () => @@ -63,12 +71,16 @@ const uninitializedSettlement = () => reason: new Error('interrupted-navigations benchmark is not initialized'), }) -function createInterruptedNavigationPairs(iterations: number) { - const random = createDeterministicRandom(13) +function createInterruptedNavigationPairs( + iterations: number, + seed: number, + prefix: string, +) { + const random = createDeterministicRandom(seed) return Array.from({ length: iterations }, (_, index) => ({ - slowId: `slow-${index}-${randomSegment(random)}`, - fastId: `fast-${index}-${randomSegment(random)}`, + slowId: `${prefix}slow-${index}-${randomSegment(random)}`, + fastId: `${prefix}fast-${index}-${randomSegment(random)}`, })) } @@ -272,15 +284,20 @@ export function createWorkload( await drainMicrotasks() } + async function runPairs( + pairs: ReadonlyArray<{ slowId: string; fastId: string }>, + ) { + for (const pair of pairs) { + await interrupt(pair.slowId, pair.fastId) + } + } + return { name: `mem client interrupted-navigations (${framework})`, before, interrupt, - async run() { - for (const pair of interruptedNavigationPairs) { - await interrupt(pair.slowId, pair.fastId) - } - }, + run: () => runPairs(interruptedNavigationPairs), + warmup: () => runPairs(interruptedNavigationWarmupPairs), async sanity() { await before() diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/solid/memory.bench.ts index e645ab38f1..a6c7c12554 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/solid/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client interrupted-navigations (solid)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts index e710e0fd12..0dcd12ab31 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client interrupted-navigations (solid)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/vue/memory.bench.ts index e645ab38f1..1f14e79149 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/vue/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client interrupted-navigations (vue)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts index b2912ce9d1..48e83a9725 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts @@ -30,7 +30,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client interrupted-navigations (vue)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/react/memory.bench.ts b/benchmarks/memory/client/scenarios/loader-data-retention/react/memory.bench.ts index e645ab38f1..88fdc69c7a 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/react/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client loader-data-retention (react)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts b/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts index 555e5f5612..1c634f0ea1 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client loader-data-retention (react)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/shared.ts b/benchmarks/memory/client/scenarios/loader-data-retention/shared.ts index bd90c023f0..616d323105 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/shared.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/shared.ts @@ -30,20 +30,26 @@ type LoaderDataRouter = { ) => () => void } -const loaderDataRetentionNavigationCount = 20 -const pageIds = createPageIds() +const loaderDataRetentionNavigationCount = 40 +const loaderDataRetentionWarmupCount = loaderDataRetentionNavigationCount +const pageIds = createPageIds(loaderDataRetentionNavigationCount, 11, '') +const warmupPageIds = createPageIds( + loaderDataRetentionWarmupCount, + 0x10ade2, + 'warmup-', +) const uninitialized = () => Promise.reject( new Error('loader-data-retention benchmark is not initialized'), ) -function createPageIds() { - const random = createDeterministicRandom(11) +function createPageIds(count: number, seed: number, prefix: string) { + const random = createDeterministicRandom(seed) return Array.from( - { length: loaderDataRetentionNavigationCount }, - (_, index) => `${index}-${randomSegment(random)}`, + { length: count }, + (_, index) => `${prefix}${index}-${randomSegment(random)}`, ) } @@ -162,15 +168,18 @@ export function createWorkload( navigateTo = uninitialized } + async function runPageIds(ids: ReadonlyArray) { + for (const id of ids) { + await navigateTo(id) + } + } + return { name: `mem client loader-data-retention (${framework})`, before, navigate: (id: string) => navigateTo(id), - async run() { - for (const id of pageIds) { - await navigateTo(id) - } - }, + run: () => runPageIds(pageIds), + warmup: () => runPageIds(warmupPageIds), async sanity() { await before() diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/loader-data-retention/solid/memory.bench.ts index e645ab38f1..269942013a 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/solid/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client loader-data-retention (solid)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts b/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts index 58d6b77acc..eef5431266 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client loader-data-retention (solid)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/loader-data-retention/vue/memory.bench.ts index e645ab38f1..f322a2fb3f 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/vue/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client loader-data-retention (vue)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts b/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts index bbf2fd02d8..7aea4733a4 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts @@ -30,7 +30,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client loader-data-retention (vue)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/mount-unmount/react/memory.bench.ts b/benchmarks/memory/client/scenarios/mount-unmount/react/memory.bench.ts index e645ab38f1..3d08ac8c6d 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/react/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client mount-unmount (react)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts b/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts index bb0f3c2e0a..1830655a92 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client mount-unmount (react)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/mount-unmount/shared.ts b/benchmarks/memory/client/scenarios/mount-unmount/shared.ts index 01cfe8c46a..44a9bbca62 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/shared.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/shared.ts @@ -12,7 +12,8 @@ type RenderRouter = { subscribe: (event: 'onRendered', listener: () => void) => () => void } -const mountUnmountIterations = 100 +const mountUnmountIterations = 200 +const mountUnmountWarmupIterations = mountUnmountIterations function assertEmptyBody() { if (document.body.childNodes.length !== 0) { @@ -57,14 +58,17 @@ export function createWorkload( } } + async function runCycles(iterations: number) { + for (let index = 0; index < iterations; index++) { + await cycle() + } + } + return { name: `mem client mount-unmount (${framework})`, cycle, - async run() { - for (let index = 0; index < mountUnmountIterations; index++) { - await cycle() - } - }, + run: () => runCycles(mountUnmountIterations), + warmup: () => runCycles(mountUnmountWarmupIterations), async sanity() { assertEmptyBody() await cycle() diff --git a/benchmarks/memory/client/scenarios/mount-unmount/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/mount-unmount/solid/memory.bench.ts index e645ab38f1..7773b41a45 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/solid/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client mount-unmount (solid)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts b/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts index 5f0ab26252..9776817d62 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client mount-unmount (solid)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/mount-unmount/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/mount-unmount/vue/memory.bench.ts index e645ab38f1..5d190dc661 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/vue/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client mount-unmount (vue)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts b/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts index 0f7573a408..10dcf9c379 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts @@ -30,7 +30,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client mount-unmount (vue)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/navigation-churn/react/memory.bench.ts b/benchmarks/memory/client/scenarios/navigation-churn/react/memory.bench.ts index e645ab38f1..41656d2b26 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/react/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client navigation-churn (react)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts b/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts index b05b999297..3b49923dd7 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client navigation-churn (react)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/navigation-churn/shared.ts b/benchmarks/memory/client/scenarios/navigation-churn/shared.ts index 7eaf1a8b66..b95b0496e1 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/shared.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/shared.ts @@ -15,7 +15,8 @@ type NavigationRouter = { subscribe: (event: 'onRendered', listener: () => void) => () => void } -const navigationChurnIterations = 300 +const navigationChurnIterations = 600 +const navigationWarmupIterations = navigationChurnIterations const uninitialized = () => Promise.reject(new Error('navigation-churn benchmark is not initialized')) @@ -104,15 +105,18 @@ export function createWorkload( navigateTo = uninitialized } + async function runNavigationLoop(iterations: number) { + for (let index = 0; index < iterations; index++) { + await navigateTo(index % 2 === 0 ? '/b' : '/a') + } + } + return { name: `mem client navigation-churn (${framework})`, before, navigate: (target: Target) => navigateTo(target), - async run() { - for (let index = 0; index < navigationChurnIterations; index++) { - await navigateTo(index % 2 === 0 ? '/b' : '/a') - } - }, + run: () => runNavigationLoop(navigationChurnIterations), + warmup: () => runNavigationLoop(navigationWarmupIterations), async sanity() { await before() diff --git a/benchmarks/memory/client/scenarios/navigation-churn/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/navigation-churn/solid/memory.bench.ts index e645ab38f1..9a9e554d6f 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/solid/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client navigation-churn (solid)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts b/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts index 6b6735891a..bb1abf61f4 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client navigation-churn (solid)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/navigation-churn/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/navigation-churn/vue/memory.bench.ts index e645ab38f1..c765bc8b5c 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/vue/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client navigation-churn (vue)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts b/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts index d26ce23d07..567d7052bc 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts @@ -30,7 +30,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client navigation-churn (vue)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/preload-churn/react/memory.bench.ts b/benchmarks/memory/client/scenarios/preload-churn/react/memory.bench.ts index e645ab38f1..c519e372ce 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/react/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client preload-churn (react)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts b/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts index f9c426656b..bb728b8698 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client preload-churn (react)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/preload-churn/shared.ts b/benchmarks/memory/client/scenarios/preload-churn/shared.ts index 8cea4aca81..0735e52c30 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/shared.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/shared.ts @@ -38,15 +38,16 @@ type PreloadRouter = { // Fixed id for the eviction navigations interleaved into the bench loop; its // payload is a constant-size steady-state resident, never part of the signal. const evictionItemId = 'nav-evict' -const preloadChurnIterations = 200 +const preloadChurnIterations = 400 +const preloadChurnWarmupIterations = preloadChurnIterations // A navigation commit is what triggers the router's clearExpiredCache -- // preloaded matches (defaultPreloadGcTime: 0) are only evicted then, never // during a preload-only loop. Interleaving a navigation every few preloads is // what makes the flat floor assert "eviction releases preloaded payloads". const preloadsPerEvictionNavigation = 10 -// Module-level so ids stay unique across runner invocations on one mount; a -// per-invocation LCG would replay identical ids, and every preload after the -// first invocation would dedupe against cachedMatches instead of doing work. +// Module-level within the isolated process so every id in the inner loop is +// unique. Every fresh CodSpeed invocation deliberately replays the same seeded +// sequence in a new router, so no cached match can dedupe work across runs. const benchmarkRandom = createDeterministicRandom(0x706c6f61) let preloadCounter = 0 @@ -179,21 +180,35 @@ export function createWorkload( } } + async function runPreloadLoop(iterations: number, createId: () => string) { + for (let index = 0; index < iterations; index++) { + await preloadItem(createId()) + + if ((index + 1) % preloadsPerEvictionNavigation === 0) { + await evictPreloads() + } + } + } + return { name: `mem client preload-churn (${framework})`, before, preload: (id: string) => preloadItem(id), evictPreloads, - async run() { - for (let index = 0; index < preloadChurnIterations; index++) { - await preloadItem( + run: () => + runPreloadLoop( + preloadChurnIterations, + () => `${(preloadCounter++).toString(36)}-${randomSegment(benchmarkRandom)}`, - ) - - if ((index + 1) % preloadsPerEvictionNavigation === 0) { - await evictPreloads() - } - } + ), + warmup() { + const random = createDeterministicRandom(0x5072e10a) + let counter = 0 + + return runPreloadLoop( + preloadChurnWarmupIterations, + () => `warmup-${(counter++).toString(36)}-${randomSegment(random)}`, + ) }, async sanity() { await before() diff --git a/benchmarks/memory/client/scenarios/preload-churn/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/preload-churn/solid/memory.bench.ts index e645ab38f1..a173079184 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/solid/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client preload-churn (solid)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts b/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts index c86a6fcbb3..796a76c4d6 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client preload-churn (solid)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/preload-churn/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/preload-churn/vue/memory.bench.ts index e645ab38f1..25b025fa6f 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/vue/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client preload-churn (vue)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts b/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts index ea7ed7b695..5e79484c45 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts @@ -30,7 +30,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client preload-churn (vue)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/react/memory.bench.ts b/benchmarks/memory/client/scenarios/unique-location-churn/react/memory.bench.ts index e645ab38f1..2ff337509a 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/react/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client unique-location-churn (react)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts b/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts index f424c57cef..5e9f221fec 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client unique-location-churn (react)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/shared.ts b/benchmarks/memory/client/scenarios/unique-location-churn/shared.ts index 3b36c7811e..a902391cee 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/shared.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/shared.ts @@ -27,9 +27,11 @@ type NavigationRouter = { subscribe: (event: 'onRendered', listener: () => void) => () => void } -const uniqueLocationChurnIterations = 300 -// Module-level so ids stay unique across runner invocations on one mount; the -// counter prefix removes any residual LCG birthday-collision risk. +const uniqueLocationChurnIterations = 600 +const uniqueLocationChurnWarmupIterations = uniqueLocationChurnIterations +// Module-level within the isolated process so ids stay unique throughout the +// inner loop; the counter prefix removes any residual LCG collision risk. A +// fresh CodSpeed invocation replays the same sequence in a fresh router. const benchmarkRandom = createDeterministicRandom(0xdecafbad) let locationCounter = 0 @@ -122,17 +124,34 @@ export function createWorkload( navigateTo = uninitialized } + async function runLocationLoop( + iterations: number, + createLocation: () => ItemLocation, + ) { + for (let index = 0; index < iterations; index++) { + await navigateTo(createLocation()) + } + } + return { name: `mem client unique-location-churn (${framework})`, before, navigate: (location: ItemLocation) => navigateTo(location), - async run() { - for (let index = 0; index < uniqueLocationChurnIterations; index++) { + run: () => + runLocationLoop(uniqueLocationChurnIterations, () => { const id = `${(locationCounter++).toString(36)}-${randomSegment(benchmarkRandom)}` const q = `q-${randomSegment(benchmarkRandom)}` - await navigateTo({ id, q }) - } + return { id, q } + }), + warmup() { + const random = createDeterministicRandom(0x10ca7100) + let counter = 0 + + return runLocationLoop(uniqueLocationChurnWarmupIterations, () => ({ + id: `warmup-${(counter++).toString(36)}-${randomSegment(random)}`, + q: `q-warmup-${randomSegment(random)}`, + })) }, async sanity() { await before() diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/unique-location-churn/solid/memory.bench.ts index e645ab38f1..ee13bb4439 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/solid/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client unique-location-churn (solid)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts b/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts index d4acf0019a..ef8519cd5c 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts @@ -28,7 +28,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client unique-location-churn (solid)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/unique-location-churn/vue/memory.bench.ts index e645ab38f1..f1b9c40669 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/vue/memory.bench.ts @@ -1,21 +1,6 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-client/bench-utils' -import { workload } from './setup' +import { registerIsolatedClientMemoryBenchmark } from '#memory-client/isolated-benchmark' -await workload.sanity() - -describe('memory', () => { - if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) - - bench(workload.name, workload.run, { - ...memoryBenchOptions, - setup: workload.before, - teardown: workload.after, - }) - return - } - - bench(workload.name, workload.run, memoryBenchOptions) +registerIsolatedClientMemoryBenchmark({ + name: 'mem client unique-location-churn (vue)', + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts b/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts index 423126019b..7e2f840055 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts @@ -30,7 +30,6 @@ export default defineConfig({ test: { name: '@benchmarks/memory-client unique-location-churn (vue)', watch: false, - environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], + environment: 'node', }, }) diff --git a/benchmarks/memory/client/tsconfig.json b/benchmarks/memory/client/tsconfig.json index 9280d5c42f..9a1165754a 100644 --- a/benchmarks/memory/client/tsconfig.json +++ b/benchmarks/memory/client/tsconfig.json @@ -1,7 +1,13 @@ { "extends": "../../../tsconfig.json", "compilerOptions": { + "allowImportingTsExtensions": true, "types": ["node", "vite/client", "vitest/globals"] }, - "include": ["bench-utils.ts", "lifecycle.ts", "vitest.setup.ts"] + "include": [ + "bench-utils.ts", + "isolated-benchmark.ts", + "lifecycle.ts", + "vitest.setup.ts" + ] } diff --git a/benchmarks/memory/server/benchmark.ts b/benchmarks/memory/server/benchmark.ts index 301a9f956d..3dc330876b 100644 --- a/benchmarks/memory/server/benchmark.ts +++ b/benchmarks/memory/server/benchmark.ts @@ -5,5 +5,6 @@ export interface ServerMemoryWorkload { export interface ServerMemoryWorkloadGroup { sanity: () => Promise | void + warmup?: () => Promise | void workloads: Array } diff --git a/benchmarks/memory/server/flame-runner.ts b/benchmarks/memory/server/flame-runner.ts index 1930085aad..6c32e65cbb 100644 --- a/benchmarks/memory/server/flame-runner.ts +++ b/benchmarks/memory/server/flame-runner.ts @@ -5,6 +5,7 @@ export async function runServerFlameBenchmark( workloadGroup: ServerMemoryWorkloadGroup, ) { await workloadGroup.sanity() + await workloadGroup.warmup?.() for (const workload of workloadGroup.workloads) { await profileFlameWorkload(workload.run, workload.name) diff --git a/benchmarks/memory/server/isolated-benchmark.ts b/benchmarks/memory/server/isolated-benchmark.ts new file mode 100644 index 0000000000..18b4dd2dad --- /dev/null +++ b/benchmarks/memory/server/isolated-benchmark.ts @@ -0,0 +1,44 @@ +import { afterEach, beforeEach, bench, describe } from 'vitest' +import { IsolatedMemoryProcess } from '../shared/isolated-process.ts' +import { memoryBenchOptions } from './bench-utils.ts' +import type { IsolatedMemoryBenchmarkKind } from '../shared/isolated-process.ts' + +type RegisterIsolatedServerMemoryBenchmarksOptions = { + names: Array + setupUrl: URL +} + +const kind = 'server' satisfies IsolatedMemoryBenchmarkKind +const isolatedProcessSetupTimeout = 60_000 + +export function registerIsolatedServerMemoryBenchmarks( + options: RegisterIsolatedServerMemoryBenchmarksOptions, +) { + const isolatedProcess = new IsolatedMemoryProcess({ + kind, + setupUrl: options.setupUrl, + workloadNames: options.names, + }) + + const run = async (workloadIndex: number) => { + try { + await isolatedProcess.run(workloadIndex) + } catch (error) { + await isolatedProcess.stop().catch(() => {}) + throw error + } + } + + describe('memory', () => { + beforeEach(() => isolatedProcess.start(), isolatedProcessSetupTimeout) + afterEach(() => isolatedProcess.stop()) + + for (const [workloadIndex, name] of options.names.entries()) { + bench(name, () => run(workloadIndex), { + ...memoryBenchOptions, + setup: () => isolatedProcess.start(), + teardown: () => isolatedProcess.stop(), + }) + } + }) +} diff --git a/benchmarks/memory/server/isolated-process.test.ts b/benchmarks/memory/server/isolated-process.test.ts new file mode 100644 index 0000000000..271c507a27 --- /dev/null +++ b/benchmarks/memory/server/isolated-process.test.ts @@ -0,0 +1,149 @@ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { IsolatedMemoryProcess } from '../shared/isolated-process.ts' + +const setupUrl = new URL( + './test-fixtures/isolated-process-setup.ts', + import.meta.url, +) + +describe('IsolatedMemoryProcess', () => { + let logPath: string + let runner: IsolatedMemoryProcess | undefined + let tempDirectory: string + + beforeEach(async () => { + tempDirectory = await mkdtemp(join(tmpdir(), 'router-memory-isolation-')) + logPath = join(tempDirectory, 'events.log') + await writeFile(logPath, '') + process.env.TSR_MEMORY_ISOLATION_TEST_LOG = logPath + }) + + afterEach(async () => { + await runner?.stop() + delete process.env.TSR_MEMORY_ISOLATION_TEST_LOG + await rm(tempDirectory, { recursive: true }) + }) + + function createRunner( + workloadNames = [ + 'fixture zero', + 'fixture one', + 'fixture failure', + 'fixture exec argv', + ], + ) { + runner = new IsolatedMemoryProcess({ + kind: 'server', + setupUrl, + workloadNames, + }) + + return runner + } + + function createClientRunner() { + runner = new IsolatedMemoryProcess({ + kind: 'client', + setupUrl, + workloadNames: ['fixture client'], + }) + + return runner + } + + async function readEvents() { + return (await readFile(logPath, 'utf8')).trim().split('\n') + } + + it('starts every invocation in a fresh process and waits for work to finish', async () => { + const processRunner = createRunner() + + await processRunner.start() + const firstPid = processRunner.pid + await processRunner.run(1) + + expect(await readEvents()).toEqual([ + `sanity:${firstPid}`, + `warmup:${firstPid}`, + `prime:${firstPid}`, + `run-1-finished:${firstPid}`, + ]) + + await processRunner.stop() + await processRunner.start() + const secondPid = processRunner.pid + await processRunner.run(0) + + expect(secondPid).not.toBe(firstPid) + expect(await readEvents()).toEqual([ + `sanity:${firstPid}`, + `warmup:${firstPid}`, + `prime:${firstPid}`, + `run-1-finished:${firstPid}`, + `sanity:${secondPid}`, + `warmup:${secondPid}`, + `prime:${secondPid}`, + `run-0:${secondPid}`, + ]) + }) + + it('warms a disposable client app before creating measured state', async () => { + const processRunner = createClientRunner() + + await processRunner.start() + const pid = processRunner.pid + + expect(await readEvents()).toEqual([ + `client-sanity:${pid}`, + `client-before:${pid}`, + `client-warmup:${pid}`, + `client-after:${pid}`, + `client-before:${pid}`, + `prime:${pid}`, + ]) + + await processRunner.run(0) + await processRunner.stop() + + expect(await readEvents()).toEqual([ + `client-sanity:${pid}`, + `client-before:${pid}`, + `client-warmup:${pid}`, + `client-after:${pid}`, + `client-before:${pid}`, + `prime:${pid}`, + `client-run:${pid}`, + `client-after:${pid}`, + ]) + }) + + it('propagates workload failures from the child', async () => { + const processRunner = createRunner() + await processRunner.start() + + await expect(processRunner.run(2)).rejects.toThrow( + 'fixture workload failed', + ) + }) + + it('starts the child with deterministic V8 flags', async () => { + const processRunner = createRunner() + await processRunner.start() + + await expect(processRunner.run(3)).rejects.toThrow( + /fixture exec argv:.*--expose-gc.*--predictable.*--no-opt.*--no-flush-bytecode.*--initial-old-space-size=64.*--min-semi-space-size=16.*--max-semi-space-size=16/, + ) + }) + + it('rejects a workload-name mismatch during setup', async () => { + const processRunner = createRunner(['wrong name']) + + await expect(processRunner.start()).rejects.toThrow( + 'Isolated memory workload names did not match', + ) + expect(processRunner.pid).toBeUndefined() + }) +}) diff --git a/benchmarks/memory/server/package.json b/benchmarks/memory/server/package.json index ae09eb8511..3e2a2ecb7f 100644 --- a/benchmarks/memory/server/package.json +++ b/benchmarks/memory/server/package.json @@ -8,7 +8,8 @@ "imports": { "#memory-server/benchmark": "./benchmark.ts", "#memory-server/bench-utils": "./bench-utils.ts", - "#memory-server/flame-runner": "./flame-runner.ts" + "#memory-server/flame-runner": "./flame-runner.ts", + "#memory-server/isolated-benchmark": "./isolated-benchmark.ts" }, "dependencies": { "@tanstack/react-router": "workspace:*", @@ -234,9 +235,25 @@ "cwd": "benchmarks/memory/server" } }, + "test:unit": { + "executor": "nx:run-commands", + "cache": false, + "options": { + "command": "vitest run ./isolated-process.test.ts", + "cwd": "benchmarks/memory/server" + } + }, + "test:types:isolated": { + "executor": "nx:run-commands", + "options": { + "command": "tsc -p ./tsconfig.json --noEmit", + "cwd": "benchmarks/memory/server" + } + }, "test:types": { "executor": "nx:noop", "dependsOn": [ + "test:types:isolated", { "projects": [ "@benchmarks/memory-server-request-churn-react", diff --git a/benchmarks/memory/server/scenarios/aborted-requests/react/memory.bench.ts b/benchmarks/memory/server/scenarios/aborted-requests/react/memory.bench.ts index 9a5c211fee..7d0b9ffca5 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/react/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/react/memory.bench.ts @@ -1,11 +1,6 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: ['mem server aborted-requests (react)'], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/scenarios/aborted-requests/shared.ts b/benchmarks/memory/server/scenarios/aborted-requests/shared.ts index c98f462538..a604e18ce6 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/shared.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/shared.ts @@ -13,7 +13,8 @@ type AbortedRequestMode = { cancelMode: AbortedRequestCancelMode } -const abortedRequestIterations = 40 +const abortedRequestIterations = 80 +const abortedRequestWarmupIterations = abortedRequestIterations let abortedRequestCounter = 0 const eagerMarker = 'data-bench="aborted-requests-eager"' const alphaFallbackMarker = 'data-bench="aborted-requests-alpha-fallback"' @@ -234,10 +235,12 @@ async function assertAbortedRequestsSanity( async function runAbortedRequestLoop( handler: StartRequestHandler, mode: AbortedRequestMode, + iterations: number, + createId: (index: number) => string, ) { - for (let index = 0; index < abortedRequestIterations; index++) { + for (let index = 0; index < iterations; index++) { const controller = new AbortController() - const id = `abort-${(abortedRequestCounter++).toString(36)}` + const id = createId(index) const request = buildStreamRequest(id, controller.signal) const response = await handler.fetch(request) validateDocumentResponse(response, request) @@ -259,10 +262,23 @@ export function createWorkloadGroup( handler: StartRequestHandler, ) { const mode = abortedRequestModes[framework] - const run = () => runAbortedRequestLoop(handler, mode) + const run = () => + runAbortedRequestLoop( + handler, + mode, + abortedRequestIterations, + () => `abort-${(abortedRequestCounter++).toString(36)}`, + ) return { sanity: () => assertAbortedRequestsSanity(handler, mode), + warmup: () => + runAbortedRequestLoop( + handler, + mode, + abortedRequestWarmupIterations, + (index) => `warmup-abort-${index.toString(36)}`, + ), workloads: [ { name: `mem server aborted-requests (${framework})`, diff --git a/benchmarks/memory/server/scenarios/aborted-requests/solid/memory.bench.ts b/benchmarks/memory/server/scenarios/aborted-requests/solid/memory.bench.ts index 9a5c211fee..a33b097a5d 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/solid/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/solid/memory.bench.ts @@ -1,11 +1,6 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: ['mem server aborted-requests (solid)'], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/scenarios/aborted-requests/vue/memory.bench.ts b/benchmarks/memory/server/scenarios/aborted-requests/vue/memory.bench.ts index 9a5c211fee..2e51326d4e 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/vue/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/vue/memory.bench.ts @@ -1,11 +1,6 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: ['mem server aborted-requests (vue)'], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/scenarios/error-paths/react/memory.bench.ts b/benchmarks/memory/server/scenarios/error-paths/react/memory.bench.ts index 9a5c211fee..d4f96573e8 100644 --- a/benchmarks/memory/server/scenarios/error-paths/react/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/error-paths/react/memory.bench.ts @@ -1,11 +1,11 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: [ + 'mem server error-paths redirect (react)', + 'mem server error-paths not-found (react)', + 'mem server error-paths error (react)', + 'mem server error-paths unmatched (react)', + ], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/scenarios/error-paths/shared.ts b/benchmarks/memory/server/scenarios/error-paths/shared.ts index 57e229b7e3..4dc35c906b 100644 --- a/benchmarks/memory/server/scenarios/error-paths/shared.ts +++ b/benchmarks/memory/server/scenarios/error-paths/shared.ts @@ -9,9 +9,11 @@ export type { StartRequestHandler } type Framework = 'react' | 'solid' | 'vue' -// Sized to sit just above the 2s measured-run floor on CI (per-iteration -// cost with the pinned collection is ~0.13-0.19s across frameworks). -const errorPathsIterations = 18 +// Twice the count needed for the original ~2s CI floor (per-iteration cost +// with the pinned collection is ~0.13-0.19s across frameworks), so the regular +// loop shape dominates the timeline and an accumulating leak is amplified. +const errorPathsIterations = 36 +const errorPathsWarmupIterations = errorPathsIterations const redirectSeed = 0xdecafbad const notFoundSeed = 0xdecafb0d const errorSeed = 0xdecafbed @@ -19,7 +21,8 @@ const unmatchedSeed = 0xdecaf00d const redirectStatus = 302 const notFoundStatus = 404 const errorStatus = 500 -// Module-level so each error-path bench keeps advancing across runner invocations. +// Module-level within the isolated process so each error-path inner loop uses +// unique URLs. Fresh CodSpeed invocations replay the sequence in a fresh handler. const redirectRandom = createDeterministicRandom(redirectSeed) const notFoundRandom = createDeterministicRandom(notFoundSeed) const errorRandom = createDeterministicRandom(errorSeed) @@ -134,6 +137,28 @@ async function assertErrorPathsSanity(handler: StartRequestHandler) { ) } +function runErrorPathWarmup( + handler: StartRequestHandler, + options: { + path: string + seed: number + validateResponse: (response: Response, request: Request) => void + }, +) { + let counter = 0 + + return runSequentialRequestLoop(handler, { + seed: options.seed, + iterations: errorPathsWarmupIterations, + buildRequest: (random) => { + const id = `warmup-${(counter++).toString(36)}-${randomSegment(random)}` + return new Request(`http://localhost/${options.path}/${id}`, requestInit) + }, + validateResponse: options.validateResponse, + pinGcBetweenIterations: true, + }) +} + export function createWorkloadGroup( framework: Framework, handler: StartRequestHandler, @@ -174,8 +199,32 @@ export function createWorkloadGroup( pinGcBetweenIterations: true, }) + async function warmup() { + await runErrorPathWarmup(handler, { + path: 'from', + seed: 0x7ed1ec71, + validateResponse: validateRedirectResponse, + }) + await runErrorPathWarmup(handler, { + path: 'missing', + seed: 0x7ed1ec72, + validateResponse: validateNotFoundResponse, + }) + await runErrorPathWarmup(handler, { + path: 'boom', + seed: 0x7ed1ec73, + validateResponse: validateErrorResponse, + }) + await runErrorPathWarmup(handler, { + path: 'nope', + seed: 0x7ed1ec74, + validateResponse: validateNotFoundResponse, + }) + } + return { sanity: () => assertErrorPathsSanity(handler), + warmup, workloads: [ { name: `mem server error-paths redirect (${framework})`, diff --git a/benchmarks/memory/server/scenarios/error-paths/solid/memory.bench.ts b/benchmarks/memory/server/scenarios/error-paths/solid/memory.bench.ts index 9a5c211fee..6756a11232 100644 --- a/benchmarks/memory/server/scenarios/error-paths/solid/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/error-paths/solid/memory.bench.ts @@ -1,11 +1,11 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: [ + 'mem server error-paths redirect (solid)', + 'mem server error-paths not-found (solid)', + 'mem server error-paths error (solid)', + 'mem server error-paths unmatched (solid)', + ], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/scenarios/error-paths/vue/memory.bench.ts b/benchmarks/memory/server/scenarios/error-paths/vue/memory.bench.ts index 9a5c211fee..0cd0328be2 100644 --- a/benchmarks/memory/server/scenarios/error-paths/vue/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/error-paths/vue/memory.bench.ts @@ -1,11 +1,11 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: [ + 'mem server error-paths redirect (vue)', + 'mem server error-paths not-found (vue)', + 'mem server error-paths error (vue)', + 'mem server error-paths unmatched (vue)', + ], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/scenarios/request-churn/react/memory.bench.ts b/benchmarks/memory/server/scenarios/request-churn/react/memory.bench.ts index 9a5c211fee..b114a1fe33 100644 --- a/benchmarks/memory/server/scenarios/request-churn/react/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/request-churn/react/memory.bench.ts @@ -1,11 +1,6 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: ['mem server request-churn (react)'], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/scenarios/request-churn/shared.ts b/benchmarks/memory/server/scenarios/request-churn/shared.ts index a196eec8b5..6e6aefff42 100644 --- a/benchmarks/memory/server/scenarios/request-churn/shared.ts +++ b/benchmarks/memory/server/scenarios/request-churn/shared.ts @@ -10,9 +10,12 @@ export type { StartRequestHandler } type Framework = 'react' | 'solid' | 'vue' const benchmarkSeed = 0xdecafbad -const requestChurnIterations = 40 +const requestChurnIterations = 80 +const requestChurnWarmupIterations = requestChurnIterations const itemPageMarker = 'data-bench="request-churn-item"' -// Module-level so CodSpeed warmups and measurement never replay URLs. +// Module-level within the isolated process so URLs stay unique throughout the +// inner loop. Every fresh CodSpeed invocation deliberately replays this same +// sequence against a fresh handler process. const benchmarkRandom = createDeterministicRandom(benchmarkSeed) let requestCounter = 0 @@ -54,14 +57,17 @@ export function createWorkloadGroup( framework: Framework, handler: StartRequestHandler, ) { - function buildItemRequest(random: () => number) { - const counter = (requestCounter++).toString(36) + function createItemRequest(random: () => number, counter: string) { const id = `${counter}-${randomSegment(random)}` const q = `q-${randomSegment(random)}` return new Request(`http://localhost/items/${id}?q=${q}`, requestInit) } + function buildItemRequest(random: () => number) { + return createItemRequest(random, (requestCounter++).toString(36)) + } + const run = () => runSequentialRequestLoop(handler, { random: benchmarkRandom, @@ -71,8 +77,22 @@ export function createWorkloadGroup( pinGcBetweenIterations: true, }) + const warmup = () => { + let counter = 0 + + return runSequentialRequestLoop(handler, { + seed: 0x5e7a11ce, + iterations: requestChurnWarmupIterations, + buildRequest: (random) => + createItemRequest(random, `warmup-${(counter++).toString(36)}`), + validateResponse: validateItemResponse, + pinGcBetweenIterations: true, + }) + } + return { sanity: () => assertRequestChurnSanity(handler), + warmup, workloads: [ { name: `mem server request-churn (${framework})`, diff --git a/benchmarks/memory/server/scenarios/request-churn/solid/memory.bench.ts b/benchmarks/memory/server/scenarios/request-churn/solid/memory.bench.ts index 9a5c211fee..e57ed4195c 100644 --- a/benchmarks/memory/server/scenarios/request-churn/solid/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/request-churn/solid/memory.bench.ts @@ -1,11 +1,6 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: ['mem server request-churn (solid)'], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/scenarios/request-churn/vue/memory.bench.ts b/benchmarks/memory/server/scenarios/request-churn/vue/memory.bench.ts index 9a5c211fee..f05a9f6fff 100644 --- a/benchmarks/memory/server/scenarios/request-churn/vue/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/request-churn/vue/memory.bench.ts @@ -1,11 +1,6 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: ['mem server request-churn (vue)'], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/react/memory.bench.ts b/benchmarks/memory/server/scenarios/server-fn-churn/react/memory.bench.ts index 9a5c211fee..8b2da7cade 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/react/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/react/memory.bench.ts @@ -1,11 +1,6 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: ['mem server server-fn-churn (react)'], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts b/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts index 6e2959e757..14761d6909 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts @@ -37,10 +37,12 @@ type SerovalNode = const benchmarkSeed = 0xdecafbad const payloadSeed = 0x51f0cafe -const fixtureCount = 16 -// Sized to sit just above the 2s measured-run floor on CI (per-iteration -// cost with the pinned collection is ~0.08-0.11s across frameworks). -const serverFnChurnIterations = 30 +// Twice the count needed for the original ~2s CI floor (per-iteration cost +// with the pinned collection is ~0.08-0.11s across frameworks), so the regular +// loop shape dominates the timeline and an accumulating leak is amplified. +const serverFnChurnIterations = 60 +const serverFnChurnWarmupIterations = serverFnChurnIterations +const fixtureCount = Math.ceil(serverFnChurnIterations / 2) const origin = 'http://localhost' const tssContentTypeFramed = 'application/x-tss-framed' const acceptHeader = `${tssContentTypeFramed}, application/x-ndjson, application/json` @@ -88,13 +90,21 @@ function serializePayload(id: string) { }) } -function createFixtures(kind: 'get' | 'post') { - const random = createDeterministicRandom(payloadSeed ^ kind.length) - - return Array.from({ length: fixtureCount }, (_, index): PayloadFixture => { - const id = [kind, index, randomSegment(random), randomSegment(random)].join( - '-', - ) +function createFixtures( + kind: 'get' | 'post', + seed = payloadSeed ^ kind.length, + prefix: string = kind, + count = fixtureCount, +) { + const random = createDeterministicRandom(seed) + + return Array.from({ length: count }, (_, index): PayloadFixture => { + const id = [ + prefix, + index, + randomSegment(random), + randomSegment(random), + ].join('-') const body = serializePayload(id) return { @@ -107,6 +117,15 @@ function createFixtures(kind: 'get' | 'post') { const getFixtures = createFixtures('get') const postFixtures = createFixtures('post') +const sanityGetFixture = createFixtures('get', 0x51f0ca01, 'sanity-get', 1)[0]! +const sanityPostFixture = createFixtures( + 'post', + 0x51f0ca02, + 'sanity-post', + 1, +)[0]! +const warmupGetFixtures = createFixtures('get', 0x51f0ca11, 'warmup-get') +const warmupPostFixtures = createFixtures('post', 0x51f0ca12, 'warmup-post') async function discoverUrls(handler: StartRequestHandler) { const response = await handler.fetch(new Request(`${origin}/api/fn-urls`)) @@ -182,7 +201,7 @@ async function assertServerFnChurnSanity( handler: StartRequestHandler, urls: FnUrls, ) { - const getFixture = getFixtures[0]! + const getFixture = sanityGetFixture const getRequest = buildGetRequest(urls.get, getFixture) const getResponse = await handler.fetch(getRequest) const getBody = await getResponse.text() @@ -190,7 +209,7 @@ async function assertServerFnChurnSanity( validateServerFnResponse(getResponse, getRequest) validateEchoedBody(getBody, getRequest, getFixture.id) - const postFixture = postFixtures[0]! + const postFixture = sanityPostFixture const postRequest = buildPostRequest(urls.post, postFixture) const postResponse = await handler.fetch(postRequest) const postBody = await postResponse.text() @@ -199,32 +218,57 @@ async function assertServerFnChurnSanity( validateEchoedBody(postBody, postRequest, postFixture.id) } +function runServerFnLoop( + handler: StartRequestHandler, + urls: FnUrls, + options: { + iterations: number + seed: number + getFixtures: ReadonlyArray + postFixtures: ReadonlyArray + }, +) { + return runSequentialRequestLoop(handler, { + seed: options.seed, + iterations: options.iterations, + pinGcBetweenIterations: true, + buildRequest: (_random, index) => { + const fixtureIndex = Math.floor(index / 2) % fixtureCount + + if (index % 2 === 0) { + const fixture = options.getFixtures[fixtureIndex]! + return buildGetRequest(urls.get, fixture) + } else { + const fixture = options.postFixtures[fixtureIndex]! + return buildPostRequest(urls.post, fixture) + } + }, + validateResponse: validateServerFnResponse, + }) +} + export async function createWorkloadGroup( framework: Framework, handler: StartRequestHandler, ) { const urls = await discoverUrls(handler) const run = () => - runSequentialRequestLoop(handler, { + runServerFnLoop(handler, urls, { seed: benchmarkSeed, iterations: serverFnChurnIterations, - pinGcBetweenIterations: true, - buildRequest: (_random, index) => { - const fixtureIndex = Math.floor(index / 2) % fixtureCount - - if (index % 2 === 0) { - const fixture = getFixtures[fixtureIndex]! - return buildGetRequest(urls.get, fixture) - } else { - const fixture = postFixtures[fixtureIndex]! - return buildPostRequest(urls.post, fixture) - } - }, - validateResponse: validateServerFnResponse, + getFixtures, + postFixtures, }) return { sanity: () => assertServerFnChurnSanity(handler, urls), + warmup: () => + runServerFnLoop(handler, urls, { + seed: 0x5e7f0c11, + iterations: serverFnChurnWarmupIterations, + getFixtures: warmupGetFixtures, + postFixtures: warmupPostFixtures, + }), workloads: [ { name: `mem server server-fn-churn (${framework})`, diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/solid/memory.bench.ts b/benchmarks/memory/server/scenarios/server-fn-churn/solid/memory.bench.ts index 9a5c211fee..663e068602 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/solid/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/solid/memory.bench.ts @@ -1,11 +1,6 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: ['mem server server-fn-churn (solid)'], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/vue/memory.bench.ts b/benchmarks/memory/server/scenarios/server-fn-churn/vue/memory.bench.ts index 9a5c211fee..734a80d415 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/vue/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/vue/memory.bench.ts @@ -1,11 +1,6 @@ -import { bench, describe } from 'vitest' -import { memoryBenchOptions } from '#memory-server/bench-utils' -import { workloadGroup } from './setup' +import { registerIsolatedServerMemoryBenchmarks } from '#memory-server/isolated-benchmark' -await workloadGroup.sanity() - -describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } +registerIsolatedServerMemoryBenchmarks({ + names: ['mem server server-fn-churn (vue)'], + setupUrl: new URL('./setup.ts', import.meta.url), }) diff --git a/benchmarks/memory/server/test-fixtures/isolated-process-setup.ts b/benchmarks/memory/server/test-fixtures/isolated-process-setup.ts new file mode 100644 index 0000000000..7e2e439838 --- /dev/null +++ b/benchmarks/memory/server/test-fixtures/isolated-process-setup.ts @@ -0,0 +1,66 @@ +import { appendFileSync } from 'node:fs' +import { appendFile } from 'node:fs/promises' + +function getLogPath() { + const logPath = process.env.TSR_MEMORY_ISOLATION_TEST_LOG + + if (!logPath) { + throw new Error('Missing TSR_MEMORY_ISOLATION_TEST_LOG') + } + + return logPath +} + +async function log(event: string) { + await appendFile(getLogPath(), `${event}:${process.pid}\n`) +} + +process.on('message', (value: unknown) => { + if ( + typeof value === 'object' && + value !== null && + 'type' in value && + value.type === 'prime' + ) { + appendFileSync(getLogPath(), `prime:${process.pid}\n`) + } +}) + +export const workloadGroup = { + sanity: () => log('sanity'), + warmup: () => log('warmup'), + workloads: [ + { + name: 'fixture zero', + run: () => log('run-0'), + }, + { + name: 'fixture one', + async run() { + await new Promise((resolve) => setTimeout(resolve, 10)) + await log('run-1-finished') + }, + }, + { + name: 'fixture failure', + run() { + throw new Error('fixture workload failed') + }, + }, + { + name: 'fixture exec argv', + run() { + throw new Error(`fixture exec argv: ${process.execArgv.join(' ')}`) + }, + }, + ], +} + +export const workload = { + name: 'fixture client', + sanity: () => log('client-sanity'), + before: () => log('client-before'), + warmup: () => log('client-warmup'), + run: () => log('client-run'), + after: () => log('client-after'), +} diff --git a/benchmarks/memory/server/tsconfig.json b/benchmarks/memory/server/tsconfig.json index 5d5dcbf825..dde9ff235a 100644 --- a/benchmarks/memory/server/tsconfig.json +++ b/benchmarks/memory/server/tsconfig.json @@ -1,7 +1,14 @@ { "extends": "../../../tsconfig.json", "compilerOptions": { + "allowImportingTsExtensions": true, "types": ["node", "vite/client", "vitest/globals"] }, - "include": ["bench-utils.ts"] + "include": [ + "bench-utils.ts", + "isolated-benchmark.ts", + "isolated-process.test.ts", + "test-fixtures/**/*.ts", + "../shared/**/*.ts" + ] } diff --git a/benchmarks/memory/shared/isolated-process-child.ts b/benchmarks/memory/shared/isolated-process-child.ts new file mode 100644 index 0000000000..68a7494274 --- /dev/null +++ b/benchmarks/memory/shared/isolated-process-child.ts @@ -0,0 +1,285 @@ +import process from 'node:process' +import { warmClientMemoryWorkload } from '../client/benchmark.ts' +import type { ClientMemoryWorkload } from '../client/benchmark.ts' +import type { ServerMemoryWorkloadGroup } from '../server/benchmark.ts' +import type { + IsolatedMemoryBenchmarkKind, + IsolatedMemoryChildMessage, + IsolatedMemoryParentMessage, +} from './isolated-process.ts' + +type RunnableWorkload = { + name: string + run: () => Promise | void +} + +type LoadedWorkloads = { + cleanup: () => Promise + workloads: Array +} + +const preparationSettleTurns = 16 +const completionSettleTurns = 4 + +function serializeError(error: unknown) { + if (error instanceof Error) { + return { + message: error.message, + name: error.name, + stack: error.stack, + } + } + + return { + message: String(error), + name: 'Error', + } +} + +function isParentMessage(value: unknown): value is IsolatedMemoryParentMessage { + if ( + typeof value !== 'object' || + value === null || + !('type' in value) || + !('requestId' in value) || + typeof value.requestId !== 'number' + ) { + return false + } + + if (value.type === 'prime' || value.type === 'stop') { + return true + } + + return ( + value.type === 'run' && + 'workloadIndex' in value && + typeof value.workloadIndex === 'number' + ) +} + +function send(message: IsolatedMemoryChildMessage) { + return new Promise((resolve, reject) => { + if (!process.send) { + reject(new Error('The isolated memory process requires an IPC channel')) + return + } + + process.send(message, (error) => { + if (error) { + reject(error) + } else { + resolve() + } + }) + }) +} + +async function settle(turns: number) { + for (let turn = 0; turn < turns; turn++) { + await new Promise((resolve) => setTimeout(resolve, 0)) + } +} + +async function prepareForMeasurement() { + await settle(preparationSettleTurns) + + if (!globalThis.gc) { + throw new Error('The isolated memory process requires --expose-gc') + } + + globalThis.gc() + await settle(1) + globalThis.gc() + await settle(1) +} + +function isClientMemoryWorkload(value: unknown): value is ClientMemoryWorkload { + return ( + typeof value === 'object' && + value !== null && + 'run' in value && + typeof value.run === 'function' && + 'name' in value && + typeof value.name === 'string' && + 'sanity' in value && + typeof value.sanity === 'function' + ) +} + +function isServerMemoryWorkloadGroup( + value: unknown, +): value is ServerMemoryWorkloadGroup { + return ( + typeof value === 'object' && + value !== null && + 'sanity' in value && + typeof value.sanity === 'function' && + 'workloads' in value && + Array.isArray(value.workloads) && + value.workloads.every( + (workload) => + typeof workload === 'object' && + workload !== null && + 'run' in workload && + typeof workload.run === 'function' && + 'name' in workload && + typeof workload.name === 'string', + ) + ) +} + +async function loadClientWorkload(setupUrl: string): Promise { + const { window } = await import('../client/jsdom.ts') + Object.defineProperty(globalThis, 'IS_REACT_ACT_ENVIRONMENT', { + configurable: true, + value: true, + writable: true, + }) + + let workload: ClientMemoryWorkload | undefined + + try { + const setupModule = (await import(setupUrl)) as { workload?: unknown } + + if (!isClientMemoryWorkload(setupModule.workload)) { + throw new Error(`Expected ${setupUrl} to export a client workload`) + } + + workload = setupModule.workload + await workload.sanity() + await warmClientMemoryWorkload(workload) + await workload.before?.() + + return { + workloads: [workload], + async cleanup() { + try { + await workload?.after?.() + } finally { + window.close() + } + }, + } + } catch (error) { + try { + await workload?.after?.() + } finally { + window.close() + } + + throw error + } +} + +async function loadServerWorkloads(setupUrl: string): Promise { + const setupModule = (await import(setupUrl)) as { workloadGroup?: unknown } + + if (!isServerMemoryWorkloadGroup(setupModule.workloadGroup)) { + throw new Error(`Expected ${setupUrl} to export a server workload group`) + } + + const workloadGroup = setupModule.workloadGroup + await workloadGroup.sanity() + await workloadGroup.warmup?.() + + return { + workloads: workloadGroup.workloads, + async cleanup() {}, + } +} + +function parseKind(value: string | undefined): IsolatedMemoryBenchmarkKind { + if (value === 'client' || value === 'server') { + return value + } + + throw new Error(`Invalid isolated memory benchmark kind: ${value}`) +} + +async function main() { + const kind = parseKind(process.argv[2]) + const setupUrl = process.argv[3] + + if (!setupUrl) { + throw new Error('Missing isolated memory benchmark setup URL') + } + + const loaded = + kind === 'client' + ? await loadClientWorkload(setupUrl) + : await loadServerWorkloads(setupUrl) + + let commandQueue = Promise.resolve() + let primed = false + let stopping = false + + process.on('message', (value: unknown) => { + if (stopping || !isParentMessage(value)) { + return + } + + const message = value + + commandQueue = commandQueue.then(async () => { + try { + if (message.type === 'prime') { + if (primed) { + throw new Error('The isolated memory process is already primed') + } + + await prepareForMeasurement() + primed = true + await send({ type: 'primed', requestId: message.requestId }) + return + } + + if (message.type === 'run') { + if (!primed) { + throw new Error('The isolated memory process is not primed') + } + + const workload = loaded.workloads[message.workloadIndex] + + if (!workload) { + throw new Error( + `Invalid isolated memory workload index ${message.workloadIndex}`, + ) + } + + await workload.run() + await settle(completionSettleTurns) + await send({ type: 'complete', requestId: message.requestId }) + return + } + + stopping = true + await loaded.cleanup() + await send({ type: 'stopped', requestId: message.requestId }) + process.exit(0) + } catch (error) { + await send({ + type: 'error', + requestId: message.requestId, + error: serializeError(error), + }) + } + }) + }) + + process.on('disconnect', () => { + process.exit(1) + }) + + await send({ + type: 'ready', + workloadNames: loaded.workloads.map((workload) => workload.name), + }) +} + +try { + await main() +} catch (error) { + await send({ type: 'error', error: serializeError(error) }).catch(() => {}) + process.exit(1) +} diff --git a/benchmarks/memory/shared/isolated-process.ts b/benchmarks/memory/shared/isolated-process.ts new file mode 100644 index 0000000000..d985d251e9 --- /dev/null +++ b/benchmarks/memory/shared/isolated-process.ts @@ -0,0 +1,412 @@ +import { fork } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +export type IsolatedMemoryBenchmarkKind = 'client' | 'server' + +type SerializedError = { + message: string + name: string + stack?: string +} + +type ChildReadyMessage = { + type: 'ready' + workloadNames: Array +} + +type ChildCompleteMessage = { + type: 'complete' + requestId: number +} + +type ChildPrimedMessage = { + type: 'primed' + requestId: number +} + +type ChildStoppedMessage = { + type: 'stopped' + requestId: number +} + +type ChildErrorMessage = { + type: 'error' + requestId?: number + error: SerializedError +} + +export type IsolatedMemoryChildMessage = + | ChildReadyMessage + | ChildCompleteMessage + | ChildPrimedMessage + | ChildStoppedMessage + | ChildErrorMessage + +export type IsolatedMemoryParentMessage = + | { + type: 'prime' + requestId: number + } + | { + type: 'run' + requestId: number + workloadIndex: number + } + | { + type: 'stop' + requestId: number + } + +type IsolatedMemoryProcessOptions = { + kind: IsolatedMemoryBenchmarkKind + setupUrl: URL + workloadNames: Array +} + +const childModulePath = fileURLToPath( + new URL('./isolated-process-child.ts', import.meta.url), +) + +// Inherit CodSpeed and scenario-specific V8 flags from the Vitest worker, then +// add the flags that keep the child heap and compilation lifecycle stable. +// Disabling optimization prevents a workload from crossing a JIT tier-up +// threshold inside the measured loop and injecting a one-off compilation +// allocation into the peak-memory result. +const deterministicChildExecArgv = [ + '--expose-gc', + '--predictable', + '--no-opt', + '--no-flush-bytecode', + '--initial-old-space-size=64', + '--min-semi-space-size=16', + '--max-semi-space-size=16', +] + +function createChildExecArgv() { + const execArgv: Array = [] + + for (let index = 0; index < process.execArgv.length; index++) { + const argument = process.execArgv[index]! + + if ( + argument === '-e' || + argument === '--eval' || + argument === '-p' || + argument === '--print' + ) { + index++ + continue + } + + if ( + argument.startsWith('--eval=') || + argument.startsWith('--print=') || + argument === '--input-type=module' || + argument === '--input-type=commonjs' || + argument === '--check' || + argument === '-c' || + argument === '--test' + ) { + continue + } + + execArgv.push(argument) + } + + for (const argument of deterministicChildExecArgv) { + const flagName = argument.split('=')[0]! + const alreadyPresent = execArgv.some( + (existing) => + existing === flagName || existing.startsWith(`${flagName}=`), + ) + + if (!alreadyPresent) { + execArgv.push(argument) + } + } + + return execArgv +} + +function isChildMessage(value: unknown): value is IsolatedMemoryChildMessage { + return ( + typeof value === 'object' && + value !== null && + 'type' in value && + typeof value.type === 'string' + ) +} + +function deserializeError(error: SerializedError) { + const result = new Error(error.message) + result.name = error.name + result.stack = error.stack + return result +} + +export class IsolatedMemoryProcess { + readonly #options: IsolatedMemoryProcessOptions + #child: ReturnType | undefined + #nextRequestId = 0 + + constructor(options: IsolatedMemoryProcessOptions) { + this.#options = options + } + + get pid() { + return this.#child?.pid + } + + async start() { + if (this.#child) { + throw new Error('The isolated memory process is already running') + } + + this.#nextRequestId = 0 + + const child = fork( + childModulePath, + [this.#options.kind, this.#options.setupUrl.href], + { + env: { + ...process.env, + NODE_ENV: 'production', + }, + execArgv: createChildExecArgv(), + stdio: ['ignore', 'inherit', 'inherit', 'ipc'], + }, + ) + + this.#child = child + + try { + const message = await this.#waitForMessage(child) + + if (message.type === 'error') { + throw deserializeError(message.error) + } + + if (message.type !== 'ready') { + throw new Error( + `Expected isolated memory process to become ready, got ${message.type}`, + ) + } + + if ( + message.workloadNames.length !== this.#options.workloadNames.length || + message.workloadNames.some( + (name, index) => name !== this.#options.workloadNames[index], + ) + ) { + throw new Error( + `Isolated memory workload names did not match: expected ${JSON.stringify(this.#options.workloadNames)}, got ${JSON.stringify(message.workloadNames)}`, + ) + } + + // Exercise both IPC directions and the child's command queue before the + // benchmark marker. The child settles and collects after receiving this + // first inbound command, so its one-time native allocations cannot + // dominate the measured workload's peak. + const primed = await this.#sendTo(child, { + type: 'prime', + requestId: this.#nextRequestId++, + }) + + if (primed.type === 'error') { + throw deserializeError(primed.error) + } + + if (primed.type !== 'primed') { + throw new Error( + `Expected isolated memory process to become primed, got ${primed.type}`, + ) + } + } catch (error) { + child.kill() + this.#child = undefined + throw error + } + } + + async run(workloadIndex: number) { + if ( + !Number.isInteger(workloadIndex) || + workloadIndex < 0 || + workloadIndex >= this.#options.workloadNames.length + ) { + throw new Error(`Invalid isolated memory workload index ${workloadIndex}`) + } + + const message = await this.#send({ + type: 'run', + requestId: this.#nextRequestId++, + workloadIndex, + }) + + if (message.type === 'error') { + throw deserializeError(message.error) + } + + if (message.type !== 'complete') { + throw new Error( + `Expected isolated memory workload to complete, got ${message.type}`, + ) + } + } + + async stop() { + const child = this.#child + + if (!child) { + return + } + + this.#child = undefined + const exit = this.#waitForExit(child) + + try { + const message = await this.#sendTo(child, { + type: 'stop', + requestId: this.#nextRequestId++, + }) + + if (message.type === 'error') { + throw deserializeError(message.error) + } + + if (message.type !== 'stopped') { + throw new Error( + `Expected isolated memory process to stop, got ${message.type}`, + ) + } + + await exit + } finally { + if (child.exitCode === null && child.signalCode === null) { + child.kill() + } + + await exit.catch(() => {}) + } + } + + async #send(message: IsolatedMemoryParentMessage) { + const child = this.#child + + if (!child) { + throw new Error('The isolated memory process is not running') + } + + return this.#sendTo(child, message) + } + + async #sendTo( + child: ReturnType, + message: IsolatedMemoryParentMessage, + ) { + const response = this.#waitForMessage(child, message.requestId) + + try { + await new Promise((resolve, reject) => { + child.send(message, (error) => { + if (error) { + reject(error) + } else { + resolve() + } + }) + }) + } catch (error) { + void response.catch(() => {}) + throw error + } + + return response + } + + #waitForMessage( + child: ReturnType, + requestId?: number, + ): Promise { + return new Promise((resolve, reject) => { + const cleanup = () => { + child.off('error', onError) + child.off('exit', onExit) + child.off('message', onMessage) + } + const onError = (error: Error) => { + cleanup() + reject(error) + } + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + cleanup() + reject( + new Error( + `Isolated memory process exited before responding (code ${code}, signal ${signal})`, + ), + ) + } + const onMessage = (value: unknown) => { + if (!isChildMessage(value)) { + return + } + + if ( + requestId !== undefined && + value.type !== 'error' && + value.type !== 'ready' && + value.requestId !== requestId + ) { + return + } + + if ( + requestId !== undefined && + value.type === 'error' && + value.requestId !== requestId + ) { + return + } + + cleanup() + resolve(value) + } + + child.on('error', onError) + child.on('exit', onExit) + child.on('message', onMessage) + }) + } + + #waitForExit(child: ReturnType): Promise { + if (child.exitCode !== null || child.signalCode !== null) { + return Promise.resolve() + } + + return new Promise((resolve, reject) => { + const cleanup = () => { + child.off('error', onError) + child.off('exit', onExit) + } + const onError = (error: Error) => { + cleanup() + reject(error) + } + const onExit = () => { + cleanup() + resolve() + } + + child.once('error', onError) + child.once('exit', onExit) + + // The process can exit between the initial state check and listener + // registration. Recheck after registering so that transition cannot be + // missed. + if (child.exitCode !== null || child.signalCode !== null) { + cleanup() + resolve() + } + }) + } +} diff --git a/benchmarks/memory/shared/tsconfig.json b/benchmarks/memory/shared/tsconfig.json new file mode 100644 index 0000000000..83513dd9cf --- /dev/null +++ b/benchmarks/memory/shared/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "allowImportingTsExtensions": true, + "types": ["node", "vite/client", "vitest/globals"] + }, + "include": ["*.ts"] +}