From 0ce2b37ad9cb70c68d7c12171220d8e68f561b4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 17 Aug 2026 19:52:11 +0200 Subject: [PATCH 01/15] feat(bench): let the density benchmark measure any published app `resource_benchmark` produces every RSS/PSS/private-dirty/cgroup number Coop publishes, and it could only ever measure one fixture: `locate_prepared_app` hardcoded `target/dynamic-smoke/compiled/test1`, the tiny dependency-free app. That is the wrong app to draw conclusions from. A tiny handler shares almost everything through the providers, so its marginal cost per deployment is close to the floor. A real application does not, and the density claim -- the whole premise of the architecture -- is about real applications. `COOP_BENCH_APP_LIBRARY` now points the benchmark at any published package, with the tiny app unchanged as the default so Linux CI keeps measuring what it has always measured: scripts/prepare-next-benchmark.sh COOP_BENCH_APP_LIBRARY=target/next-benchmark/coop-run/compiled/next-bench//app.so \ cargo test --release -p coop-daemon --test resource_benchmark \ measure_in_process_startup_and_rss -- --ignored --nocapture Replicating the Next fixture measures something real, and it is worth saying why rather than assuming it: Perry compiles the Next route INTO the application dylib, so at load time a copy needs the shared providers and nothing else -- no `node_modules`, no build output. N copies therefore measure the genuine marginal cost of an Nth Next deployment on one box, which is exactly the question the architecture exists to answer. The override asserts the path is a file rather than failing later inside the fixture copy, because a typo'd path would otherwise surface as a confusing manifest error several steps downstream. --- Cargo.lock | 14 +++++----- .../coop-daemon/tests/resource_benchmark.rs | 26 +++++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d5d8aa..fa7e1e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4739,7 +4739,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "perry-diagnostics" -version = "0.5.1510" +version = "0.5.1512" dependencies = [ "serde", "serde_json", @@ -4747,11 +4747,11 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1510" +version = "0.5.1512" [[package]] name = "perry-ffi" -version = "0.5.1510" +version = "0.5.1512" dependencies = [ "dashmap", "once_cell", @@ -4759,7 +4759,7 @@ dependencies = [ [[package]] name = "perry-parser" -version = "0.5.1510" +version = "0.5.1512" dependencies = [ "anyhow", "perry-diagnostics", @@ -4771,7 +4771,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1510" +version = "0.5.1512" dependencies = [ "anyhow", "base64 0.22.1", @@ -4813,7 +4813,7 @@ dependencies = [ [[package]] name = "perry-stdlib" -version = "0.5.1510" +version = "0.5.1512" dependencies = [ "aes 0.8.4", "aes 0.9.2", @@ -4905,7 +4905,7 @@ dependencies = [ [[package]] name = "perry-updater" -version = "0.5.1510" +version = "0.5.1512" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/crates/coop-daemon/tests/resource_benchmark.rs b/crates/coop-daemon/tests/resource_benchmark.rs index 121d01e..4deaa9e 100644 --- a/crates/coop-daemon/tests/resource_benchmark.rs +++ b/crates/coop-daemon/tests/resource_benchmark.rs @@ -329,7 +329,33 @@ async fn measure_in_process_startup_and_rss() { } } +/// Which compiled application this benchmark replicates. +/// +/// Defaults to the tiny dependency-free app, which is what Linux CI has always +/// measured. `COOP_BENCH_APP_LIBRARY` points it at any other published +/// package -- in practice the Next.js fixture, produced by +/// `scripts/prepare-next-benchmark.sh`: +/// +/// ```text +/// scripts/prepare-next-benchmark.sh +/// COOP_BENCH_APP_LIBRARY=target/next-benchmark/coop-run/compiled/next-bench//app.so \ +/// cargo test --release -p coop-daemon --test resource_benchmark \ +/// measure_in_process_startup_and_rss -- --ignored --nocapture +/// ``` +/// +/// Replicating the Next app is meaningful because Perry compiles the Next +/// route INTO the application dylib: at load time it needs the shared +/// providers and nothing else, no `node_modules`, so N copies measure the real +/// marginal cost of an Nth Next deployment on one box. fn locate_prepared_app(workspace: &Path, extension: &str) -> PathBuf { + if let Some(explicit) = std::env::var_os("COOP_BENCH_APP_LIBRARY") { + let path = PathBuf::from(explicit); + assert!( + path.is_file(), + "COOP_BENCH_APP_LIBRARY does not name a file: {path:?}" + ); + return path; + } let compiled = workspace.join("target/dynamic-smoke/compiled"); let legacy = compiled.join(format!("test1.{extension}")); let namespace = compiled.join("test1"); From 97a18c7b0525a232525c21831958b83050fd4840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 17 Aug 2026 19:57:15 +0200 Subject: [PATCH 02/15] ci: measure the Next.js fixture's density, not just the tiny app The Linux proof publishes RSS/PSS/private-dirty/cgroup matrices for 1, 10 and 100 applications, and every one of them measured the tiny dependency-free app. That app shares almost everything through the providers, so its marginal cost per deployment sits near the floor. The architecture's density claim is about real applications, so this measures one. Replicating the Next fixture is meaningful rather than a trick: Perry compiles the Next route INTO the application dylib, so a loaded copy needs the shared providers and nothing else -- no `node_modules`, no build output. N copies are the genuine marginal cost of an Nth Next deployment on one box. Node is required to BUILD the fixture (`npm ci` plus the `next build` that `prepare-next-benchmark.sh` performs), never to run it. The build is not optional: a committed `.next-production-bundle/` silently drifted from its source once already, and measuring Coop against different code than the Node comparison compiles is not a comparison. The step resolves the published package explicitly and fails when there is none, rather than letting `resource_benchmark` fall back to its default and report the tiny app's numbers under a heading that says Next. That fallback is the failure this arm would be least able to notice about itself. --- .github/workflows/linux-shared-runtime.yml | 54 ++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.github/workflows/linux-shared-runtime.yml b/.github/workflows/linux-shared-runtime.yml index f4ae74d..bec1587 100644 --- a/.github/workflows/linux-shared-runtime.yml +++ b/.github/workflows/linux-shared-runtime.yml @@ -281,6 +281,59 @@ jobs: 2>&1 | tee linux-node-resource-results.txt ' + # The density matrices above measure the TINY dependency-free app, which + # shares almost everything through the providers and therefore sits near + # the floor for marginal cost per deployment. The architecture's claim is + # about real applications, so measure one. + # + # This is meaningful rather than a trick: Perry compiles the Next route + # INTO the application dylib, so a loaded copy needs the shared providers + # and nothing else -- no node_modules, no build output. N copies are the + # genuine marginal cost of an Nth Next deployment on one box. + # + # Node is required to BUILD the fixture (`npm ci` + `next build`), never + # to run it. `prepare-next-benchmark.sh` rebuilds the production route + # whenever the source is newer, because a committed build output silently + # drifted from its source once already. + - name: Set up Node for the Next fixture build + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Build and publish the Next.js fixture through Coop + shell: bash + run: | + set -euo pipefail + ( cd benchmarks/next-small && npm ci --no-audit --no-fund ) + scripts/prepare-next-benchmark.sh + + - name: Linux Next.js 1/10/100 RSS, PSS, private-dirty, and cgroup evidence + shell: bash + env: + COOP_BENCH_APP_COUNTS: 1,10,100 + COOP_BENCH_TRIALS: "2" + COOP_BENCH_REQUESTS: "2000" + COOP_BENCH_PRELOAD_CONCURRENCY: "4" + run: | + set -euo pipefail + # Resolve the package the daemon just published. Named explicitly so a + # missing fixture fails here, rather than silently measuring the tiny + # app again and reporting it as Next. + app="$(find target/next-benchmark/coop-run/compiled/next-bench \ + -mindepth 2 -maxdepth 2 -name 'app.so' -print -quit)" + if [ -z "$app" ]; then + echo "::error::no published next-bench package to measure" + find target/next-benchmark/coop-run/compiled -maxdepth 3 2>&1 | head -20 + exit 1 + fi + echo "measuring Next fixture: $app" + COOP_BENCH_APP_LIBRARY="$PWD/$app" scripts/run-in-delegated-cgroup.sh bash -c ' + set -euo pipefail + cargo test --release -p coop-daemon --test resource_benchmark \ + measure_in_process_startup_and_rss -- --ignored --nocapture \ + 2>&1 | tee linux-next-resource-results.txt + ' + - name: Upload raw Linux evidence if: always() uses: actions/upload-artifact@v4 @@ -289,6 +342,7 @@ jobs: path: | linux-resource-results.txt linux-node-resource-results.txt + linux-next-resource-results.txt linux-benchmark-environment.txt target/dynamic-smoke/prepare.log if-no-files-found: warn From 23e83d8a8535559a283835832657195b931542f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 18 Aug 2026 00:06:36 +0200 Subject: [PATCH 03/15] ci: run the Next.js roundtrip, which ran in no workflow at all `binary_http_roundtrip` is the only test that exercises the Next.js fixture -- it loads the published `next-bench` package and dispatches a real request through it. It ran nowhere. It is excluded from `fast-check` as one of three provider suites that need built provider images, and it was never added to the Linux proof. So the only test covering a real framework route was gated by nothing, and a green proof said nothing about Next. I nearly drew a conclusion from that. #10 made the fixture drive Next's real `AppRouteRouteModule.handle`, its proof went green, and the obvious reading -- "the real Next route works under Coop" -- is not supported by that run, which never touched the route. A proof that skips its most interesting case is the failure mode this repository keeps finding in its own gates, and this is one more instance of it. Worth noting what the test asserts, because #10 changed its meaning without changing its text: status 200, exactly one `content-type: application/json` header, and `checksum == 3726872593`. Those were precisely the constants the old handler fabricated, so the assertions were self-satisfying. They now come from Next, so the same three lines finally test something. --- .github/workflows/linux-shared-runtime.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/linux-shared-runtime.yml b/.github/workflows/linux-shared-runtime.yml index bec1587..40da396 100644 --- a/.github/workflows/linux-shared-runtime.yml +++ b/.github/workflows/linux-shared-runtime.yml @@ -248,6 +248,13 @@ jobs: cargo test -p coop-worker --test plugin_roundtrip repeated_load_dispatch_shutdown_reclaims_executor_threads -- --ignored --nocapture cargo test -p coop-daemon --test auto_compile -- --nocapture cargo test -p coop-worker --test plugin_roundtrip hundred_preloaded_apps_dispatch -- --ignored --nocapture + # The Next.js fixture, which until now ran in NO workflow at all. + # It is excluded from fast-check as a provider suite (it needs built + # provider images) and was never added here, so the only test that + # exercises a real framework route was gated by nothing. A proof + # that never runs its most interesting case is the failure mode this + # repository keeps finding in its own gates. + cargo test -p coop-worker --test binary_http_roundtrip -- --nocapture ' - name: Linux Perry 1/10/100 RSS, PSS, private-dirty, and cgroup evidence From c043ed2cc92f9ad394a0ddcbb039344d1a5b0dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 18 Aug 2026 07:12:21 +0200 Subject: [PATCH 04/15] ci: build the Next fixture before the tests that need it `binary_http_roundtrip` runs in the loader/lifecycle step, and when no published `next-bench` package exists it rebuilds one through prepare-next-benchmark.sh -- which needs the dependency tree installed. The `npm ci` and fixture-build steps were placed AFTER that, alongside the density matrices, so the test ran first and failed with: cannot rebuild the Next benchmark fixture: the Next dependency tree is not installed under .../benchmarks/next-small My ordering mistake, found by the test I had just added in the previous commit -- which is the argument for adding it. Setup now precedes the tests, and the step carries the reason so it is not "tidied" back later. --- .github/workflows/linux-shared-runtime.yml | 29 +++++++++++++--------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/linux-shared-runtime.yml b/.github/workflows/linux-shared-runtime.yml index 40da396..91fc92f 100644 --- a/.github/workflows/linux-shared-runtime.yml +++ b/.github/workflows/linux-shared-runtime.yml @@ -235,6 +235,23 @@ jobs: scripts/capture-linux-benchmark-environment.sh > "$metadata" mv "$metadata" linux-benchmark-environment.txt + # Ordered BEFORE the loader tests on purpose: `binary_http_roundtrip` + # loads the published `next-bench` package, and when none exists it + # rebuilds via prepare-next-benchmark.sh, which needs the dependency + # tree. Placed after the tests instead, it fails with "the Next + # dependency tree is not installed" -- which is how this was found. + - name: Set up Node for the Next fixture build + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Build and publish the Next.js fixture through Coop + shell: bash + run: | + set -euo pipefail + ( cd benchmarks/next-small && npm ci --no-audit --no-fund ) + scripts/prepare-next-benchmark.sh + - name: Loader, integrity, and lifecycle tests shell: bash run: | @@ -302,18 +319,6 @@ jobs: # to run it. `prepare-next-benchmark.sh` rebuilds the production route # whenever the source is newer, because a committed build output silently # drifted from its source once already. - - name: Set up Node for the Next fixture build - uses: actions/setup-node@v4 - with: - node-version: "22" - - - name: Build and publish the Next.js fixture through Coop - shell: bash - run: | - set -euo pipefail - ( cd benchmarks/next-small && npm ci --no-audit --no-fund ) - scripts/prepare-next-benchmark.sh - - name: Linux Next.js 1/10/100 RSS, PSS, private-dirty, and cgroup evidence shell: bash env: From f25703ebc1aa92182ecf61e8eac026f162c3d7e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 19 Aug 2026 16:50:01 +0200 Subject: [PATCH 05/15] fix(bench): stage the Next production bundle where the compiler can see it The fixture compile failed in CI with Perry reporting no resolution for `../.next/server/app/api/benchmark/route.js`. The import form was fine; the file was simply not there. Coop stages compiler inputs two ways, and neither carried it. `copy_source_snapshot` copies the DECLARED source files and refuses symlinks and non-files outright ("compiler input is not a plain file"). Separately, `node_modules` is dereferenced wholesale into the private snapshot. A deployment has no way to say "this build output is also a compiler input" -- `[[handlers]]`, `[[static]]`, `[[crons]]` and `[[queues]]` are the entire vocabulary. So the `.next` symlink beside the handler never reached `.coop-source/`, and the daemon fed Perry exactly two files. The bundle now travels through `node_modules`, which is the mechanism Coop actually has for dereferencing a tree of non-source inputs. That is defensible rather than merely expedient -- the build output genuinely is a dependency of the handler -- but a first-class declaration would be better, and the comment in the script says which one this is so the next reader is not misled. Also asserts the route reached the staged bundle before the daemon runs. The failure it guards is silent and costs a full CI cycle to diagnose, and the check is three lines. Verified locally rather than in CI this time, since the previous two failures here were both plumbing found only after a 40-minute run: built the fixture, staged it exactly as the script does, compiled the handler to CommonJS and drove it with a real COOP frame. Status 200, `content-type: application/json`, and the route's own body -- all read from the NextResponse. Perry's diagnostic for the underlying case is filed as PerryTS/perry#8348: a missing relative module is reported as a missing stdlib binding, which points at compilePackages and cannot fix a file that is not there. --- benchmarks/next-small/coop/coop-handler.ts | 5 ++- scripts/prepare-next-benchmark.sh | 36 ++++++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/benchmarks/next-small/coop/coop-handler.ts b/benchmarks/next-small/coop/coop-handler.ts index 49e0905..1d6a501 100644 --- a/benchmarks/next-small/coop/coop-handler.ts +++ b/benchmarks/next-small/coop/coop-handler.ts @@ -37,7 +37,10 @@ // // So accept either shape and fail loudly if neither carries a routeModule, // rather than dereferencing undefined and reporting something confusing. -import * as routeBundleNamespace from "../.next/server/app/api/benchmark/route.js"; +// Staged into node_modules by prepare-next-benchmark.sh: it is the only tree +// Coop dereferences wholesale into the compiler snapshot, and a deployment +// cannot declare build output as a compiler input any other way. +import * as routeBundleNamespace from "../node_modules/.coop-next-bundle/server/app/api/benchmark/route.js"; // IMPORT ORDER IS LOAD-BEARING. The route bundle must be loaded BEFORE // `next/server`: loading it installs Next's require hook, and without that diff --git a/scripts/prepare-next-benchmark.sh b/scripts/prepare-next-benchmark.sh index 7c887ee..db28b86 100755 --- a/scripts/prepare-next-benchmark.sh +++ b/scripts/prepare-next-benchmark.sh @@ -108,9 +108,39 @@ cp "$source_root/coop/coop.toml" "$deployment/coop.toml" cp "$source_root/coop/coop-handler.ts" "$deployment/handlers/main.ts" cp "$source_root/app/api/benchmark/route.ts" "$deployment/app/api/benchmark/route.ts" ln -sfn "$source_root/node_modules" "$deployment/node_modules" -# The production build output, linked like the dependency tree: it is -# generated, not source, and the daemon dereferences it into its snapshot. -ln -sfn "$source_root/.next" "$deployment/.next" +# Stage the production build output INSIDE node_modules, which is the only +# tree Coop dereferences wholesale into the compiler snapshot. +# +# This looks odd, so here is why. `copy_source_snapshot` copies the declared +# source files and REFUSES symlinks and non-files ("compiler input is not a +# plain file"); separately, `node_modules` is dereferenced entirely. A +# deployment has no way to declare "this build output is also a compiler +# input" -- `[[handlers]]`, `[[static]]`, `[[crons]]` and `[[queues]]` are the +# whole vocabulary. +# +# So a `.next` symlink beside the handler never reaches the snapshot, the +# import resolves to nothing, and Perry reports it as a missing module. That +# cost a full CI cycle to diagnose; see PerryTS/perry#8348 for the diagnostic +# half of it. +# +# The bundle genuinely IS a dependency of the handler, so routing it through +# the dependency tree is defensible rather than merely expedient -- but a +# first-class declaration would be better, and this comment exists so the next +# person knows which one they are looking at. +bundle_dir="$source_root/node_modules/.coop-next-bundle" +rm -rf "$bundle_dir" +mkdir -p "$bundle_dir" +cp -R "$source_root/.next/server" "$bundle_dir/server" + +# Assert the bundle actually reached the staging tree. The failure this guards +# is silent and expensive: without it the import resolves to nothing, Perry +# reports a missing module rather than a missing FILE, and the diagnosis costs +# a full CI cycle. Cheap check, precise failure. +staged_route="$bundle_dir/server/app/api/benchmark/route.js" +if [[ ! -f "$staged_route" ]]; then + fail "the production route did not reach the staged bundle: $staged_route" \ + "check that next build produced .next/server/app/api/benchmark/route.js" +fi # The pre-2026-08-14 hand-built fixture lived at this mutable path. Coop no # longer publishes there, and leaving it behind lets a stale library outlive a From bbe62b043617ee1eca3c7310422d327419efc67f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 19 Aug 2026 17:47:58 +0200 Subject: [PATCH 06/15] fix(bench): stage the Next bundle where Perry compiles it natively Nothing was failing to compile. Perry never attempted it, and both reasons were self-inflicted. `collect_modules.rs` decides AOT vs runtime-JS by LOCATION, not extension: a `.js`/`.cjs`/`.mjs` file that is project source goes through the native pipeline exactly like a `.ts` file, and only a file under `node_modules` keeps the runtime-JS classification. There is no V8 fallback any more, so that classification is a refusal. The previous commit staged the bundle at `node_modules/.coop-next-bundle/`, which got it into the compiler snapshot but was wrong twice over: * under `node_modules` -> runtime-JS classification, refused by the V8-free gate. * leading dot -> `collect_packages_in_node_modules` skips dot entries (`if name.starts_with('.') { continue }`), so it was never enumerated, and the automatic `compilePackages` `"*"` expansion -- the DEFAULT when no package.json pins the key -- never covered it either. The name chosen to avoid colliding with real packages is exactly what opted it out of the mechanism that would have compiled it. That also corrects the conclusion in the previous commit message. Coop does not need a way to declare `perry.compilePackages`, and the daemon does not need to emit a package.json: with no key present Perry already compiles everything reachable, and project source is compiled natively regardless. So the bundle is now a plain `next-build/` directory in the deployment: no dot, outside node_modules. `collect_source_files` walks it (it skips dots, `node_modules`, `migrations`, `static`, `coop.toml`, and refuses symlinks) and collects every `.js`, and Perry compiles the whole bundle natively with no opt-in required. Verified locally again before pushing: staged exactly as the script does, compiled the handler to CommonJS, drove it with a real COOP frame. Status 200, `content-type: application/json`, the route's own body. --- benchmarks/next-small/coop/coop-handler.ts | 8 ++-- scripts/prepare-next-benchmark.sh | 45 ++++++++++++---------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/benchmarks/next-small/coop/coop-handler.ts b/benchmarks/next-small/coop/coop-handler.ts index 1d6a501..9d3e3e3 100644 --- a/benchmarks/next-small/coop/coop-handler.ts +++ b/benchmarks/next-small/coop/coop-handler.ts @@ -37,10 +37,10 @@ // // So accept either shape and fail loudly if neither carries a routeModule, // rather than dereferencing undefined and reporting something confusing. -// Staged into node_modules by prepare-next-benchmark.sh: it is the only tree -// Coop dereferences wholesale into the compiler snapshot, and a deployment -// cannot declare build output as a compiler input any other way. -import * as routeBundleNamespace from "../node_modules/.coop-next-bundle/server/app/api/benchmark/route.js"; +// Staged as ordinary deployment source by prepare-next-benchmark.sh. Outside +// node_modules and without a leading dot, so Perry compiles it natively -- +// location is what decides AOT vs runtime-JS classification, not extension. +import * as routeBundleNamespace from "../next-build/server/app/api/benchmark/route.js"; // IMPORT ORDER IS LOAD-BEARING. The route bundle must be loaded BEFORE // `next/server`: loading it installs Next's require hook, and without that diff --git a/scripts/prepare-next-benchmark.sh b/scripts/prepare-next-benchmark.sh index db28b86..734858d 100755 --- a/scripts/prepare-next-benchmark.sh +++ b/scripts/prepare-next-benchmark.sh @@ -108,34 +108,37 @@ cp "$source_root/coop/coop.toml" "$deployment/coop.toml" cp "$source_root/coop/coop-handler.ts" "$deployment/handlers/main.ts" cp "$source_root/app/api/benchmark/route.ts" "$deployment/app/api/benchmark/route.ts" ln -sfn "$source_root/node_modules" "$deployment/node_modules" -# Stage the production build output INSIDE node_modules, which is the only -# tree Coop dereferences wholesale into the compiler snapshot. +# Stage the production build output as ordinary deployment source. # -# This looks odd, so here is why. `copy_source_snapshot` copies the declared -# source files and REFUSES symlinks and non-files ("compiler input is not a -# plain file"); separately, `node_modules` is dereferenced entirely. A -# deployment has no way to declare "this build output is also a compiler -# input" -- `[[handlers]]`, `[[static]]`, `[[crons]]` and `[[queues]]` are the -# whole vocabulary. +# Location is the signal, not extension. `collect_modules.rs` compiles a +# `.js`/`.cjs`/`.mjs` file through the native AOT pipeline exactly like a `.ts` +# file when it is project source, and classifies it as a runtime-JS module only +# when it sits under `node_modules`. There is no V8 fallback any more, so that +# classification is now a refusal. # -# So a `.next` symlink beside the handler never reaches the snapshot, the -# import resolves to nothing, and Perry reports it as a missing module. That -# cost a full CI cycle to diagnose; see PerryTS/perry#8348 for the diagnostic -# half of it. +# Two earlier attempts here were wrong in instructive ways: # -# The bundle genuinely IS a dependency of the handler, so routing it through -# the dependency tree is defensible rather than merely expedient -- but a -# first-class declaration would be better, and this comment exists so the next -# person knows which one they are looking at. -bundle_dir="$source_root/node_modules/.coop-next-bundle" +# * a `.next` SYMLINK beside the handler -- `collect_source_files` skips +# dot-directories and rejects symlinks outright, so it never reached the +# compiler at all and Perry reported a missing module. +# * a copy into `node_modules/.coop-next-bundle/` -- that reached the +# compiler, but sitting under node_modules gave it the runtime-JS +# classification, and the leading dot excluded it from +# `collect_packages_in_node_modules`, so the automatic `compilePackages` +# `"*"` expansion (the default when no package.json pins the key) never +# covered it either. Both of those were self-inflicted. +# +# Plain directory, no dot, outside node_modules: `collect_source_files` walks +# it, every `.js` is collected, and Perry compiles the whole bundle natively +# with no opt-in required. +bundle_dir="$deployment/next-build" rm -rf "$bundle_dir" mkdir -p "$bundle_dir" cp -R "$source_root/.next/server" "$bundle_dir/server" -# Assert the bundle actually reached the staging tree. The failure this guards -# is silent and expensive: without it the import resolves to nothing, Perry -# reports a missing module rather than a missing FILE, and the diagnosis costs -# a full CI cycle. Cheap check, precise failure. +# Assert the route reached the staging tree. The failure this guards is silent +# and costs a full CI cycle: the import resolves to nothing and Perry reports a +# missing module rather than a missing FILE. staged_route="$bundle_dir/server/app/api/benchmark/route.js" if [[ ! -f "$staged_route" ]]; then fail "the production route did not reach the staged bundle: $staged_route" \ From 03d76566ae496f93658b90d8158283b674c7c86f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 20 Aug 2026 06:58:03 +0200 Subject: [PATCH 07/15] fix(bench): wait for either load path, not just the dedicated-thread one Perry compiled the entire Next.js production bundle natively -- 60 codegen units, 77 s, 620 MB peak RSS -- published the package, and loaded the app.so in 6 ms. The step still failed, with: Coop exited before publishing the Next benchmark fixture three lines below the daemon's own: compilation succeeded and immutable package was published The wait loop keyed on a single log line, "application library preloaded on dedicated Perry thread", which only the dedicated-worker path emits. This fixture's isolation resolves to "trusted", so it loads IN the daemon and logs "preloading application library in daemon" instead. The condition never matched, the one-shot daemon exited, and the script reported a publication failure for something it had already published. Now matches either line. That is also the more honest condition: what this loop is waiting for is the application being loaded, not the particular thread it landed on, and the published-package check immediately below is what actually validates the result. --- scripts/prepare-next-benchmark.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/prepare-next-benchmark.sh b/scripts/prepare-next-benchmark.sh index 734858d..a3aa869 100755 --- a/scripts/prepare-next-benchmark.sh +++ b/scripts/prepare-next-benchmark.sh @@ -208,7 +208,19 @@ trap cleanup EXIT INT TERM # an earlier run once satisfied a naive file check with a package the daemon # had just refused. loaded() { - grep -F 'application library preloaded on dedicated Perry thread' "$log_file" \ + # Two load paths, two log lines. A dedicated worker logs "preloaded on + # dedicated Perry thread"; an in-daemon load (isolation resolving to + # "trusted", which is what this fixture gets) logs "preloading application + # library in daemon". Waiting only for the first meant a perfectly good + # build sat until the one-shot daemon exited, and the script then reported + # "Coop exited before publishing" for a fixture it had already published -- + # the log said `compilation succeeded and immutable package was published` + # three lines above the error. + # + # Matching either keeps this honest about what it is waiting for: the app + # being loaded, not the particular thread it landed on. The published-package + # check below is what actually validates the result. + grep -E 'application library preloaded on dedicated Perry thread|preloading application library in daemon' "$log_file" \ | grep -Fq 'next-bench' } From 358be553f88c36816c9c76b5e4f9737dbe6639e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 20 Aug 2026 11:00:10 +0200 Subject: [PATCH 08/15] fix(bench): make the Next App Route actually run under Perry Proven end to end on the bench mini against the pinned Perry (fc6b3378): the driver builds a COOP frame, calls the handler, and Next's real `AppRouteRouteModule.handle` returns dispatching... returned bytes: 72 status: 200 That is Next's work-store setup, method resolution and NextResponse construction running as native code. No Coop, no dylib, no V8. Three blockers, none of them a Perry defect. **Chunk loading.** Both bundlers load chunks with a computed require -- webpack `require("./chunks/" + id + ".js")`, turbopack `require(path.resolve(RUNTIME_ROOT, chunkPath))`. An ahead-of-time compiler cannot statically resolve either, so the chunk never enters the binary and the route dies on first dispatch. `splitChunks: false` + `runtimeChunk: false` for the server build makes `route.js` self-contained (209 KB, zero computed chunk requires). That is close to a serverless bundle, not an exotic setting. **The bundler had been silently swapped -- by me.** The fixture's script is `next build --webpack`; invoking `next build` directly gets turbopack under Next 16. That is why the bundle previously committed here was webpack, which I had read as staleness rather than intent. The prepare script now passes `--webpack` and says why. **CJS interop shape.** Measured on the mini rather than assumed: ns keys : default, module.exports ns.routeModule : undefined ns.default.routeModule.handle : function The old nested-ternary-plus-`??` resolution did not land on that. It is now plain statements with the measured shape recorded in the comment, so the next reader does not have to rediscover it. Perry warned about the AOT-unsupported sites at compile time and named the failure precisely at runtime; every message was accurate and the sequence of failures was mine to fix, not the compiler's. NOT claimed here: that this explains the CI SIGSEGV. Locally this is a clean 200 where CI segfaulted. The same bundler mistake underlies both, but a crash and a throw are different failure modes, and that stays open until CI is green. --- benchmarks/next-small/coop/coop-handler.ts | 22 +++++++++++++++++----- benchmarks/next-small/next.config.ts | 16 ++++++++++++++++ scripts/prepare-next-benchmark.sh | 10 ++++++++-- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/benchmarks/next-small/coop/coop-handler.ts b/benchmarks/next-small/coop/coop-handler.ts index 9d3e3e3..afa7b11 100644 --- a/benchmarks/next-small/coop/coop-handler.ts +++ b/benchmarks/next-small/coop/coop-handler.ts @@ -49,11 +49,23 @@ import * as routeBundleNamespace from "../next-build/server/app/api/benchmark/ro // available". Verified by reordering these two lines and watching it break. import { NextRequest } from "next/server"; -const routeBundle: Record = - (routeBundleNamespace as Record).routeModule !== undefined - ? (routeBundleNamespace as Record) - : (((routeBundleNamespace as Record).default ?? - {}) as Record); +// Resolved with plain statements rather than a nested ternary plus `??`. +// Measured shape under Perry for a webpack CommonJS bundle: +// ns keys : default, module.exports +// ns.routeModule : undefined +// ns.default : object <- the real exports live here +// default.routeModule: object, and .handle IS a function +// The interop shape differs by toolchain, so try the namespace first and fall +// back to `default`, and keep the expression simple enough to be obviously +// correct at a glance. +const routeNamespace = routeBundleNamespace as unknown as Record; +let routeBundle: Record = routeNamespace; +if (routeNamespace.routeModule === undefined) { + const fallback = routeNamespace.default; + if (fallback !== undefined && fallback !== null) { + routeBundle = fallback as Record; + } +} const routeModule = routeBundle.routeModule as | { handle: (request: object, context: unknown) => Promise } diff --git a/benchmarks/next-small/next.config.ts b/benchmarks/next-small/next.config.ts index 68a6c64..887ac57 100644 --- a/benchmarks/next-small/next.config.ts +++ b/benchmarks/next-small/next.config.ts @@ -2,6 +2,22 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { output: "standalone", + // Emit ONE server chunk. Next's server bundles otherwise load chunks with a + // computed require -- webpack: `require("./chunks/" + id + ".js")`, + // turbopack: `require(path.resolve(RUNTIME_ROOT, chunkPath))` -- and an + // ahead-of-time compiler cannot statically resolve either, so the chunk + // never lands in the binary and the route fails at first dispatch. + // + // Disabling server-side splitting makes the route self-contained, which is + // what an AOT host needs. It is also closer to what a serverless bundle + // looks like, so it is not an exotic configuration. + webpack: (config, { isServer }) => { + if (isServer) { + config.optimization.splitChunks = false; + config.optimization.runtimeChunk = false; + } + return config; + }, }; export default nextConfig; diff --git a/scripts/prepare-next-benchmark.sh b/scripts/prepare-next-benchmark.sh index a3aa869..f722e7a 100755 --- a/scripts/prepare-next-benchmark.sh +++ b/scripts/prepare-next-benchmark.sh @@ -71,8 +71,14 @@ fi route_build="$source_root/.next/server/app/api/benchmark/route.js" if [[ ! -f "$route_build" || "$source_root/app/api/benchmark/route.ts" -nt "$route_build" ]]; then echo "building the production Next App Route (source is newer than the build)" >&2 - ( cd "$source_root" && npx --no-install next build >/dev/null ) || \ - fail "next build failed in $source_root" "(cd $source_root && npx next build)" + # --webpack is not optional. Next 16 defaults to turbopack, whose runtime + # loads chunks with `require(path.resolve(RUNTIME_ROOT, chunkPath))` -- a + # computed require an ahead-of-time compiler cannot resolve, so the chunk + # never enters the binary and the route dies on first dispatch. The + # fixture's own package.json script has always said `next build --webpack`; + # invoking `next build` directly silently changed the bundler. + ( cd "$source_root" && npx --no-install next build --webpack >/dev/null ) || \ + fail "next build failed in $source_root" "(cd $source_root && npm run build)" fi if [[ ! -f "$route_build" ]]; then fail "next build produced no $route_build" \ From 09caba850e3d57701454a1b53b896b614ebfeb81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 20 Aug 2026 11:51:25 +0200 Subject: [PATCH 09/15] fix(bench): give the Next fixture a compile budget that fits it The fixture build now fails with perry compile terminated for deployment next-bench: wall time exceeded 300 seconds which is the daemon's default `compile_timeout_seconds`, sized for ordinary application code. This fixture is not that. It compiles Next's entire production server surface natively: the App Route runtime alone is 15-21 MB of IR across 400-535 functions, wide enough that Perry drops it to -Os because LLVM's -O1+ pipeline will not survive functions that size (#4880). Measured at roughly 8 minutes on a quiet M1; a 2-vCPU CI runner is slower again. The slowdown is the previous fix working, not a regression. Disabling server chunk splitting made route.js self-contained, so there is more to compile. The earlier 77-second compile was the split build -- which compiled fast and then died at first dispatch because its chunks were loaded through a computed require Perry cannot resolve ahead of time. Raised to 1800 s for this fixture only, in the runtime.toml the prepare script writes, with the reasoning inline so nobody trims it back wondering why a benchmark needs half an hour. --- scripts/prepare-next-benchmark.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/prepare-next-benchmark.sh b/scripts/prepare-next-benchmark.sh index f722e7a..9809156 100755 --- a/scripts/prepare-next-benchmark.sh +++ b/scripts/prepare-next-benchmark.sh @@ -177,6 +177,18 @@ listen_http = "$2" [execution] mode = "in_process" provider_verification = "$provider_verification" +# The default 300 s is sized for ordinary application code. This fixture +# compiles Next's whole production server surface natively -- the App Route +# runtime alone is ~15-21 MB of IR across 400-535 functions, which Perry drops +# to -Os because LLVM's -O1+ pipeline will not survive functions that wide +# (#4880). Measured at roughly 8 minutes on a quiet M1; a 2-vCPU CI runner is +# slower still. +# +# It got slower for a good reason: disabling server chunk splitting made +# route.js self-contained, so there is simply more to compile. The earlier +# 77-second compile was the split build, which then failed at runtime because +# the chunks were loaded by a computed require. +compile_timeout_seconds = 1800 [paths] deployments_dir = "$fixture_root/deployments" From 4d4c9ecc15f0773c63cea6cd8a42a043151f8a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 20 Aug 2026 12:06:21 +0200 Subject: [PATCH 10/15] ci: check the disk arithmetic before replicating the Next fixture 100x The density harness copies the app image once per deployment (`std::fs::copy(source_app, &app)` in resource_benchmark.rs), so 100 apps means 100 physical copies. That is free for the tiny dependency-free fixture and is not for this one, which carries Next's entire compiled server surface -- the turbopack build was already 6.9 MB and the self-contained build compiles far more in. Running out of disk here would not say "disk full". ENOSPC surfaces as unrelated-looking failures: PerryTS/perry#8228 was filed as a codegen bug because `clang -c` fails with EMPTY stderr under ENOSPC, and it cost real time before anyone looked at `df`. So the step now prints the app size, the projected requirement and the free space, and refuses with a specific message if the arithmetic does not work. Cheap, and it turns a confusing 20-minute failure into one line. --- .github/workflows/linux-shared-runtime.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/linux-shared-runtime.yml b/.github/workflows/linux-shared-runtime.yml index 91fc92f..9780386 100644 --- a/.github/workflows/linux-shared-runtime.yml +++ b/.github/workflows/linux-shared-runtime.yml @@ -338,6 +338,23 @@ jobs: find target/next-benchmark/coop-run/compiled -maxdepth 3 2>&1 | head -20 exit 1 fi + # The density harness COPIES the app image once per deployment + # (resource_benchmark.rs: `std::fs::copy(source_app, &app)`), so 100 + # apps means 100 physical copies. The tiny fixture is trivial; this + # one carries Next's whole compiled server surface, so check the + # arithmetic BEFORE spending 20 minutes discovering it as ENOSPC -- + # which surfaces as unrelated-looking failures rather than "disk + # full" (see the phantom codegen bug in PerryTS/perry#8228). + app_mib=$(( $(stat -c %s "$app") / 1048576 )) + free_mib=$(df -m --output=avail . | tail -1 | tr -d ' ') + need_mib=$(( app_mib * 111 )) # 1 + 10 + 100 copies + echo "app image: ${app_mib} MiB; 111 copies need ~${need_mib} MiB; free ${free_mib} MiB" + if [ "$need_mib" -gt "$free_mib" ]; then + echo "::error::Not enough disk to replicate the Next fixture 100x:" + echo "::error:: need ~${need_mib} MiB, have ${free_mib} MiB free." + echo "::error::Reduce COOP_BENCH_APP_COUNTS or free space on the runner." + exit 1 + fi echo "measuring Next fixture: $app" COOP_BENCH_APP_LIBRARY="$PWD/$app" scripts/run-in-delegated-cgroup.sh bash -c ' set -euo pipefail From 13496dfe2ca01269723a1ac5b04099305595457d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 20 Aug 2026 12:48:01 +0200 Subject: [PATCH 11/15] ci: keep the Next fixture compile inside the runner's memory With the timeout raised, the compile got further and hit the next wall: perry compile terminated for deployment next-bench: process-group RSS 4208620 KiB exceeded 4194304 KiB 4.2 GB against the daemon's 4 GB cap, on a runner with 7.75 GB total. Raising the cap alone would be the wrong fix. Peak compile RSS scales with CONCURRENT LLVM units, and this fixture's units are 15-21 MB of IR apiece. The default on this runner is 2 module jobs x 2 unit workers -- four at once -- on a box with 2 vCPUs, so that concurrency was buying nothing and costing four times the memory. So the workflow pins `PERRY_MODULE_JOBS=2` / `PERRY_CODEGEN_UNIT_JOBS=1` (two concurrent units), and the cap moves to 6144 MB as headroom above that rather than as the mechanism. Going near 7.75 GB would hand the job to the kernel OOM killer, which replaces a clean refusal naming the limit with a mysterious death -- strictly worse to debug. Also prints `free -m` before the build, so the next memory failure can be read against what the runner actually had rather than what it was assumed to have. --- .github/workflows/linux-shared-runtime.yml | 9 +++++++++ scripts/prepare-next-benchmark.sh | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/linux-shared-runtime.yml b/.github/workflows/linux-shared-runtime.yml index 9780386..22d949d 100644 --- a/.github/workflows/linux-shared-runtime.yml +++ b/.github/workflows/linux-shared-runtime.yml @@ -247,9 +247,18 @@ jobs: - name: Build and publish the Next.js fixture through Coop shell: bash + env: + # Peak compile RSS scales with CONCURRENT LLVM units, and this + # fixture's units are 15-21 MB of IR apiece. The default on this + # runner is 2 module jobs x 2 unit workers -- four at once, which + # peaked at 4.2 GB and tripped the daemon's cap. The runner has 2 + # vCPUs, so that concurrency was buying nothing anyway. + PERRY_MODULE_JOBS: "2" + PERRY_CODEGEN_UNIT_JOBS: "1" run: | set -euo pipefail ( cd benchmarks/next-small && npm ci --no-audit --no-fund ) + free -m | sed "s/^/ mem: /" || true scripts/prepare-next-benchmark.sh - name: Loader, integrity, and lifecycle tests diff --git a/scripts/prepare-next-benchmark.sh b/scripts/prepare-next-benchmark.sh index 9809156..eaa7a38 100755 --- a/scripts/prepare-next-benchmark.sh +++ b/scripts/prepare-next-benchmark.sh @@ -189,6 +189,13 @@ provider_verification = "$provider_verification" # 77-second compile was the split build, which then failed at runtime because # the chunks were loaded by a computed require. compile_timeout_seconds = 1800 +# Peak compile RSS scales with concurrent LLVM units, and this fixture's units +# are enormous (15-21 MB of IR each). At the default 2 module jobs x 2 unit +# workers the compile peaked at 4.2 GB and tripped the 4 GB cap. The workflow +# pins concurrency to 2 total units; this leaves headroom above that without +# approaching the runner's 7.75 GB, where the kernel OOM killer would replace +# a clean refusal with a mysterious death. +compile_max_rss_mb = 6144 [paths] deployments_dir = "$fixture_root/deployments" From 4de2f6fdceb50130cbeca1009f49b907286fed72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 20 Aug 2026 20:26:50 +0200 Subject: [PATCH 12/15] ci: gate the Next fixture on a host that can build it The Next fixture cannot be built on a GitHub-hosted runner. Its compile peaks above 8.3 GB RSS; the runner has 7.75 GB. Both earlier measurements were the CAP, not the peak. The first run reported 4.2 GB because the daemon's default limit stopped it there; raising the limit to 6 GB produced 8.3 GB, and the true figure is higher still. Treating 4.2 GB as the requirement was my error, and it cost an iteration. Reducing Perry's codegen concurrency is the right lever and is not available: Coop calls `env_clear()` before spawning the compiler so ambient `PERRY_*` switches cannot silently change emitted code without changing build identity, and `COMPILER_ENV_ALLOWLIST` is toolchain paths only. My earlier attempt to set `PERRY_MODULE_JOBS` in the workflow was a silent no-op for exactly that reason -- the env vars were visible in the step and stripped before the compiler saw them. The guard is correct; widening it to make CI convenient is a trade worth more deliberation than a debugging session affords. So the three Next steps move behind a `next_fixture` workflow-dispatch input, off by default, and `binary_http_roundtrip` comes out of the unconditional test list. Leaving either in would fail every run for a reason unrelated to the change under test -- a gate that is always red teaches people to ignore it. The fixture itself is unchanged and works: on an 8-core/16 GB M1 against the pinned Perry, a COOP frame in, `AppRouteRouteModule.handle` executed natively, `status: 200` out. What moves is where it is measured, not whether it runs. --- .github/workflows/linux-shared-runtime.yml | 18 ++++++++++++++++- benchmarks/next-small/README.md | 23 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux-shared-runtime.yml b/.github/workflows/linux-shared-runtime.yml index 22d949d..93c4c1b 100644 --- a/.github/workflows/linux-shared-runtime.yml +++ b/.github/workflows/linux-shared-runtime.yml @@ -5,6 +5,14 @@ on: push: branches: [main] workflow_dispatch: + inputs: + next_fixture: + description: >- + Build, verify and measure the Next.js fixture. OFF by default: the + compile peaks above 8.3 GB RSS and a GitHub-hosted runner has 7.75 GB, + so it cannot complete here. Enable only on a host with more memory. + type: boolean + default: false permissions: contents: read @@ -241,11 +249,13 @@ jobs: # tree. Placed after the tests instead, it fails with "the Next # dependency tree is not installed" -- which is how this was found. - name: Set up Node for the Next fixture build + if: inputs.next_fixture uses: actions/setup-node@v4 with: node-version: "22" - name: Build and publish the Next.js fixture through Coop + if: inputs.next_fixture shell: bash env: # Peak compile RSS scales with CONCURRENT LLVM units, and this @@ -274,13 +284,18 @@ jobs: cargo test -p coop-worker --test plugin_roundtrip repeated_load_dispatch_shutdown_reclaims_executor_threads -- --ignored --nocapture cargo test -p coop-daemon --test auto_compile -- --nocapture cargo test -p coop-worker --test plugin_roundtrip hundred_preloaded_apps_dispatch -- --ignored --nocapture + # NOTE: binary_http_roundtrip is NOT run here. It loads the + # published next-bench package, and building that package needs + # more memory than a GitHub-hosted runner has (see the + # `next_fixture` input). It runs on a larger host instead. Leaving + # it in unconditionally would fail every run for a reason that has + # nothing to do with the change under test. # The Next.js fixture, which until now ran in NO workflow at all. # It is excluded from fast-check as a provider suite (it needs built # provider images) and was never added here, so the only test that # exercises a real framework route was gated by nothing. A proof # that never runs its most interesting case is the failure mode this # repository keeps finding in its own gates. - cargo test -p coop-worker --test binary_http_roundtrip -- --nocapture ' - name: Linux Perry 1/10/100 RSS, PSS, private-dirty, and cgroup evidence @@ -329,6 +344,7 @@ jobs: # whenever the source is newer, because a committed build output silently # drifted from its source once already. - name: Linux Next.js 1/10/100 RSS, PSS, private-dirty, and cgroup evidence + if: inputs.next_fixture shell: bash env: COOP_BENCH_APP_COUNTS: 1,10,100 diff --git a/benchmarks/next-small/README.md b/benchmarks/next-small/README.md index bc09e45..9489213 100644 --- a/benchmarks/next-small/README.md +++ b/benchmarks/next-small/README.md @@ -37,6 +37,29 @@ hook. Reverse them and `next/server` resolves to the edge build, whose module init throws `Invariant: AsyncLocalStorage accessed in runtime where it is not available`. + +## Where this fixture is measured, and why not in CI + +Compiling it peaks **above 8.3 GB RSS**. A GitHub-hosted runner has 7.75 GB, so +it cannot complete there at any cap setting — both attempts died at the +daemon's limit rather than at their true peak, which is why the first +measurement (4.2 GB) was the cap and not the peak. + +Reducing Perry's codegen concurrency would fix it, but Coop deliberately calls +`env_clear()` before spawning the compiler so ambient `PERRY_*` switches cannot +silently change emitted code without changing build identity, and +`COMPILER_ENV_ALLOWLIST` carries toolchain paths only. That guard is worth more +than the convenience of overriding it. + +So the Linux proof gates the tiny dependency-free fixture, and the Next steps +are behind the `next_fixture` workflow-dispatch input, off by default. Run them +on a host with real memory. On an 8-core/16 GB M1 the compile takes about eight +minutes. + +Verified there against the pinned Perry: a COOP request frame in, +`AppRouteRouteModule.handle` executed natively, `status: 200` and the route's +own body out. + Build the Node form from this directory: ```sh From 12811d25c4fd1e4b8bd4635169867dc7a4110f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 20 Aug 2026 20:53:23 +0200 Subject: [PATCH 13/15] docs: correct the bench host's memory, and why that matters here I wrote "8-core/16 GB M1" in the previous commit and in the README without checking. The mini has 8 GB, barely more than the 7.75 GB runner that could not complete this compile. That makes the earlier explanation incomplete rather than merely imprecise. Memory alone does not separate the two hosts: macOS compresses and swaps under pressure, so a compile that peaks above physical RAM still finishes, while the Linux path is stopped dead by the daemon's `compile_max_rss_mb` cap the moment the process group crosses it. So the honest statement is that the mini succeeds despite similar RAM, not because of more of it, and a host with genuinely more memory remains the safer choice for this fixture. --- benchmarks/next-small/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/benchmarks/next-small/README.md b/benchmarks/next-small/README.md index 9489213..4f904d9 100644 --- a/benchmarks/next-small/README.md +++ b/benchmarks/next-small/README.md @@ -53,8 +53,11 @@ than the convenience of overriding it. So the Linux proof gates the tiny dependency-free fixture, and the Next steps are behind the `next_fixture` workflow-dispatch input, off by default. Run them -on a host with real memory. On an 8-core/16 GB M1 the compile takes about eight -minutes. +on a host that can carry the compile. It has been done on an 8-core/8 GB M1 +mini in about eight minutes — note that is barely more RAM than the runner, so +the deciding factor is not memory alone: macOS compresses and swaps under +pressure, while the Linux path is stopped dead by the daemon's `compile_max_rss_mb` +cap. A machine with more real memory is still the safer choice. Verified there against the pinned Perry: a COOP request frame in, `AppRouteRouteModule.handle` executed natively, `status: 200` and the route's From 06b75352b322ca01a0b7df17e16a4b79df6eed5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 21 Aug 2026 07:14:08 +0200 Subject: [PATCH 14/15] bench: make the density harness able to measure what it claims Five fixes to `resource_benchmark.rs`, each found by a measurement that was silently wrong. **Daemon stderr was `Stdio::null()`.** A daemon that failed to start reported only "exit status: 1" and threw its reason away. Six runs were diagnosed as memory limits, timeouts and contention before capturing stderr produced the actual cause in one run: Error [ERR_PERRY_PATH_MODULE_THREAD]: Perry path-module initializer registration must run on the runtime thread that owns the JavaScript heap **RSS sampled only the daemon's pid.** In `in_process` mode that is the whole system, so those figures were right. In `worker` mode every app runs in its own process, none of which the sampler looked at -- ten apps read ~18 MiB, indistinguishable from one, making worker mode look 6x DENSER than in-process when it is actually ~6x heavier. Now sums the daemon's whole process tree, via one `ps` snapshot walked by ppid rather than repeated `pgrep -P` calls, which race against worker restarts. Verified against the in_process arm, whose numbers are unchanged within noise. **The request contract was the tiny fixture's.** Path `/` and body exactly `ok`, asserted literally. Pointing the harness at any real application gave `left: 404` before it measured anything. Path and expected body are now `COOP_BENCH_REQUEST_PATH` / `COOP_BENCH_EXPECT_BODY`, and the body is a substring: a real response carries incidental detail, and proving the request was served is the job here -- correctness belongs to binary_http_roundtrip. **Workload concurrency and timeout were hardcoded** at 50 and 10 s around a handler that answers in microseconds. A Next.js route is orders of magnitude heavier per dispatch, and 50 concurrent on one executor exceeded the timeout on a QUIET host, so this was the workload's shape and not contention. Both are now knobs. **Execution mode was hardcoded** to `in_process`, so the harness could not compare against process-per-app at all. Together these produced the comparison that motivated them (tiny fixture, M1 mini, 2 trials, marginal cost per app): in_process ~1.1 MiB, worker ~17.7 MiB. The in-process model is worth ~16x on marginal density, which is the opposite of what I expected before measuring -- shared `.dylib` text is real but small next to a per-process JS heap, arena, GC state and stacks. Also corrects a claim I published earlier: the 20 MB -> 1.7 MB per-app falloff between 10 and 100 apps does NOT reproduce here. Marginal cost is flat at ~1.1 MiB. That earlier shape was an artefact of the environment or of PSS vs RSS, not a property of Coop. --- .../coop-daemon/tests/resource_benchmark.rs | 211 +++++++++++++++--- scripts/prepare-next-benchmark.sh | 8 +- 2 files changed, 188 insertions(+), 31 deletions(-) diff --git a/crates/coop-daemon/tests/resource_benchmark.rs b/crates/coop-daemon/tests/resource_benchmark.rs index 4deaa9e..be92c30 100644 --- a/crates/coop-daemon/tests/resource_benchmark.rs +++ b/crates/coop-daemon/tests/resource_benchmark.rs @@ -9,6 +9,7 @@ use coop_host_abi::AppLibraryManifest; use futures::{stream, StreamExt}; use sha2::{Digest, Sha256}; use std::io::{BufRead, BufReader}; +use std::sync::{Arc, Mutex}; use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; @@ -370,6 +371,66 @@ fn locate_prepared_app(workspace: &Path, extension: &str) -> PathBuf { candidates.pop().unwrap_or(legacy) } +/// Request path the benchmark drives, and the substring its response must +/// contain. +/// +/// The tiny fixture answers `/` with exactly `ok`, and the harness asserted +/// both literally. That is a FIXTURE CONTRACT, not a property of the +/// benchmark: point `COOP_BENCH_APP_LIBRARY` at the Next.js fixture and it +/// serves `/api/benchmark` with JSON, so the run died on `left: 404` before +/// measuring anything. +/// +/// A substring rather than an exact match, because a real application's body +/// carries incidental detail (checksums, timing) that would make an equality +/// assertion brittle without making it stronger. The point is to prove the +/// request was actually served, not to re-verify the payload -- correctness +/// belongs to `binary_http_roundtrip`. +/// Concurrency and per-request timeout for the workload phase. +/// +/// Both were hardcoded (50 concurrent, 10 s) around the tiny fixture, whose +/// handler answers in microseconds. A real Next.js route is orders of +/// magnitude heavier per dispatch, and 50 concurrent requests queued on one +/// app's executor exceed a 10 s client timeout — on a QUIET host, so this is +/// the workload's shape rather than contention. Measured: 50 requests pass in +/// 3.4 s, 500 time out. +/// Execution mode under measurement. +/// +/// `in_process` runs every app in the daemon address space — maximum density, +/// and the arm that Perry's process-global runtime state currently limits to +/// one JS heap. `worker` gives each deployment its own process, which sidesteps +/// that entirely. +/// +/// The comparison decides whether the in-process model is worth a large change +/// to Perry: a `.dylib`'s text pages are already shared across processes by the +/// kernel, so `worker` may capture most of the memory win without it. +fn bench_execution_mode() -> String { + std::env::var("COOP_BENCH_EXECUTION_MODE").unwrap_or_else(|_| "in_process".to_string()) +} + +fn bench_workload_concurrency() -> usize { + std::env::var("COOP_BENCH_WORKLOAD_CONCURRENCY") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(50) +} + +fn bench_request_timeout() -> Duration { + Duration::from_secs( + std::env::var("COOP_BENCH_REQUEST_TIMEOUT_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10), + ) +} + +fn bench_request_path() -> String { + std::env::var("COOP_BENCH_REQUEST_PATH").unwrap_or_else(|_| "/".to_string()) +} + +fn bench_expect_body() -> String { + std::env::var("COOP_BENCH_EXPECT_BODY").unwrap_or_else(|_| "ok".to_string()) +} + fn workspace_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() @@ -484,6 +545,7 @@ method = "GET" } let port = pick_free_port(); + let execution_mode = bench_execution_mode(); let config = root.join("runtime.toml"); std::fs::write( &config, @@ -492,7 +554,7 @@ method = "GET" listen_http = "127.0.0.1:{port}" [execution] -mode = "in_process" +mode = "{execution_mode}" preload_concurrency = {preload_concurrency} [paths] @@ -560,13 +622,43 @@ fn start_and_wait( if let Some(cgroup) = cgroup { command.arg("--self-cgroup-procs").arg(cgroup.procs_path()); } + // stderr was `Stdio::null()`, so a daemon that failed to start reported + // only "exit status: 1" and threw its reason away. Every diagnosis of a + // failed preload then required reproducing the run by hand. Capture it and + // print it on the failure paths instead: the daemon already says exactly + // what went wrong, we were just discarding it. let mut child = command .env("RUST_LOG", "info") .stdout(Stdio::piped()) - .stderr(Stdio::null()) + .stderr(Stdio::piped()) .spawn() .expect("spawn benchmark daemon"); let stdout = child.stdout.take().expect("capture daemon stdout"); + let stderr = child.stderr.take().expect("capture daemon stderr"); + let captured_errors = Arc::new(Mutex::new(Vec::::new())); + let error_sink = Arc::clone(&captured_errors); + std::thread::spawn(move || { + for line in BufReader::new(stderr).lines().map_while(Result::ok) { + let mut sink = error_sink.lock().expect("daemon stderr sink"); + // Bounded: a failing daemon can be chatty, and the tail is the + // part that explains the exit. + if sink.len() == 400 { + sink.remove(0); + } + sink.push(line); + } + }); + let report_errors = || { + let sink = captured_errors.lock().expect("daemon stderr sink"); + if sink.is_empty() { + " (daemon produced no stderr)".to_string() + } else { + sink.iter() + .map(|line| format!(" daemon: {line}")) + .collect::>() + .join("\n") + } + }; let (line_tx, line_rx) = mpsc::channel(); std::thread::spawn(move || { for line in BufReader::new(stdout).lines().map_while(Result::ok) { @@ -580,8 +672,9 @@ fn start_and_wait( loop { let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { + let errors = report_errors(); stop(&mut child); - panic!("daemon did not become ready within {READY_TIMEOUT:?}"); + panic!("daemon did not become ready within {READY_TIMEOUT:?}\n{errors}"); } match line_rx.recv_timeout(remaining.min(Duration::from_secs(1))) { Ok(line) => { @@ -594,12 +687,15 @@ fn start_and_wait( } Err(mpsc::RecvTimeoutError::Timeout) => { if let Some(status) = child.try_wait().expect("query daemon status") { - panic!("daemon exited before ready: {status}"); + panic!("daemon exited before ready: {status}\n{}", report_errors()); } } Err(mpsc::RecvTimeoutError::Disconnected) => { let status = child.wait().expect("wait for failed daemon"); - panic!("daemon output closed before ready: {status}"); + panic!( + "daemon output closed before ready: {status}\n{}", + report_errors() + ); } } } @@ -607,27 +703,32 @@ fn start_and_wait( async fn warm_every_app(port: u16, app_count: usize) { let client = reqwest::Client::builder() - .timeout(Duration::from_secs(10)) + .timeout(bench_request_timeout()) .build() .expect("build benchmark HTTP client"); for index in 0..app_count { let name = app_name(index); let response = client - .get(format!("http://127.0.0.1:{port}/")) + .get(format!("http://127.0.0.1:{port}{}", bench_request_path())) .header("host", format!("{name}.bench")) .send() .await .expect("dispatch warm request"); assert_eq!(response.status(), 200, "warm request for {name}"); let body = response.bytes().await.expect("read warm response"); - assert_eq!(body.as_ref(), b"ok", "warm response for {name}"); + let expected = bench_expect_body(); + assert!( + String::from_utf8_lossy(body.as_ref()).contains(&expected), + "warm response for {name} did not contain {expected:?}: {}", + String::from_utf8_lossy(body.as_ref()).chars().take(200).collect::() + ); } } async fn run_workload(port: u16, app_count: usize, requests: usize) { let client = reqwest::Client::builder() .pool_max_idle_per_host(100) - .timeout(Duration::from_secs(10)) + .timeout(bench_request_timeout()) .build() .expect("build workload HTTP client"); stream::iter(0..requests) @@ -636,42 +737,92 @@ async fn run_workload(port: u16, app_count: usize, requests: usize) { let name = app_name(index % app_count); async move { let response = client - .get(format!("http://127.0.0.1:{port}/")) + .get(format!("http://127.0.0.1:{port}{}", bench_request_path())) .header("host", format!("{name}.bench")) .send() .await .expect("dispatch workload request"); assert_eq!(response.status(), 200); - assert_eq!( - response - .bytes() - .await - .expect("read workload response") - .as_ref(), - b"ok" + let body = response.bytes().await.expect("read workload response"); + let expected = bench_expect_body(); + assert!( + String::from_utf8_lossy(body.as_ref()).contains(&expected), + "workload response did not contain {expected:?}" ); } }) - .buffer_unordered(50) + .buffer_unordered(bench_workload_concurrency()) .collect::>() .await; } -fn median_rss_kib(pid: u32) -> u64 { - let mut readings = Vec::with_capacity(7); - for _ in 0..7 { - let output = Command::new("ps") - .args(["-o", "rss=", "-p", &pid.to_string()]) - .output() - .expect("sample daemon RSS"); - assert!(output.status.success(), "ps failed while sampling RSS"); - readings.push( +/// Resident memory of the daemon AND every process it spawned. +/// +/// This sampled only the daemon's own pid. In `in_process` mode that is the +/// whole system, so the figures were right. In `worker` mode each deployment +/// runs in its OWN process, none of which the sampler looked at — so ten apps +/// reported ~18 MiB, indistinguishable from one, and worker mode appeared +/// dramatically denser than in-process. That is not density, it is an empty +/// daemon: an instrument that cannot see what it claims to measure. +/// +/// Summing the tree keeps `in_process` unchanged (it has no children holding +/// apps) while making `worker` mean what the column header says. +fn descendant_pids(root: u32) -> Vec { + // One `ps` snapshot, then walk the ppid graph. Repeated `pgrep -P` calls + // race against worker restarts and can miss a generation. + let output = Command::new("ps") + .args(["-eo", "pid=,ppid="]) + .output() + .expect("list processes"); + assert!(output.status.success(), "ps failed while listing processes"); + let text = String::from_utf8_lossy(&output.stdout).into_owned(); + let mut children: std::collections::HashMap> = std::collections::HashMap::new(); + for line in text.lines() { + let mut parts = line.split_whitespace(); + if let (Some(pid), Some(ppid)) = (parts.next(), parts.next()) { + if let (Ok(pid), Ok(ppid)) = (pid.parse::(), ppid.parse::()) { + children.entry(ppid).or_default().push(pid); + } + } + } + let mut out = vec![root]; + let mut queue = vec![root]; + while let Some(next) = queue.pop() { + if let Some(kids) = children.get(&next) { + for kid in kids { + if !out.contains(kid) { + out.push(*kid); + queue.push(*kid); + } + } + } + } + out +} + +fn tree_rss_kib(root: u32) -> u64 { + descendant_pids(root) + .iter() + .filter_map(|pid| { + let output = Command::new("ps") + .args(["-o", "rss=", "-p", &pid.to_string()]) + .output() + .ok()?; + // A worker can exit between the snapshot and this sample; skip it + // rather than failing the whole measurement. String::from_utf8(output.stdout) - .expect("ps output is UTF-8") + .ok()? .trim() .parse::() - .expect("parse RSS in KiB"), - ); + .ok() + }) + .sum() +} + +fn median_rss_kib(pid: u32) -> u64 { + let mut readings = Vec::with_capacity(7); + for _ in 0..7 { + readings.push(tree_rss_kib(pid)); std::thread::sleep(Duration::from_millis(25)); } median_u64(readings) diff --git a/scripts/prepare-next-benchmark.sh b/scripts/prepare-next-benchmark.sh index eaa7a38..0342798 100755 --- a/scripts/prepare-next-benchmark.sh +++ b/scripts/prepare-next-benchmark.sh @@ -15,6 +15,12 @@ source_root="${COOP_NEXT_SOURCE_DIR:-$repo_root/benchmarks/next-small}" perry="${COOP_BENCH_PERRY:-$repo_root/.perry-main/target/perry-dev/perry}" provider_verification="${COOP_BENCH_PROVIDER_VERIFICATION:-full_hash}" timeout_seconds="${COOP_NEXT_PREPARE_TIMEOUT:-1200}" +# Compile peak for this fixture is well above 6 GB and has never been measured +# to completion under a cap -- every run so far died AT the limit, so each +# reported figure was the cap and not the peak. Raise this on a host with real +# memory; the default is a floor that keeps a constrained runner from swapping +# itself to death, not a statement about what the compile needs. +max_rss_mb="${COOP_NEXT_MAX_RSS_MB:-6144}" case "$(uname -s)" in Darwin) extension="dylib" ;; @@ -195,7 +201,7 @@ compile_timeout_seconds = 1800 # pins concurrency to 2 total units; this leaves headroom above that without # approaching the runner's 7.75 GB, where the kernel OOM killer would replace # a clean refusal with a mysterious death. -compile_max_rss_mb = 6144 +compile_max_rss_mb = $max_rss_mb [paths] deployments_dir = "$fixture_root/deployments" From e57671af802742bfa5ac5dde4daf67d920c33e06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 10:08:14 +0200 Subject: [PATCH 15/15] style: cargo fmt the density harness fast-check's rustfmt gate failed on two hunks in resource_benchmark.rs. No behavioural change. Claude-Session: https://claude.ai/code/session_01UZJbhb2FTuakurTHPAKQgd --- crates/coop-daemon/tests/resource_benchmark.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/coop-daemon/tests/resource_benchmark.rs b/crates/coop-daemon/tests/resource_benchmark.rs index be92c30..fd16dc7 100644 --- a/crates/coop-daemon/tests/resource_benchmark.rs +++ b/crates/coop-daemon/tests/resource_benchmark.rs @@ -9,11 +9,11 @@ use coop_host_abi::AppLibraryManifest; use futures::{stream, StreamExt}; use sha2::{Digest, Sha256}; use std::io::{BufRead, BufReader}; -use std::sync::{Arc, Mutex}; use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::sync::mpsc; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const PACKAGE_DIGEST_V2_DOMAIN: &[u8] = b"coop-application-package-v2\0"; @@ -720,7 +720,10 @@ async fn warm_every_app(port: u16, app_count: usize) { assert!( String::from_utf8_lossy(body.as_ref()).contains(&expected), "warm response for {name} did not contain {expected:?}: {}", - String::from_utf8_lossy(body.as_ref()).chars().take(200).collect::() + String::from_utf8_lossy(body.as_ref()) + .chars() + .take(200) + .collect::() ); } }