diff --git a/.github/workflows/linux-shared-runtime.yml b/.github/workflows/linux-shared-runtime.yml index f4ae74d..93c4c1b 100644 --- a/.github/workflows/linux-shared-runtime.yml +++ b/.github/workflows/linux-shared-runtime.yml @@ -5,6 +5,14 @@ on: push: branches: [main] workflow_dispatch: + inputs: + next_fixture: + description: >- + Build, verify and measure the Next.js fixture. OFF by default: the + compile peaks above 8.3 GB RSS and a GitHub-hosted runner has 7.75 GB, + so it cannot complete here. Enable only on a host with more memory. + type: boolean + default: false permissions: contents: read @@ -235,6 +243,34 @@ jobs: scripts/capture-linux-benchmark-environment.sh > "$metadata" mv "$metadata" linux-benchmark-environment.txt + # Ordered BEFORE the loader tests on purpose: `binary_http_roundtrip` + # loads the published `next-bench` package, and when none exists it + # rebuilds via prepare-next-benchmark.sh, which needs the dependency + # tree. Placed after the tests instead, it fails with "the Next + # dependency tree is not installed" -- which is how this was found. + - name: Set up Node for the Next fixture build + if: inputs.next_fixture + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Build and publish the Next.js fixture through Coop + if: inputs.next_fixture + shell: bash + env: + # Peak compile RSS scales with CONCURRENT LLVM units, and this + # fixture's units are 15-21 MB of IR apiece. The default on this + # runner is 2 module jobs x 2 unit workers -- four at once, which + # peaked at 4.2 GB and tripped the daemon's cap. The runner has 2 + # vCPUs, so that concurrency was buying nothing anyway. + PERRY_MODULE_JOBS: "2" + PERRY_CODEGEN_UNIT_JOBS: "1" + run: | + set -euo pipefail + ( cd benchmarks/next-small && npm ci --no-audit --no-fund ) + free -m | sed "s/^/ mem: /" || true + scripts/prepare-next-benchmark.sh + - name: Loader, integrity, and lifecycle tests shell: bash run: | @@ -248,6 +284,18 @@ jobs: cargo test -p coop-worker --test plugin_roundtrip repeated_load_dispatch_shutdown_reclaims_executor_threads -- --ignored --nocapture cargo test -p coop-daemon --test auto_compile -- --nocapture cargo test -p coop-worker --test plugin_roundtrip hundred_preloaded_apps_dispatch -- --ignored --nocapture + # NOTE: binary_http_roundtrip is NOT run here. It loads the + # published next-bench package, and building that package needs + # more memory than a GitHub-hosted runner has (see the + # `next_fixture` input). It runs on a larger host instead. Leaving + # it in unconditionally would fail every run for a reason that has + # nothing to do with the change under test. + # The Next.js fixture, which until now ran in NO workflow at all. + # It is excluded from fast-check as a provider suite (it needs built + # provider images) and was never added here, so the only test that + # exercises a real framework route was gated by nothing. A proof + # that never runs its most interesting case is the failure mode this + # repository keeps finding in its own gates. ' - name: Linux Perry 1/10/100 RSS, PSS, private-dirty, and cgroup evidence @@ -281,6 +329,65 @@ jobs: 2>&1 | tee linux-node-resource-results.txt ' + # The density matrices above measure the TINY dependency-free app, which + # shares almost everything through the providers and therefore sits near + # the floor for marginal cost per deployment. The architecture's claim is + # about real applications, so measure one. + # + # This is meaningful rather than a trick: Perry compiles the Next route + # INTO the application dylib, so a loaded copy needs the shared providers + # and nothing else -- no node_modules, no build output. N copies are the + # genuine marginal cost of an Nth Next deployment on one box. + # + # Node is required to BUILD the fixture (`npm ci` + `next build`), never + # to run it. `prepare-next-benchmark.sh` rebuilds the production route + # whenever the source is newer, because a committed build output silently + # drifted from its source once already. + - name: Linux Next.js 1/10/100 RSS, PSS, private-dirty, and cgroup evidence + if: inputs.next_fixture + shell: bash + env: + COOP_BENCH_APP_COUNTS: 1,10,100 + COOP_BENCH_TRIALS: "2" + COOP_BENCH_REQUESTS: "2000" + COOP_BENCH_PRELOAD_CONCURRENCY: "4" + run: | + set -euo pipefail + # Resolve the package the daemon just published. Named explicitly so a + # missing fixture fails here, rather than silently measuring the tiny + # app again and reporting it as Next. + app="$(find target/next-benchmark/coop-run/compiled/next-bench \ + -mindepth 2 -maxdepth 2 -name 'app.so' -print -quit)" + if [ -z "$app" ]; then + echo "::error::no published next-bench package to measure" + find target/next-benchmark/coop-run/compiled -maxdepth 3 2>&1 | head -20 + exit 1 + fi + # The density harness COPIES the app image once per deployment + # (resource_benchmark.rs: `std::fs::copy(source_app, &app)`), so 100 + # apps means 100 physical copies. The tiny fixture is trivial; this + # one carries Next's whole compiled server surface, so check the + # arithmetic BEFORE spending 20 minutes discovering it as ENOSPC -- + # which surfaces as unrelated-looking failures rather than "disk + # full" (see the phantom codegen bug in PerryTS/perry#8228). + app_mib=$(( $(stat -c %s "$app") / 1048576 )) + free_mib=$(df -m --output=avail . | tail -1 | tr -d ' ') + need_mib=$(( app_mib * 111 )) # 1 + 10 + 100 copies + echo "app image: ${app_mib} MiB; 111 copies need ~${need_mib} MiB; free ${free_mib} MiB" + if [ "$need_mib" -gt "$free_mib" ]; then + echo "::error::Not enough disk to replicate the Next fixture 100x:" + echo "::error:: need ~${need_mib} MiB, have ${free_mib} MiB free." + echo "::error::Reduce COOP_BENCH_APP_COUNTS or free space on the runner." + exit 1 + fi + echo "measuring Next fixture: $app" + COOP_BENCH_APP_LIBRARY="$PWD/$app" scripts/run-in-delegated-cgroup.sh bash -c ' + set -euo pipefail + cargo test --release -p coop-daemon --test resource_benchmark \ + measure_in_process_startup_and_rss -- --ignored --nocapture \ + 2>&1 | tee linux-next-resource-results.txt + ' + - name: Upload raw Linux evidence if: always() uses: actions/upload-artifact@v4 @@ -289,6 +396,7 @@ jobs: path: | linux-resource-results.txt linux-node-resource-results.txt + linux-next-resource-results.txt linux-benchmark-environment.txt target/dynamic-smoke/prepare.log if-no-files-found: warn diff --git a/benchmarks/next-small/README.md b/benchmarks/next-small/README.md index bc09e45..4f904d9 100644 --- a/benchmarks/next-small/README.md +++ b/benchmarks/next-small/README.md @@ -37,6 +37,32 @@ hook. Reverse them and `next/server` resolves to the edge build, whose module init throws `Invariant: AsyncLocalStorage accessed in runtime where it is not available`. + +## Where this fixture is measured, and why not in CI + +Compiling it peaks **above 8.3 GB RSS**. A GitHub-hosted runner has 7.75 GB, so +it cannot complete there at any cap setting — both attempts died at the +daemon's limit rather than at their true peak, which is why the first +measurement (4.2 GB) was the cap and not the peak. + +Reducing Perry's codegen concurrency would fix it, but Coop deliberately calls +`env_clear()` before spawning the compiler so ambient `PERRY_*` switches cannot +silently change emitted code without changing build identity, and +`COMPILER_ENV_ALLOWLIST` carries toolchain paths only. That guard is worth more +than the convenience of overriding it. + +So the Linux proof gates the tiny dependency-free fixture, and the Next steps +are behind the `next_fixture` workflow-dispatch input, off by default. Run them +on a host that can carry the compile. It has been done on an 8-core/8 GB M1 +mini in about eight minutes — note that is barely more RAM than the runner, so +the deciding factor is not memory alone: macOS compresses and swaps under +pressure, while the Linux path is stopped dead by the daemon's `compile_max_rss_mb` +cap. A machine with more real memory is still the safer choice. + +Verified there against the pinned Perry: a COOP request frame in, +`AppRouteRouteModule.handle` executed natively, `status: 200` and the route's +own body out. + Build the Node form from this directory: ```sh diff --git a/benchmarks/next-small/coop/coop-handler.ts b/benchmarks/next-small/coop/coop-handler.ts index 49e0905..afa7b11 100644 --- a/benchmarks/next-small/coop/coop-handler.ts +++ b/benchmarks/next-small/coop/coop-handler.ts @@ -37,7 +37,10 @@ // // So accept either shape and fail loudly if neither carries a routeModule, // rather than dereferencing undefined and reporting something confusing. -import * as routeBundleNamespace from "../.next/server/app/api/benchmark/route.js"; +// Staged as ordinary deployment source by prepare-next-benchmark.sh. Outside +// node_modules and without a leading dot, so Perry compiles it natively -- +// location is what decides AOT vs runtime-JS classification, not extension. +import * as routeBundleNamespace from "../next-build/server/app/api/benchmark/route.js"; // IMPORT ORDER IS LOAD-BEARING. The route bundle must be loaded BEFORE // `next/server`: loading it installs Next's require hook, and without that @@ -46,11 +49,23 @@ import * as routeBundleNamespace from "../.next/server/app/api/benchmark/route.j // available". Verified by reordering these two lines and watching it break. import { NextRequest } from "next/server"; -const routeBundle: Record = - (routeBundleNamespace as Record).routeModule !== undefined - ? (routeBundleNamespace as Record) - : (((routeBundleNamespace as Record).default ?? - {}) as Record); +// Resolved with plain statements rather than a nested ternary plus `??`. +// Measured shape under Perry for a webpack CommonJS bundle: +// ns keys : default, module.exports +// ns.routeModule : undefined +// ns.default : object <- the real exports live here +// default.routeModule: object, and .handle IS a function +// The interop shape differs by toolchain, so try the namespace first and fall +// back to `default`, and keep the expression simple enough to be obviously +// correct at a glance. +const routeNamespace = routeBundleNamespace as unknown as Record; +let routeBundle: Record = routeNamespace; +if (routeNamespace.routeModule === undefined) { + const fallback = routeNamespace.default; + if (fallback !== undefined && fallback !== null) { + routeBundle = fallback as Record; + } +} const routeModule = routeBundle.routeModule as | { handle: (request: object, context: unknown) => Promise } diff --git a/benchmarks/next-small/next.config.ts b/benchmarks/next-small/next.config.ts index 68a6c64..887ac57 100644 --- a/benchmarks/next-small/next.config.ts +++ b/benchmarks/next-small/next.config.ts @@ -2,6 +2,22 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { output: "standalone", + // Emit ONE server chunk. Next's server bundles otherwise load chunks with a + // computed require -- webpack: `require("./chunks/" + id + ".js")`, + // turbopack: `require(path.resolve(RUNTIME_ROOT, chunkPath))` -- and an + // ahead-of-time compiler cannot statically resolve either, so the chunk + // never lands in the binary and the route fails at first dispatch. + // + // Disabling server-side splitting makes the route self-contained, which is + // what an AOT host needs. It is also closer to what a serverless bundle + // looks like, so it is not an exotic configuration. + webpack: (config, { isServer }) => { + if (isServer) { + config.optimization.splitChunks = false; + config.optimization.runtimeChunk = false; + } + return config; + }, }; export default nextConfig; diff --git a/crates/coop-daemon/tests/resource_benchmark.rs b/crates/coop-daemon/tests/resource_benchmark.rs index 4deaa9e..fd16dc7 100644 --- a/crates/coop-daemon/tests/resource_benchmark.rs +++ b/crates/coop-daemon/tests/resource_benchmark.rs @@ -13,6 +13,7 @@ use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::sync::mpsc; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const PACKAGE_DIGEST_V2_DOMAIN: &[u8] = b"coop-application-package-v2\0"; @@ -370,6 +371,66 @@ fn locate_prepared_app(workspace: &Path, extension: &str) -> PathBuf { candidates.pop().unwrap_or(legacy) } +/// Request path the benchmark drives, and the substring its response must +/// contain. +/// +/// The tiny fixture answers `/` with exactly `ok`, and the harness asserted +/// both literally. That is a FIXTURE CONTRACT, not a property of the +/// benchmark: point `COOP_BENCH_APP_LIBRARY` at the Next.js fixture and it +/// serves `/api/benchmark` with JSON, so the run died on `left: 404` before +/// measuring anything. +/// +/// A substring rather than an exact match, because a real application's body +/// carries incidental detail (checksums, timing) that would make an equality +/// assertion brittle without making it stronger. The point is to prove the +/// request was actually served, not to re-verify the payload -- correctness +/// belongs to `binary_http_roundtrip`. +/// Concurrency and per-request timeout for the workload phase. +/// +/// Both were hardcoded (50 concurrent, 10 s) around the tiny fixture, whose +/// handler answers in microseconds. A real Next.js route is orders of +/// magnitude heavier per dispatch, and 50 concurrent requests queued on one +/// app's executor exceed a 10 s client timeout — on a QUIET host, so this is +/// the workload's shape rather than contention. Measured: 50 requests pass in +/// 3.4 s, 500 time out. +/// Execution mode under measurement. +/// +/// `in_process` runs every app in the daemon address space — maximum density, +/// and the arm that Perry's process-global runtime state currently limits to +/// one JS heap. `worker` gives each deployment its own process, which sidesteps +/// that entirely. +/// +/// The comparison decides whether the in-process model is worth a large change +/// to Perry: a `.dylib`'s text pages are already shared across processes by the +/// kernel, so `worker` may capture most of the memory win without it. +fn bench_execution_mode() -> String { + std::env::var("COOP_BENCH_EXECUTION_MODE").unwrap_or_else(|_| "in_process".to_string()) +} + +fn bench_workload_concurrency() -> usize { + std::env::var("COOP_BENCH_WORKLOAD_CONCURRENCY") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(50) +} + +fn bench_request_timeout() -> Duration { + Duration::from_secs( + std::env::var("COOP_BENCH_REQUEST_TIMEOUT_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10), + ) +} + +fn bench_request_path() -> String { + std::env::var("COOP_BENCH_REQUEST_PATH").unwrap_or_else(|_| "/".to_string()) +} + +fn bench_expect_body() -> String { + std::env::var("COOP_BENCH_EXPECT_BODY").unwrap_or_else(|_| "ok".to_string()) +} + fn workspace_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() @@ -484,6 +545,7 @@ method = "GET" } let port = pick_free_port(); + let execution_mode = bench_execution_mode(); let config = root.join("runtime.toml"); std::fs::write( &config, @@ -492,7 +554,7 @@ method = "GET" listen_http = "127.0.0.1:{port}" [execution] -mode = "in_process" +mode = "{execution_mode}" preload_concurrency = {preload_concurrency} [paths] @@ -560,13 +622,43 @@ fn start_and_wait( if let Some(cgroup) = cgroup { command.arg("--self-cgroup-procs").arg(cgroup.procs_path()); } + // stderr was `Stdio::null()`, so a daemon that failed to start reported + // only "exit status: 1" and threw its reason away. Every diagnosis of a + // failed preload then required reproducing the run by hand. Capture it and + // print it on the failure paths instead: the daemon already says exactly + // what went wrong, we were just discarding it. let mut child = command .env("RUST_LOG", "info") .stdout(Stdio::piped()) - .stderr(Stdio::null()) + .stderr(Stdio::piped()) .spawn() .expect("spawn benchmark daemon"); let stdout = child.stdout.take().expect("capture daemon stdout"); + let stderr = child.stderr.take().expect("capture daemon stderr"); + let captured_errors = Arc::new(Mutex::new(Vec::::new())); + let error_sink = Arc::clone(&captured_errors); + std::thread::spawn(move || { + for line in BufReader::new(stderr).lines().map_while(Result::ok) { + let mut sink = error_sink.lock().expect("daemon stderr sink"); + // Bounded: a failing daemon can be chatty, and the tail is the + // part that explains the exit. + if sink.len() == 400 { + sink.remove(0); + } + sink.push(line); + } + }); + let report_errors = || { + let sink = captured_errors.lock().expect("daemon stderr sink"); + if sink.is_empty() { + " (daemon produced no stderr)".to_string() + } else { + sink.iter() + .map(|line| format!(" daemon: {line}")) + .collect::>() + .join("\n") + } + }; let (line_tx, line_rx) = mpsc::channel(); std::thread::spawn(move || { for line in BufReader::new(stdout).lines().map_while(Result::ok) { @@ -580,8 +672,9 @@ fn start_and_wait( loop { let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { + let errors = report_errors(); stop(&mut child); - panic!("daemon did not become ready within {READY_TIMEOUT:?}"); + panic!("daemon did not become ready within {READY_TIMEOUT:?}\n{errors}"); } match line_rx.recv_timeout(remaining.min(Duration::from_secs(1))) { Ok(line) => { @@ -594,12 +687,15 @@ fn start_and_wait( } Err(mpsc::RecvTimeoutError::Timeout) => { if let Some(status) = child.try_wait().expect("query daemon status") { - panic!("daemon exited before ready: {status}"); + panic!("daemon exited before ready: {status}\n{}", report_errors()); } } Err(mpsc::RecvTimeoutError::Disconnected) => { let status = child.wait().expect("wait for failed daemon"); - panic!("daemon output closed before ready: {status}"); + panic!( + "daemon output closed before ready: {status}\n{}", + report_errors() + ); } } } @@ -607,27 +703,35 @@ fn start_and_wait( async fn warm_every_app(port: u16, app_count: usize) { let client = reqwest::Client::builder() - .timeout(Duration::from_secs(10)) + .timeout(bench_request_timeout()) .build() .expect("build benchmark HTTP client"); for index in 0..app_count { let name = app_name(index); let response = client - .get(format!("http://127.0.0.1:{port}/")) + .get(format!("http://127.0.0.1:{port}{}", bench_request_path())) .header("host", format!("{name}.bench")) .send() .await .expect("dispatch warm request"); assert_eq!(response.status(), 200, "warm request for {name}"); let body = response.bytes().await.expect("read warm response"); - assert_eq!(body.as_ref(), b"ok", "warm response for {name}"); + let expected = bench_expect_body(); + assert!( + String::from_utf8_lossy(body.as_ref()).contains(&expected), + "warm response for {name} did not contain {expected:?}: {}", + String::from_utf8_lossy(body.as_ref()) + .chars() + .take(200) + .collect::() + ); } } async fn run_workload(port: u16, app_count: usize, requests: usize) { let client = reqwest::Client::builder() .pool_max_idle_per_host(100) - .timeout(Duration::from_secs(10)) + .timeout(bench_request_timeout()) .build() .expect("build workload HTTP client"); stream::iter(0..requests) @@ -636,42 +740,92 @@ async fn run_workload(port: u16, app_count: usize, requests: usize) { let name = app_name(index % app_count); async move { let response = client - .get(format!("http://127.0.0.1:{port}/")) + .get(format!("http://127.0.0.1:{port}{}", bench_request_path())) .header("host", format!("{name}.bench")) .send() .await .expect("dispatch workload request"); assert_eq!(response.status(), 200); - assert_eq!( - response - .bytes() - .await - .expect("read workload response") - .as_ref(), - b"ok" + let body = response.bytes().await.expect("read workload response"); + let expected = bench_expect_body(); + assert!( + String::from_utf8_lossy(body.as_ref()).contains(&expected), + "workload response did not contain {expected:?}" ); } }) - .buffer_unordered(50) + .buffer_unordered(bench_workload_concurrency()) .collect::>() .await; } -fn median_rss_kib(pid: u32) -> u64 { - let mut readings = Vec::with_capacity(7); - for _ in 0..7 { - let output = Command::new("ps") - .args(["-o", "rss=", "-p", &pid.to_string()]) - .output() - .expect("sample daemon RSS"); - assert!(output.status.success(), "ps failed while sampling RSS"); - readings.push( +/// Resident memory of the daemon AND every process it spawned. +/// +/// This sampled only the daemon's own pid. In `in_process` mode that is the +/// whole system, so the figures were right. In `worker` mode each deployment +/// runs in its OWN process, none of which the sampler looked at — so ten apps +/// reported ~18 MiB, indistinguishable from one, and worker mode appeared +/// dramatically denser than in-process. That is not density, it is an empty +/// daemon: an instrument that cannot see what it claims to measure. +/// +/// Summing the tree keeps `in_process` unchanged (it has no children holding +/// apps) while making `worker` mean what the column header says. +fn descendant_pids(root: u32) -> Vec { + // One `ps` snapshot, then walk the ppid graph. Repeated `pgrep -P` calls + // race against worker restarts and can miss a generation. + let output = Command::new("ps") + .args(["-eo", "pid=,ppid="]) + .output() + .expect("list processes"); + assert!(output.status.success(), "ps failed while listing processes"); + let text = String::from_utf8_lossy(&output.stdout).into_owned(); + let mut children: std::collections::HashMap> = std::collections::HashMap::new(); + for line in text.lines() { + let mut parts = line.split_whitespace(); + if let (Some(pid), Some(ppid)) = (parts.next(), parts.next()) { + if let (Ok(pid), Ok(ppid)) = (pid.parse::(), ppid.parse::()) { + children.entry(ppid).or_default().push(pid); + } + } + } + let mut out = vec![root]; + let mut queue = vec![root]; + while let Some(next) = queue.pop() { + if let Some(kids) = children.get(&next) { + for kid in kids { + if !out.contains(kid) { + out.push(*kid); + queue.push(*kid); + } + } + } + } + out +} + +fn tree_rss_kib(root: u32) -> u64 { + descendant_pids(root) + .iter() + .filter_map(|pid| { + let output = Command::new("ps") + .args(["-o", "rss=", "-p", &pid.to_string()]) + .output() + .ok()?; + // A worker can exit between the snapshot and this sample; skip it + // rather than failing the whole measurement. String::from_utf8(output.stdout) - .expect("ps output is UTF-8") + .ok()? .trim() .parse::() - .expect("parse RSS in KiB"), - ); + .ok() + }) + .sum() +} + +fn median_rss_kib(pid: u32) -> u64 { + let mut readings = Vec::with_capacity(7); + for _ in 0..7 { + readings.push(tree_rss_kib(pid)); std::thread::sleep(Duration::from_millis(25)); } median_u64(readings) diff --git a/scripts/prepare-next-benchmark.sh b/scripts/prepare-next-benchmark.sh index 7c887ee..0342798 100755 --- a/scripts/prepare-next-benchmark.sh +++ b/scripts/prepare-next-benchmark.sh @@ -15,6 +15,12 @@ source_root="${COOP_NEXT_SOURCE_DIR:-$repo_root/benchmarks/next-small}" perry="${COOP_BENCH_PERRY:-$repo_root/.perry-main/target/perry-dev/perry}" provider_verification="${COOP_BENCH_PROVIDER_VERIFICATION:-full_hash}" timeout_seconds="${COOP_NEXT_PREPARE_TIMEOUT:-1200}" +# Compile peak for this fixture is well above 6 GB and has never been measured +# to completion under a cap -- every run so far died AT the limit, so each +# reported figure was the cap and not the peak. Raise this on a host with real +# memory; the default is a floor that keeps a constrained runner from swapping +# itself to death, not a statement about what the compile needs. +max_rss_mb="${COOP_NEXT_MAX_RSS_MB:-6144}" case "$(uname -s)" in Darwin) extension="dylib" ;; @@ -71,8 +77,14 @@ fi route_build="$source_root/.next/server/app/api/benchmark/route.js" if [[ ! -f "$route_build" || "$source_root/app/api/benchmark/route.ts" -nt "$route_build" ]]; then echo "building the production Next App Route (source is newer than the build)" >&2 - ( cd "$source_root" && npx --no-install next build >/dev/null ) || \ - fail "next build failed in $source_root" "(cd $source_root && npx next build)" + # --webpack is not optional. Next 16 defaults to turbopack, whose runtime + # loads chunks with `require(path.resolve(RUNTIME_ROOT, chunkPath))` -- a + # computed require an ahead-of-time compiler cannot resolve, so the chunk + # never enters the binary and the route dies on first dispatch. The + # fixture's own package.json script has always said `next build --webpack`; + # invoking `next build` directly silently changed the bundler. + ( cd "$source_root" && npx --no-install next build --webpack >/dev/null ) || \ + fail "next build failed in $source_root" "(cd $source_root && npm run build)" fi if [[ ! -f "$route_build" ]]; then fail "next build produced no $route_build" \ @@ -108,9 +120,42 @@ cp "$source_root/coop/coop.toml" "$deployment/coop.toml" cp "$source_root/coop/coop-handler.ts" "$deployment/handlers/main.ts" cp "$source_root/app/api/benchmark/route.ts" "$deployment/app/api/benchmark/route.ts" ln -sfn "$source_root/node_modules" "$deployment/node_modules" -# The production build output, linked like the dependency tree: it is -# generated, not source, and the daemon dereferences it into its snapshot. -ln -sfn "$source_root/.next" "$deployment/.next" +# Stage the production build output as ordinary deployment source. +# +# Location is the signal, not extension. `collect_modules.rs` compiles a +# `.js`/`.cjs`/`.mjs` file through the native AOT pipeline exactly like a `.ts` +# file when it is project source, and classifies it as a runtime-JS module only +# when it sits under `node_modules`. There is no V8 fallback any more, so that +# classification is now a refusal. +# +# Two earlier attempts here were wrong in instructive ways: +# +# * a `.next` SYMLINK beside the handler -- `collect_source_files` skips +# dot-directories and rejects symlinks outright, so it never reached the +# compiler at all and Perry reported a missing module. +# * a copy into `node_modules/.coop-next-bundle/` -- that reached the +# compiler, but sitting under node_modules gave it the runtime-JS +# classification, and the leading dot excluded it from +# `collect_packages_in_node_modules`, so the automatic `compilePackages` +# `"*"` expansion (the default when no package.json pins the key) never +# covered it either. Both of those were self-inflicted. +# +# Plain directory, no dot, outside node_modules: `collect_source_files` walks +# it, every `.js` is collected, and Perry compiles the whole bundle natively +# with no opt-in required. +bundle_dir="$deployment/next-build" +rm -rf "$bundle_dir" +mkdir -p "$bundle_dir" +cp -R "$source_root/.next/server" "$bundle_dir/server" + +# Assert the route reached the staging tree. The failure this guards is silent +# and costs a full CI cycle: the import resolves to nothing and Perry reports a +# missing module rather than a missing FILE. +staged_route="$bundle_dir/server/app/api/benchmark/route.js" +if [[ ! -f "$staged_route" ]]; then + fail "the production route did not reach the staged bundle: $staged_route" \ + "check that next build produced .next/server/app/api/benchmark/route.js" +fi # The pre-2026-08-14 hand-built fixture lived at this mutable path. Coop no # longer publishes there, and leaving it behind lets a stale library outlive a @@ -138,6 +183,25 @@ listen_http = "$2" [execution] mode = "in_process" provider_verification = "$provider_verification" +# The default 300 s is sized for ordinary application code. This fixture +# compiles Next's whole production server surface natively -- the App Route +# runtime alone is ~15-21 MB of IR across 400-535 functions, which Perry drops +# to -Os because LLVM's -O1+ pipeline will not survive functions that wide +# (#4880). Measured at roughly 8 minutes on a quiet M1; a 2-vCPU CI runner is +# slower still. +# +# It got slower for a good reason: disabling server chunk splitting made +# route.js self-contained, so there is simply more to compile. The earlier +# 77-second compile was the split build, which then failed at runtime because +# the chunks were loaded by a computed require. +compile_timeout_seconds = 1800 +# Peak compile RSS scales with concurrent LLVM units, and this fixture's units +# are enormous (15-21 MB of IR each). At the default 2 module jobs x 2 unit +# workers the compile peaked at 4.2 GB and tripped the 4 GB cap. The workflow +# pins concurrency to 2 total units; this leaves headroom above that without +# approaching the runner's 7.75 GB, where the kernel OOM killer would replace +# a clean refusal with a mysterious death. +compile_max_rss_mb = $max_rss_mb [paths] deployments_dir = "$fixture_root/deployments" @@ -175,7 +239,19 @@ trap cleanup EXIT INT TERM # an earlier run once satisfied a naive file check with a package the daemon # had just refused. loaded() { - grep -F 'application library preloaded on dedicated Perry thread' "$log_file" \ + # Two load paths, two log lines. A dedicated worker logs "preloaded on + # dedicated Perry thread"; an in-daemon load (isolation resolving to + # "trusted", which is what this fixture gets) logs "preloading application + # library in daemon". Waiting only for the first meant a perfectly good + # build sat until the one-shot daemon exited, and the script then reported + # "Coop exited before publishing" for a fixture it had already published -- + # the log said `compilation succeeded and immutable package was published` + # three lines above the error. + # + # Matching either keeps this honest about what it is waiting for: the app + # being loaded, not the particular thread it landed on. The published-package + # check below is what actually validates the result. + grep -E 'application library preloaded on dedicated Perry thread|preloading application library in daemon' "$log_file" \ | grep -Fq 'next-bench' }