Compile in-process on OCaml domains in OCaml rewatch

CONTRIBUTING.md

@@ -284,14 +284,14 @@ npm link rescript
 
 #### Use Local BSC with Existing ReScript Installation
 
-Alternatively, you can set the `RESCRIPT_BSC_EXE` environment variable to point to your locally compiled `bsc.exe`.
+The `rescript` build system has the compiler built in, so to test a local compiler change, link the local package as shown above.
+
+The Rust build system, `rescript-rust`, still runs `bsc.exe` and accepts the `RESCRIPT_BSC_EXE` environment variable pointing to your locally compiled `bsc.exe`:
 
 ```sh
-RESCRIPT_BSC_EXE=your-rescript-repo/packages/@rescript/darwin-arm64/bin/bsc.exe npx rescript
+RESCRIPT_BSC_EXE=your-rescript-repo/packages/@rescript/darwin-arm64/bin/bsc.exe npx rescript-rust
 ```
 
-This will test the local compiler while still using the build system from the installed Node module.
-
 ### Running Automatic Tests
 
 We provide different test suites for different levels of the compiler and build system infrastructure. Always make sure to locally build your compiler before running any tests.

changes/in-process-compilation.polish.md

@@ -0,0 +1 @@
+- Speed up `rescript` builds by compiling modules inside the build system on several CPU cores, instead of starting `bsc` for every file. https://github.com/rescript-lang/rescript/pull/8781

changes/rescript-bsc-exe.breaking.md

@@ -0,0 +1 @@
+- `rescript` compiles with its embedded compiler and ignores `RESCRIPT_BSC_EXE`, which now only affects `rescript-rust`. https://github.com/rescript-lang/rescript/pull/8781

compiler/bsc/build_identity_gen.ml

@@ -0,0 +1,38 @@
+let () =
+  let relevant file =
+    Filename.basename file = "dune"
+    || List.exists (Filename.check_suffix file) [".ml"; ".mli"; ".c"; ".h"]
+  in
+  let files =
+    Array.to_list Sys.argv |> List.tl |> List.filter relevant
+    |> List.sort String.compare
+  in
+  let normalize path =
+    String.map
+      (function
+        | '\\' -> '/'
+        | character -> character)
+      path
+  in
+  let has_suffix suffix =
+    List.exists
+      (fun file -> Filename.check_suffix (normalize file) suffix)
+      files
+  in
+  [
+    "/flow_parser/parser/expression_parser.ml";
+    "/ext/ext_platform_primitives_stubs.c";
+    "/ext/platform/native/ext_platform_primitives.ml";
+    "/core/platform/native/js_name_of_module_id.ml";
+    "/syntax/src/res_driver.ml";
+    "/syntax/src/dune";
+  ]
+  |> List.iter (fun suffix ->
+      if not (has_suffix suffix) then
+        failwith ("compiler identity is missing required input " ^ suffix));
+  let digest =
+    files
+    |> List.map (fun file -> file ^ "\000" ^ Digest.file file)
+    |> String.concat "\000" |> Digest.string
+  in
+  Printf.printf "let value = %S\n" (Digest.to_hex digest)

compiler/bsc/dune

@@ -3,10 +3,25 @@
   (flags
    (:standard -ccopt -static))))
 
+(executable
+ (name build_identity_gen)
+ (modules build_identity_gen))
+
+(rule
+ (target rescript_compiler_build_identity.ml)
+ ; Every compiler source, so code in a new directory is covered. Sources that
+ ; are not linked into bsc only cause an extra clean rebuild after they change.
+ (deps
+  (source_tree ..))
+ (action
+  (with-stdout-to
+   %{target}
+   (run ./build_identity_gen.exe %{deps}))))
+
 (library
  (name rescript_compiler_driver)
  (wrapped false)
- (modules rescript_compiler_driver)
+ (modules rescript_compiler_driver rescript_compiler_build_identity)
  (instrumentation
   (backend bisect_ppx))
  (libraries common core depends flow_parser gentype syntax unix))

compiler/bsc/rescript_compiler_driver.ml

@@ -554,6 +554,8 @@ let () =
 
 type result = {exit_code: int; stdout: string; stderr: string}
 
+let build_identity = Rescript_compiler_build_identity.value
+
 (* Request_local gives every request the state of a new compiler process.
    On top of that, a request needs bsc's own defaults, and allocates the shared
    predefined type graphs in a fixed order: lazy first use on different worker

compiler/bsc/rescript_compiler_driver.mli

@@ -1,5 +1,10 @@
 type result = {exit_code: int; stdout: string; stderr: string}
 
+val build_identity : string
+(** A digest of the compiler implementation linked into this driver. The
+    standalone compiler and embedded callers therefore use the same identity
+    without fingerprinting a separately installed executable. *)
+
 val run_request :
   run_external:(string -> int * string * string) option ->
   cwd:string ->

rewatch-ocaml/README.md

@@ -7,8 +7,10 @@ implementation in [`../rewatch`](../rewatch) ships next to it as
 `rescript-rust`, and both are tested with the shared integration suite in
 [`../rewatch/tests`](../rewatch/tests).
 
-The port follows Rust rewatch's architecture and algorithms. Every parse and
-compile runs `bsc` as a separate process.
+The port follows Rust rewatch's architecture and algorithms, but it links the
+compiler in: parses and compiles run in-process on a pool of OCaml domains
+rather than as `bsc` subprocesses (see
+[In-process compilation](#in-process-compilation)).
 
 ## Building and running
 
@@ -20,16 +22,20 @@ make                                     # builds bsc and packages/@rescript/<pl
 dune build rewatch-ocaml/rescript_ocaml.exe   # just this executable
 ```
 
-The packaged `rescript.exe` finds `bsc.exe` next to itself, and the npm
-launcher (`cli/rescript.js`) supplies the runtime path. To run the dune
-executable directly, point it at a compiler and runtime:
+The npm launcher (`cli/rescript.js`) supplies the runtime path. To run the
+dune executable directly, point it at the runtime:
 
 ```sh
-export RESCRIPT_BSC_EXE="$PWD/packages/@rescript/linux-arm64/bin/bsc.exe"
 export RESCRIPT_RUNTIME="$PWD/packages/@rescript/runtime"
 _build/default/rewatch-ocaml/rescript_ocaml.exe build path/to/project
 ```
 
+Builds and `rescript format` use the embedded compiler, so
+`RESCRIPT_BSC_EXE` only affects Rust rewatch (`rescript-rust`). The
+`bsc.exe` next to `rescript.exe` is still recorded in
+`lib/bs/compiler-info.json`, where editor tooling looks for the platform
+binaries.
+
 Linux release builds use dune's `static` profile, so the published executable
 does not depend on the runner's libc. The version lives in
 `rewatch_version.ml` and is kept in sync with the compiler and Rust rewatch by
@@ -44,7 +50,8 @@ does not depend on the runner's libc. The version lives in
 | Projects and packages | `project_context` (workspace roots and dependency lookup), `package_resolution`, `package_traversal`, `package_graph`, `package_plan`, `package_diagnostics` |
 | Sources | `source`, `source_filter` (`--filter`), `source_dirs` |
 | Build orchestration | `build` (one build or watch rebuild), `build_preparation`, `build_session` (state kept across watch rebuilds), `build_attempt` (state of one attempt), `build_state`, `build_report` |
-| Parsing and compiling | `package_parse`, `package_compilation`, `compiler_args`, `compiler_process`, `compiler_scheduler`, `compiler_info` (fingerprints that decide when a package is cleaned), `compile_assets`, `module_graph`, `graph` |
+| Parsing and compiling | `package_parse`, `package_compilation`, `compiler_args`, `compiler_process` (logical compiler jobs and their in-process execution), `compiler_execution_mode` (worker count), `compiler_scheduler`, `compiler_info` (fingerprints that decide when a package is cleaned), `compile_assets`, `module_graph`, `graph` |
+| Embedded compiler | `compiler/bsc/rescript_compiler_driver` (one request: argv in, exit code and captured output out), `compiler/ext/compiler_request_state` and `compiler_request_output` (per-request working directory and output capture) |
 | Artifacts | `build_artifacts` (output paths, publication, stale cleanup), `build_freshness`, `clean`, `file_util` |
 | Processes | `process` (worker pool), `process_child` (one child and its pipes), `after_build` |
 | Platform | `platform.mli` implemented by `platform_unix.ml` or `platform_windows.ml` (selected by dune), `windows_job_stubs.c`, `termination_signals`, `build_lock` |
@@ -55,6 +62,25 @@ Native filesystem notifications only wake the watcher. The snapshot taken
 afterwards decides what changed, so merged, reordered, or dropped events can't
 cause missed rebuilds.
 
+## In-process compilation
+
+Independent parse and compile requests run on a bounded pool of OCaml domains.
+The pool has `min(8, max(1, CPUs - 1))` workers; set `REWATCH_COMPILER_DOMAINS`
+to choose another count: a positive integer, lowered to the machine's limit of
+`max(12, min(32, CPUs))`. The main domain schedules dependencies
+and publishes every artifact; workers only run compiler requests.
+
+The driver gives each request fresh compiler state (flags, warnings, caches,
+predefined types, identifier counters) and restores it on every exit, so
+requests on different domains don't interfere and repeated requests produce
+byte-identical artifacts. Compiler output is captured per request instead of
+going to the process's stdout and stderr. The compiler build identity is
+derived from the compiler sources, so compiler changes clean the build while
+rewatch-only changes don't.
+
+External PPXs, `js-post-build` hooks, and `--after-build` remain subprocesses
+under the process owners described below.
+
 ## Differences from Rust rewatch
 
 These are intentional:
@@ -82,6 +108,9 @@ These are intentional:
 - Watch mode also notices a missing dependency being installed into an
   existing `node_modules` of an ancestor directory, where the dependency
   lookup finds it.
+- A damaged published AST, such as one cut short by a killed build, makes its
+  module parse again. Rust reads whatever dependency lines it finds in it, and
+  panics if the file can't be read.
 - OpenTelemetry export is not implemented.
 
 ## Platforms

rewatch-ocaml/ast_header.ml

@@ -6,15 +6,27 @@ let read path =
   Fun.protect
     ~finally:(fun () -> close_in_noerr channel)
     (fun () ->
-      (try ignore (input_line channel) with End_of_file -> ());
-      let rec loop dependencies =
-        match input_line channel with
-        | line ->
-          let line = String.trim line in
-          if line = "" then loop dependencies
-          else if Filename.is_relative line then loop (line :: dependencies)
-          else {dependencies = List.rev dependencies; source = Some line}
-        | exception End_of_file ->
-          {dependencies = List.rev dependencies; source = None}
+      (* The first four bytes give the exact length of the newline-delimited
+         dependency block. The following source path can itself be relative;
+         scanning until an absolute path would read the marshalled AST. *)
+      let read_header () =
+        let dependency_length = input_binary_int channel in
+        (* Checked before allocating: a damaged length can claim gigabytes. *)
+        if
+          dependency_length < 0
+          || dependency_length > in_channel_length channel - pos_in channel
+        then raise End_of_file;
+        let dependencies =
+          really_input_string channel dependency_length
+          |> String.split_on_char '\n'
+          |> List.filter (fun name -> name <> "")
+        in
+        let source = try Some (input_line channel) with End_of_file -> None in
+        {dependencies; source}
       in
-      loop [])
+      (* A damaged AST, such as one cut short by a killed build, has no usable
+         header: it ends early or gives an impossible length. Like a missing
+         AST, it makes the module parse again. *)
+      try read_header ()
+      with End_of_file | Invalid_argument _ ->
+        {dependencies = []; source = None})

rewatch-ocaml/ast_header.mli

@@ -1,7 +1,7 @@
 type t = {dependencies: string list; source: string option}
 (** Published AST headers are read without deserializing the compiler's full AST.
     Cleanup needs the original source path, while graph construction needs only
-    dependency names; keeping one reader prevents those wire-format rules from
-    drifting. *)
+    dependency names. The encoded dependency byte length separates those names
+    from either an absolute or a relative source path and the marshalled AST. *)
 
 val read : string -> t

rewatch-ocaml/bench/README.md

@@ -82,6 +82,14 @@ that a differently-cased recreated CMI exists to `stat` and then return
 `ENOENT` from the immediately following `open`; that host-filesystem artifact
 is not valid scheduler or benchmark evidence. The harness's default `mktemp`
 workspace normally stays on the container filesystem.
+
+The OCaml implementation compiles in-process, so it has no compiler `execve`
+to trace. The gate counts its compiler requests from the log it writes when
+`REWATCH_COMPILER_CALL_LOG` is set, and still traces PPXs as processes.
+`RESCRIPT_BSC_EXE` only selects Rust's compiler; the OCaml executable uses the
+compiler linked into it, so build both from the same checkout. Set
+`REWATCH_COMPILER_DOMAINS` to measure a specific number of compiler workers.
+
 Set `REWATCH_PERFORMANCE_THRESHOLD_PERCENT` to change the 125% threshold. Set `KEEP_REWATCH_BENCHMARK_WORKDIR=1` to retain traces and raw
 stdout/stderr for investigation. For a quick correctness-only check, an odd run
 count below five is accepted only with `REWATCH_ALLOW_SMOKE_RUN=1`; its timing
@@ -100,8 +108,11 @@ rewatch-ocaml/bench/watch_performance_gate.sh \
 ```
 
 It warms both implementations, interleaves an odd number of timed edits,
-requires byte-identical generated JavaScript and normalized parser/compiler
-argument logs, and samples file descriptors, tasks, and RSS after every build.
+requires byte-identical generated JavaScript and equal parser/compiler work
+counts (Rust's through the counting `bsc` proxy, OCaml's through
+`REWATCH_COMPILER_CALL_LOG`), and samples file descriptors, tasks, and RSS after
+every build. Set `REWATCH_WATCH_COMPILER_DOMAINS` to run the OCaml watcher with
+a specific number of compiler workers.
 This catches retained-state implementations that appear fast by skipping work,
 as well as resource growth that a single build cannot show. The
 default median-latency limit is 150% of Rust because individual watch events

rewatch-ocaml/bench/performance_gate.sh

@@ -89,10 +89,11 @@ tree_resources() {
       for (pass = 0; pass < NR; pass++)
         for (i = 1; i <= NR; i++)
           if (live[parent[pids[i]]]) live[pids[i]] = 1
-      for (pid in live) {
-        total_memory += memory[pid]
-        total_tasks += tasks[pid]
-      }
+      for (pid in live)
+        if (live[pid]) {
+          total_memory += memory[pid]
+          total_tasks += tasks[pid]
+        }
       print total_memory + 0, total_tasks + 0
     }'
 }
@@ -171,20 +172,35 @@ trace_and_classify() {
   local implementation=$1 scenario=$2 executable=$3 fixture=$4 manifest=$5
   local clean_first=$6
   local trace_prefix="$work_root/${implementation}-${scenario}.execve"
+  local call_log="$work_root/${implementation}-${scenario}.compiler"
   if [[ "$clean_first" == 1 ]]; then
     "$executable" clean "$fixture" >/dev/null 2>&1
   fi
-  strace -f -ff -qq -s 4096 -e trace=execve,chdir -o "$trace_prefix" \
-    "$executable" build "$fixture" \
-    >"$work_root/${implementation}-${scenario}-trace.out" \
-    2>"$work_root/${implementation}-${scenario}-trace.stderr"
+  if [[ $implementation == ocaml ]]; then
+    # Embedded requests have no compiler execve. Record them at their shared
+    # logical boundary while tracing PPXs as external processes.
+    strace -f -ff -qq -s 4096 -e trace=execve,chdir -o "$trace_prefix" \
+      env REWATCH_COMPILER_CALL_LOG="$call_log" \
+      "$executable" build "$fixture" \
+      >"$work_root/${implementation}-${scenario}-trace.out" \
+      2>"$work_root/${implementation}-${scenario}-trace.stderr"
+  else
+    strace -f -ff -qq -s 4096 -e trace=execve,chdir -o "$trace_prefix" \
+      "$executable" build "$fixture" \
+      >"$work_root/${implementation}-${scenario}-trace.out" \
+      2>"$work_root/${implementation}-${scenario}-trace.stderr"
+  fi
   local trace_files=("$trace_prefix".*)
   local implementation_root=${fixture%/rewatch/testrepo}
   local trace_file exec_line argv cwd_line cwd phase input identity
   : >"$manifest.unsorted"
   for trace_file in "${trace_files[@]}"; do
-    exec_line=$(grep -m1 -F "execve(\"$RESCRIPT_BSC_EXE\"" "$trace_file" \
-      || grep -m1 -E 'execve\("[^"]*sury-ppx' "$trace_file" || true)
+    if [[ $implementation == ocaml ]]; then
+      exec_line=$(grep -m1 -E 'execve\("[^"]*sury-ppx' "$trace_file" || true)
+    else
+      exec_line=$(grep -m1 -F "execve(\"$RESCRIPT_BSC_EXE\"" "$trace_file" \
+        || grep -m1 -E 'execve\("[^"]*sury-ppx' "$trace_file" || true)
+    fi
     if [[ -z "$exec_line" ]]; then
       continue
     fi
@@ -193,7 +209,7 @@ trace_and_classify() {
     cwd_line=$(grep -m1 '^chdir("' "$trace_file" || true)
     cwd=${cwd_line#chdir(\"}
     cwd=${cwd%%\"*}
-    if [[ "$exec_line" == *"execve(\"$RESCRIPT_BSC_EXE\""* ]]; then
+    if [[ $implementation == rust && "$exec_line" == *"execve(\"$RESCRIPT_BSC_EXE\""* ]]; then
       if [[ "$argv" == *'"-bs-ast"'* ]]; then
         phase=parse
       elif [[ "$argv" == *'.mlmap"'* ]]; then
@@ -205,24 +221,35 @@ trace_and_classify() {
       input=${input%]}
       identity="$cwd"$'\t'"$phase"$'\t'"$input"
     else
-      # PPX temporary input/output names are deliberately randomized. Its
-      # executable identity and count are the stable unit of work.
+      # PPX temporary input/output names are deliberately randomized.
       identity=ppx$'\t'"${argv%%,*}"
     fi
     printf '%s\n' "$identity" \
       | sed "s#$implementation_root#<ROOT>#g" >>"$manifest.unsorted"
   done
-  sort "$manifest.unsorted" >"$manifest"
   local invocations parse namespace compile interface ppx
-  invocations=$(grep -hF -c "execve(\"$RESCRIPT_BSC_EXE\"" "${trace_files[@]}" \
-    | awk '{ total += $1 } END { print total + 0 }')
-  parse=$(grep -hF "execve(\"$RESCRIPT_BSC_EXE\"" "${trace_files[@]}" \
-    | grep -F -c '"-bs-ast"' || true)
-  namespace=$(grep -hF "execve(\"$RESCRIPT_BSC_EXE\"" "${trace_files[@]}" \
-    | grep -E -c '\.mlmap"' || true)
-  interface=$(grep -hF "execve(\"$RESCRIPT_BSC_EXE\"" "${trace_files[@]}" \
-    | grep -vF '"-bs-ast"' | grep -vE '\.mlmap"' \
-    | grep -E -c '\.iast"' || true)
+  if [[ $implementation == ocaml ]]; then
+    while IFS=$'\t' read -r phase cwd input; do
+      if [[ $phase != parse && $phase != namespace ]]; then phase=compile; fi
+      printf '%s\t%s\t"%s"\n' "$cwd" "$phase" "$input" \
+        | sed "s#$implementation_root#<ROOT>#g" >>"$manifest.unsorted"
+    done <"$call_log"
+    invocations=$(wc -l <"$call_log")
+    parse=$(awk -F '\t' '$1 == "parse" { n++ } END { print n+0 }' "$call_log")
+    namespace=$(awk -F '\t' '$1 == "namespace" { n++ } END { print n+0 }' "$call_log")
+    interface=$(awk -F '\t' '$1 == "interface" { n++ } END { print n+0 }' "$call_log")
+  else
+    invocations=$(grep -hF -c "execve(\"$RESCRIPT_BSC_EXE\"" "${trace_files[@]}" \
+      | awk '{ total += $1 } END { print total + 0 }')
+    parse=$(grep -hF "execve(\"$RESCRIPT_BSC_EXE\"" "${trace_files[@]}" \
+      | grep -F -c '"-bs-ast"' || true)
+    namespace=$(grep -hF "execve(\"$RESCRIPT_BSC_EXE\"" "${trace_files[@]}" \
+      | grep -E -c '\.mlmap"' || true)
+    interface=$(grep -hF "execve(\"$RESCRIPT_BSC_EXE\"" "${trace_files[@]}" \
+      | grep -vF '"-bs-ast"' | grep -vE '\.mlmap"' \
+      | grep -E -c '\.iast"' || true)
+  fi
+  sort "$manifest.unsorted" >"$manifest"
   compile=$((invocations - parse - namespace))
   ppx=$(grep -hE -c 'execve\("[^"]*sury-ppx' "${trace_files[@]}" \
     | awk '{ total += $1 } END { print total + 0 }')