test(fspy): benchmark tracking overhead

.github/workflows/ci.yml

@@ -358,7 +358,9 @@ jobs:
       - run: cargo autoinherit && git diff --exit-code
       - run: cargo shear --deny-warnings
       - run: cargo fmt --check
-      - run: RUSTDOCFLAGS='-D warnings' cargo doc --no-deps --document-private-items
+      # --exclude fspy_benchmark: documenting it would build its musl artifact
+      # dependency, which needs a rust-std this job otherwise has no use for.
+      - run: RUSTDOCFLAGS='-D warnings' cargo doc --workspace --exclude fspy_benchmark --no-deps --document-private-items
 
       - uses: crate-ci/typos@bee27e3a4fd1ea2111cf90ab89cd076c870fce14 # v1.48.0
         with:

.github/workflows/fspy-benchmark.yml

@@ -0,0 +1,178 @@
+name: fspy benchmark
+
+permissions: {}
+
+on:
+  workflow_dispatch:
+  pull_request:
+  # Runs on main keep the build cache warm and record the overhead history.
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.ref_name != 'main' }}
+
+defaults:
+  run:
+    shell: bash
+
+# Benchmark results are never compared across runs: runner instances disagree
+# by more than a regression worth catching. Instead, each pull request job
+# builds the benchmark launcher twice — against the fspy under review and
+# against the fspy of the merge base — and the harness interleaves both builds
+# on the same machine. See crates/fspy_benchmark/README.md.
+jobs:
+  benchmark:
+    name: Benchmark (${{ matrix.platform }})
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux
+            os: namespace-profile-linux-x64-default
+            static: true
+          - platform: macos
+            os: namespace-profile-mac-default
+            static: false
+          - platform: windows
+            os: namespace-profile-windows-4c-8g
+            static: false
+    runs-on: ${{ matrix.os }}
+    env:
+      BASE_DIR: ${{ github.workspace }}/../fspy-benchmark-base
+    steps:
+      - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2
+
+      - name: Update Detours submodule
+        if: matrix.platform == 'windows'
+        run: git submodule update --init --recursive
+
+      - uses: oxc-project/setup-rust@3d6fb132fbe7cdcb66bf8ec193911c2945369d12 # v1.0.17
+        with:
+          # Caches saved by pull request runs are scoped to their branch, so
+          # only runs on main seed the cache every other run restores.
+          save-cache: ${{ github.ref_name == 'main' }}
+          cache-key: fspy-benchmark-${{ matrix.platform }}
+
+      - name: Install static target
+        if: matrix.static
+        run: rustup target add x86_64-unknown-linux-musl
+
+      - name: Check out merge base
+        if: github.event_name == 'pull_request'
+        run: |
+          # The checkout is the pull request merged into the base branch, so
+          # its first parent is exactly the base tip the merge was built on.
+          git fetch --depth 2 origin "refs/pull/${{ github.event.pull_request.number }}/merge"
+          git worktree add "$BASE_DIR" "$(git rev-parse FETCH_HEAD^1)"
+
+      - name: Update baseline Detours submodule
+        if: github.event_name == 'pull_request' && matrix.platform == 'windows'
+        working-directory: ${{ env.BASE_DIR }}
+        run: git submodule update --init --recursive
+
+      - name: Build baseline launcher
+        if: github.event_name == 'pull_request'
+        working-directory: ${{ env.BASE_DIR }}
+        env:
+          # Share the head build's target directory: the crates.io graph is
+          # fingerprinted by package and profile, not by workspace, so only the
+          # fspy path crates build twice.
+          CARGO_TARGET_DIR: ${{ github.workspace }}/target
+        run: |
+          # Both fspy revisions must be measured by identical code, and the
+          # baseline may predate the launcher crate, so overlay the head's
+          # launcher source before building it against the baseline fspy.
+          rm -rf crates/fspy_benchmark_launcher
+          cp -R "$GITHUB_WORKSPACE/crates/fspy_benchmark_launcher" crates/fspy_benchmark_launcher
+          cargo build --release -p fspy_benchmark_launcher
+          # Set the binary aside before the head build overwrites it.
+          exe=""
+          [[ "$RUNNER_OS" == "Windows" ]] && exe=.exe
+          cp "$CARGO_TARGET_DIR/release/fspy_benchmark_launcher$exe" "$RUNNER_TEMP/fspy-base-launcher$exe"
+
+      - name: Run benchmark
+        run: |
+          exe=""
+          [[ "$RUNNER_OS" == "Windows" ]] && exe=.exe
+          args=()
+          if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
+            args=(--base-launcher "$RUNNER_TEMP/fspy-base-launcher$exe")
+          fi
+          mkdir -p .benchmark-results
+          set -o pipefail
+          cargo run --release -p fspy_benchmark -- "${args[@]}" |
+            tee ".benchmark-results/${{ matrix.platform }}.txt"
+
+      - name: Add job summary
+        if: ${{ !cancelled() }}
+        run: |
+          {
+            echo '```text'
+            cat ".benchmark-results/${{ matrix.platform }}.txt" ||
+              echo 'no report; the benchmark failed before producing one'
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload benchmark report
+        # Matrix jobs have isolated filesystems, so persist each report for the
+        # comment job to combine into one sticky comment after all platforms
+        # finish. Fork pull requests get no comment, so nothing consumes their
+        # reports; the job summary already carries them.
+        if: >-
+          !cancelled() &&
+          github.event_name == 'pull_request' &&
+          github.event.pull_request.head.repo.full_name == github.repository
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: fspy-benchmark-report-${{ matrix.platform }}
+          path: .benchmark-results/${{ matrix.platform }}.txt
+          # A job that failed before producing a report has nothing to upload;
+          # the comment job prints its own placeholder for the platform.
+          if-no-files-found: warn
+          retention-days: 1
+
+  comment:
+    name: Report benchmark
+    if: >-
+      !cancelled() &&
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    needs: benchmark
+    runs-on: namespace-profile-linux-x64-default
+    permissions:
+      actions: read
+      pull-requests: write
+    steps:
+      - name: Download benchmark reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: fspy-benchmark-report-*
+          path: .benchmark-results
+          merge-multiple: true
+
+      - name: Assemble comment
+        run: |
+          {
+            echo '## fspy benchmark'
+            echo
+            for platform in linux macos windows; do
+              echo "### $platform"
+              echo
+              echo '```text'
+              cat ".benchmark-results/$platform.txt" ||
+                echo "no report; the $platform benchmark job failed before producing one"
+              echo '```'
+              echo
+            done
+          } > .benchmark-results/comment.md
+
+      - name: Update PR comment
+        uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4
+        with:
+          header: fspy-benchmark
+          path: .benchmark-results/comment.md

Cargo.toml

@@ -72,6 +72,8 @@ materialized_artifact = { path = "crates/materialized_artifact" }
 materialized_artifact_build = { path = "crates/materialized_artifact_build" }
 flate2 = "1.0.35"
 fspy = { path = "crates/fspy" }
+fspy_benchmark_launcher = { path = "crates/fspy_benchmark_launcher", artifact = "bin" }
+fspy_benchmark_target = { path = "crates/fspy_benchmark_target", artifact = "bin" }
 fspy_detours_sys = { path = "crates/fspy_detours_sys" }
 fspy_preload_unix = { path = "crates/fspy_preload_unix", artifact = "cdylib", target = "target" }
 fspy_preload_windows = { path = "crates/fspy_preload_windows", artifact = "cdylib", target = "target" }

crates/fspy/Cargo.toml

@@ -18,7 +18,7 @@ ouroboros = { workspace = true }
 rustc-hash = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
-tokio = { workspace = true, features = ["net", "process", "io-util", "sync", "rt"] }
+tokio = { workspace = true, features = ["macros", "net", "process", "io-util", "sync", "rt"] }
 tokio-util = { workspace = true }
 which = { workspace = true, features = ["tracing"] }
 

crates/fspy_benchmark/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "fspy_benchmark"
+version = "0.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish = false
+rust-version.workspace = true
+
+[lints]
+workspace = true
+
+[[bin]]
+name = "fspy_benchmark"
+test = false
+doctest = false
+
+[dependencies]
+fspy_benchmark_launcher = { workspace = true }
+fspy_benchmark_target = { workspace = true }
+
+[target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dependencies]
+fspy_benchmark_static_target = { path = "../fspy_benchmark_static_target", artifact = "bin", target = "x86_64-unknown-linux-musl" }
+
+[package.metadata.cargo-shear]
+ignored = ["fspy_benchmark_launcher", "fspy_benchmark_target", "fspy_benchmark_static_target"]

crates/fspy_benchmark/README.md

@@ -0,0 +1,32 @@
+# fspy benchmark
+
+Measures what fspy costs a process it tracks, and whether a change to fspy moved that cost. It reports two rows:
+
+- `launch`: the wall clock of a tracked launch that opens nothing. This is the cost of starting a process under tracking: injection, session setup, and teardown.
+- `access`: how long a batch of opens takes, timed by two threads that each open their own path. This is the cost of interception itself, under the concurrency a tracked process normally has.
+
+Linux measures a dynamically linked target (`LD_PRELOAD`) and a `x86_64-unknown-linux-musl` target (seccomp user notification). macOS measures `DYLD_INSERT_LIBRARIES`. Windows measures Detours injection.
+
+Run the overhead report locally with:
+
+```sh
+just benchmark-fspy
+```
+
+## How regressions are caught
+
+Comparing a pull request run against a saved result from an earlier `main` run does not work. The two runs land on different runner instances, and identical instances disagree by more than any regression worth catching: tens of percent on absolute times, and still up to ten percent after normalizing against an untracked launch. A threshold above that noise is a threshold above every real regression.
+
+So nothing is ever compared across runs. Both fspy revisions run side by side on one machine, in one job. Three pieces make that possible:
+
+- `fspy_benchmark_target` (and its static twin) is the workload. It opens missing paths from several threads and prints the median batch time. It does not link fspy. Batches are a few microseconds long, so the median lands on batches the scheduler left alone; a whole-run wall clock would add up every disturbance instead.
+- `fspy_benchmark_launcher` runs the target once, tracked through `fspy::Command` or untracked, and prints the launch wall clock plus the target's number. It is the only piece that links fspy.
+- `fspy_benchmark` is the harness. CI builds the launcher twice: against the fspy under review, and against the merge-base fspy. Both builds use the same launcher source, so both revisions are measured by identical code. The harness then launches both builds back to back, cycling every ordering. Whatever the runner does to the numbers, it does to both.
+
+Each iteration gives one head/base ratio per row. The reported change is the median of those ratios, printed with its quartile spread. The benchmark only reports; it never fails the job. Running the same fspy on both sides stays within a couple of percent, so read a change well past that as real.
+
+An untracked launch runs in the same rotation. It prices tracking itself, and is reported as context, never gated.
+
+Locally there is no second fspy build, so `just benchmark-fspy` reports only the overhead column. Pass `--base-launcher` with another build of `fspy_benchmark_launcher` to compare two revisions by hand.
+
+One caveat: the launcher is compiled against both revisions from the head's source. A pull request that breaks the small part of the `fspy::Command` API the launcher uses will fail the baseline build. Keep that API surface small; if it must break, expect a red benchmark on that one pull request.