diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..ced9065 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,122 @@ +# Performance benchmarks -- INFORMATIONAL ONLY. +# +# This workflow never fails on a timing. There are deliberately no thresholds, +# no regression gates and no "must not be slower than" comparisons, because +# GitHub's shared runners cannot support one: the CPU model varies between jobs, +# the host is shared with other tenants, and run-to-run noise on the same commit +# routinely reaches tens of percent. A gate built on that would either fire +# constantly on noise or be set so loose it catches nothing, and either way the +# first thing anyone would do is start ignoring it. +# +# What it IS for: producing a benchmark artifact on demand, from a known commit, +# with a known input, so a human can download two of them and compare. When a +# number matters, run `benchmarks/run_all.sh` on one machine against both +# branches back to back -- see docs/features/benchmarking.md. + +name: bench + +on: + workflow_dispatch: + inputs: + preset: + description: "Synthetic repo size for the harness" + required: false + default: medium + type: choice + options: [small, medium, large] + pull_request: + types: [labeled, synchronize] + +permissions: + contents: read + +jobs: + bench: + # On a PR, only when it carries the `bench` label; always on manual dispatch. + if: >- + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'bench') + runs-on: ubuntu-latest + timeout-minutes: 45 + + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: ". -> target" + + - name: Build harness + run: cargo build --release --example perf_harness -p cc-core + + - name: Generate synthetic repo + env: + PRESET: ${{ github.event.inputs.preset || 'medium' }} + run: | + mkdir -p "$RUNNER_TEMP/bench-repos" + python3 benchmarks/gen_repo.py \ + --preset "$PRESET" \ + --out "$RUNNER_TEMP/bench-repos/cc-$PRESET" \ + --force + + - name: Run end-to-end harness + env: + PRESET: ${{ github.event.inputs.preset || 'medium' }} + run: | + mkdir -p bench-results + ./target/release/examples/perf_harness \ + --repo "$RUNNER_TEMP/bench-repos/cc-$PRESET" \ + --label "${GITHUB_REF_NAME}/${PRESET}" \ + --out bench-results/harness-synthetic.json + + - name: Run end-to-end harness on this repo + run: | + ./target/release/examples/perf_harness \ + --repo . \ + --label "${GITHUB_REF_NAME}/self" \ + --out bench-results/harness-self.json + + # Reduced sampling: enough to see an order of magnitude, cheap enough to + # finish inside the job timeout. Not enough for a small-percentage claim -- + # which is fine, because this workflow does not make one. + - name: Run criterion + run: | + cargo bench -p cc-core -- \ + --warm-up-time 1 --measurement-time 2 --sample-size 10 \ + | tee bench-results/criterion.txt + + - uses: pnpm/action-setup@v4 + with: + version: 9 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install frontend deps + run: pnpm install --frozen-lockfile + + - name: Run edge-routing benchmark + working-directory: packages/app + run: | + node benchmarks/edgeRouting.bench.ts \ + --label "${GITHUB_REF_NAME}" \ + --sizes 300x200,800x500 \ + --reps 3 \ + --json ../../bench-results/edge-routing.json + + - name: Collect criterion raw estimates + if: always() + run: | + if [ -d target/criterion ]; then + tar -czf bench-results/criterion-raw.tar.gz target/criterion + fi + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bench-${{ github.sha }} + path: bench-results/ + retention-days: 30 diff --git a/benchmarks/RESULTS.md b/benchmarks/RESULTS.md new file mode 100644 index 0000000..1a168c9 --- /dev/null +++ b/benchmarks/RESULTS.md @@ -0,0 +1,360 @@ +# `main` vs `feat/perf-integration` -- measured + +One machine, sequential runs, same inputs. Method and caveats are stated before +the numbers because several of them change what the numbers mean. + +- **Baseline**: `origin/main` @ `30ab9b5` (2026-08-02) +- **Branch**: `origin/feat/perf-integration` @ `1e8903e` (2026-08-11), PR #51 +- **Measured from**: `test/perf-benchmarks` @ `ab94907`, whose benchmark files + were copied UNCOMMITTED into a detached `main` worktree so both sides run + byte-identical measurement code. No source patch was needed: the harness and + both bench files compile unchanged on `main`. The only extra file copied in was + `crates/cc-core/Cargo.toml`, for its `[lib] bench = false` line (without it + `cargo bench -p cc-core -- ` fails before running anything). + +## Environment + +| | | +| --- | --- | +| CPU | AMD Ryzen 7 7840U (8 cores / 16 threads, 3.30 GHz max) | +| Memory | 60 GiB | +| OS | Ubuntu 26.04 LTS, Linux 7.0.0-29-generic x86_64 | +| Rust | rustc 1.99.0-nightly (8ab9fdff5 2026-07-30) | +| Node | v24.10.0 | +| Python | 3.14.4 | + +Laptop CPU with boost and thermal headroom in play: treat sub-5% differences as +noise. The frontend control scenario below shows the actual run-to-run agreement +achieved (under 1%). + +## Build profiles -- read this before the criterion table + +`cargo bench` inherits from `[profile.release]`, and the branch changes it: + +| | `main` | `feat/perf-integration` | +| --- | --- | --- | +| `[profile.release]` | cargo defaults (`lto = false`, `codegen-units = 16`) | `lto = "thin"`, `codegen-units = 1` | + +That asymmetry is real shipping behaviour, so the primary comparison keeps it. +But it inflates every number, including ones the branch did not touch, so there +is also a **profile-matched** column: the branch re-benched with +`CARGO_PROFILE_BENCH_LTO=false CARGO_PROFILE_BENCH_CODEGEN_UNITS=16`. The gap +between the two columns is the build flags; what survives in the matched column +is the algorithm. + +The matched column reads as a control: every benchmark the branch did not touch +lands within ±2% (`add_edge_unique`, `add_edge_mixed`, `rebuild_adjacency`, +`extract_many_files`, `full_pipeline`), while the targeted ones keep their full +win. The 5-8% that those untouched benchmarks show in the shipping column is +thin LTO, not a change in the code. + +## Exact commands + +```bash +# Inputs, generated once, outside both checkouts +for p in small medium large; do + python3 benchmarks/gen_repo.py --preset $p --out "$BENCH/cc-$p" --force +done + +# Baseline +git worktree add --detach "$BENCH/main-baseline" origin/main +# ... copy benchmarks/, crates/cc-core/examples/, crates/cc-core/benches/, +# packages/app/benchmarks/ and crates/cc-core/Cargo.toml in, uncommitted +cd "$BENCH/main-baseline" && pnpm install --dir packages/app +bash benchmarks/run_all.sh main "$BENCH/results/main" "$BENCH" "$BENCH/main-baseline" + +# Branch +bash benchmarks/run_all.sh perf-integration "$BENCH/results/integration" \ + "$BENCH" "$BENCH/main-baseline" + +# Profile-matched criterion re-run on the branch +CARGO_PROFILE_BENCH_LTO=false CARGO_PROFILE_BENCH_CODEGEN_UNITS=16 \ + cargo bench -p cc-core -- --warm-up-time 1 --measurement-time 3 --sample-size 20 +``` + +Both harness runs point at the SAME real-repo path (`$BENCH/main-baseline`), so +the "real repo" case reads an identical source tree on both sides. Synthetic +repos come from the same seed (`20240611`) and are byte-identical. + +Criterion: `--warm-up-time 1 --measurement-time 3 --sample-size 20`; groups that +set their own sampling (the newer, heavier ones) keep it at 20 samples / 3 s. + +--- + +# Headline + +| Metric | `main` | `feat/perf-integration` | Delta | +| --- | --- | --- | --- | +| Open a 10k-file repo, full pipeline | 10 750 ms | 3 068 ms | **-71.5%** | +| ...of which symbol resolution | 8 435 ms | 789 ms | **-90.6%** | +| 200 neighborhood queries (10k-file repo) | 24 147 ms | 6 170 ms | **-74.4%** | +| Subgraph battery, 24 extractions (10k-file repo) | 14 021 ms | 11 331 ms | -19.2% | +| 50 edge-detail drill-ins (10k-file repo) | 12 686 ms | 7 185 ms | -43.4% | +| Parse payload, 10k-file repo | 69.61 MB | 64.21 MB | -7.8% | +| Canvas redraw, 1500 nodes / 1200 edges | 3 643 ms | ~0 ms (routing skipped) | **-100%** | +| Canvas redraw, 800 nodes / 500 edges | 789 ms | 136 ms | **-82.7%** | + +--- + +## 1. End-to-end harness + +`crates/cc-core/examples/perf_harness.rs`, release build, seed 20240611. +Subgraph render sets are measured 6x each; `steady` is the mean of samples 2-6. + +### 1a. Synthetic `large` -- 11 365 files, 142 730 nodes, 374 703 edges, 950 000 raw refs + +| Metric | `main` | branch | Delta | +| --- | --- | --- | --- | +| scan | 131.5 ms | 128.8 ms | -2.1% | +| parse files (rayon) | 741.9 ms | 676.4 ms | -8.8% | +| merge blocks | 238.9 ms | 228.3 ms | -4.5% | +| build symbol table | 533.9 ms | 567.4 ms | +6.3% | +| resolve imports | 104.4 ms | 118.0 ms | +12.9% | +| insert import edges | 46.3 ms | 49.9 ms | +7.8% | +| **resolve symbols** | **8 434.7 ms** | **789.2 ms** | **-90.6%** | +| insert symbol edges | 518.4 ms | 509.8 ms | -1.7% | +| **pipeline total** | **10 750.1 ms** | **3 067.8 ms** | **-71.5%** | +| payload bytes | 69.61 MB | 64.21 MB | -7.8% | +| payload build | 285.7 ms | 174.4 ms | -39.0% | +| payload serialize | 144.0 ms | 217.4 ms | **+51.0%** | +| payload build + serialize | 429.7 ms | 391.8 ms | -8.8% | +| subgraph battery (4 render sets x 6) | 14 021.2 ms | 11 330.7 ms | -19.2% | +|   directories_only (1 365 render nodes), steady | 591.7 ms | 481.9 ms | -18.6% | +|   directories_and_files (12 730), steady | 558.4 ms | 446.0 ms | -20.1% | +|   quarter_of_files_expanded (32 770), steady | 651.4 ms | 512.2 ms | -21.4% | +|   fully_expanded (142 730), steady | 538.6 ms | 436.0 ms | -19.1% | +| neighborhood x200, depth 2 | 24 146.5 ms | 6 169.9 ms | **-74.4%** | +| edge_detail x50 | 12 685.8 ms | 7 184.5 ms | -43.4% | + +### 1b. Synthetic `medium` -- 2 333 files, 28 666 nodes, 75 451 edges + +| Metric | `main` | branch | Delta | +| --- | --- | --- | --- | +| resolve symbols | 331.4 ms | 143.0 ms | -56.8% | +| pipeline total | 744.4 ms | 542.2 ms | -27.2% | +| payload bytes | 12.73 MB | 11.66 MB | -8.4% | +| payload build + serialize | 52.2 ms | 51.9 ms | -0.6% | +| subgraph battery | 2 061.7 ms | 1 620.4 ms | -21.4% | +| neighborhood x200 | 3 151.4 ms | 1 080.7 ms | -65.7% | +| edge_detail x50 | 1 656.0 ms | 1 095.3 ms | -33.9% | + +### 1c. Synthetic `small` -- 233 files, 2 666 nodes, 3 880 edges + +| Metric | `main` | branch | Delta | +| --- | --- | --- | --- | +| pipeline total | 33.1 ms | 27.2 ms | -17.9% | +| payload bytes | 0.99 MB | 0.89 MB | -9.8% | +| subgraph battery | 61.7 ms | 47.1 ms | -23.5% | +| neighborhood x200 | 142.3 ms | 37.0 ms | -74.0% | +| edge_detail x50 | 56.2 ms | 29.6 ms | -47.4% | + +### 1d. The CodeCartographer repo itself -- 120 files, 1 317 nodes, 4 147 edges + +The realistic small-repo case: mixed Rust/TypeScript, real symbol distribution. + +| Metric | `main` | branch | Delta | +| --- | --- | --- | --- | +| pipeline total | 48.2 ms | 41.7 ms | -13.4% | +| payload bytes | 0.70 MB | 0.64 MB | -9.1% | +| payload build + serialize | 2.0 ms | 1.7 ms | -15.4% | +| subgraph battery | 81.9 ms | 66.5 ms | -18.8% | +| neighborhood x200 | 142.3 ms | 76.8 ms | -46.0% | +| edge_detail x50 | 60.0 ms | 43.7 ms | -27.2% | + +Note how flat the branch's subgraph times are across render-set sizes in every +table: on `main` the cost is dominated by rebuilding the parent map, which +depends on the GRAPH size, not the render set, so all four render sets cost about +the same. Removing that rebuild is what makes the numbers start tracking the work +the query actually does. + +## 2. Criterion microbenchmarks + +Median of 20 samples. "shipping" is each branch's own release profile; "matched" +is the branch rebuilt with `main`'s LTO/codegen-units. + +### Targeted by the perf work + +| Benchmark | `main` | branch (shipping) | branch (matched) | Delta (matched) | +| --- | --- | --- | --- | --- | +| `resolve_hub_ambiguity/ambiguous_resolve/1000` | 1.222 s | 3.77 ms | 4.53 ms | **-99.6%** | +| `resolve_hub_ambiguity/ambiguous_resolve/500` | 282.7 ms | 1.94 ms | 2.23 ms | **-99.2%** | +| `neighborhood_bfs/50000` | 1.021 s | 26.08 ms | 24.67 ms | **-97.6%** | +| `neighborhood_bfs/10000` | 101.1 ms | 3.90 ms | 3.77 ms | **-96.3%** | +| `subgraph_fully_expanded/50000` | 46.26 ms | 17.07 ms | 16.92 ms | -63.4% | +| `subgraph_fully_expanded/10000` | 5.49 ms | 1.88 ms | 2.00 ms | -63.6% | +| `subgraph_nested_collapsed/directories_only/10000` | 9.43 ms | 4.46 ms | 4.25 ms | -55.0% | +| `subgraph_nested_collapsed/directories_only/2000` | 1.066 ms | 532.6 us | 545.4 us | -48.8% | +| `subgraph_nested_collapsed/dirs_and_files/2000` | 1.037 ms | 524.0 us | 535.3 us | -48.4% | +| `subgraph_nested_collapsed/dirs_and_files/10000` | 7.98 ms | 4.17 ms | 4.18 ms | -47.6% | +| `subgraph_nested_collapsed/dirs_and_files/50000` | 71.05 ms | 39.06 ms | 40.02 ms | -43.7% | +| `subgraph_nested_collapsed/directories_only/50000` | 66.36 ms | 38.59 ms | 39.35 ms | -40.7% | +| `add_edge_all_duplicates/1000` | 112.3 us | 65.8 us | 72.6 us | -35.4% | +| `parse_result_serialize/10000` | 14.17 ms | 9.94 ms | 9.79 ms | -30.9% | +| `parse_result_serialize/50000` | 92.63 ms | 81.00 ms | 79.10 ms | -14.6% | + +### Not targeted -- the control + +| Benchmark | `main` | branch (shipping) | branch (matched) | Delta (matched) | +| --- | --- | --- | --- | --- | +| `add_edge_unique/5000` | 4.101 ms | 3.773 ms | 4.069 ms | -0.8% | +| `add_edge_mixed/5000` | 3.397 ms | 3.116 ms | 3.371 ms | -0.7% | +| `rebuild_adjacency/5000` | 4.223 ms | 4.177 ms | 4.156 ms | -1.6% | +| `subgraph_flat_no_hierarchy/nodes/2000` | 426.3 us | 401.2 us | 410.1 us | -3.8% | +| `extract_many_files/100` | 47.17 ms | 46.48 ms | 47.42 ms | +0.5% | +| `full_pipeline/50` | 26.45 ms | 25.76 ms | 26.34 ms | -0.4% | +| `resolve_hub_ambiguity/unique_control_resolve/1000` | 2.662 ms | 2.405 ms | 2.639 ms | -0.9% | +| `resolve_hub_ambiguity/ambiguous_build_table/1000` | 5.518 ms | 5.451 ms | 5.737 ms | +4.0% | + +`subgraph_flat_no_hierarchy` is the OLD subgraph fixture. It moves -3.8%, and +that is the entire point: on a flat graph the parent map is empty, the ancestor +walk never runs, and the change under test is invisible. The same code path on +the nested fixture moves -40% to -55%. A fixture that skips the hot path reports +a confident zero. + +## 3. Frontend edge routing + +`packages/app/benchmarks/edgeRouting.bench.ts`, mean of 5 reps, Node v24.10.0. + +| Layout | Scenario | `main` | branch | Delta | +| --- | --- | --- | --- | --- | +| 300n / 200e | `full_scan_routing` (control) | 134.5 ms | 133.2 ms | -1.0% | +| | `indexed_routing` | n/a (no `obstacleIndex.ts`) | 47.6 ms | | +| | **`shipped_redraw`** | **127.8 ms** | **45.4 ms** (mode `full`) | **-64.5%** | +| 800n / 500e | `full_scan_routing` (control) | 794.1 ms | 785.3 ms | -1.1% | +| | `indexed_routing` | n/a | 252.1 ms | | +| | **`shipped_redraw`** | **789.4 ms** | **136.3 ms** (mode `obstacles`) | **-82.7%** | +| 1500n / 1200e | `full_scan_routing` (control) | 3 721.6 ms | 3 718.9 ms | -0.1% | +| | `indexed_routing` | n/a | 1 223.8 ms | | +| | **`shipped_redraw`** | **3 643.1 ms** | **~0 ms** (mode `none`) | **-100%** | + +The control scenario -- obstacle collection by full scan, reimplemented inside +the bench so it is identical on both branches -- agrees to within 1.1% at every +size. That is the evidence that the two runs are comparable and that the other +rows are the code, not the machine. + +Reading the branch's own two rows against each other separates the two +mechanisms: the R-tree obstacle index alone is a 2.8x-3.0x win +(`full_scan_routing` -> `indexed_routing`), and the budget gate contributes the +rest by dropping crossing-aware scoring above 250 edges and skipping routing +entirely above 500 edges or 2000 nodes. + +`main` never took 3.6 s of main-thread time in one go for a 1200-edge view in +practice, because it hit ELK's own node-count guard first -- but nothing bounded +the redraw pass itself, which is what these numbers measure. + +--- + +# Claims: confirmed and contradicted + +## Confirmed + +1. **Cached parent map (`get_subgraph` / `neighborhood` / `edge_detail`).** + Strongly confirmed, and larger than advertised for BFS. `neighborhood_bfs` at + 50k nodes drops 97.6%, because `main` rebuilt a 50k-entry `HashMap` -- two `String` clones per child link -- on every single query. In the + harness, 200 neighborhood queries on the 10k-file repo go from 24.1 s to 6.2 s + (-74.4%), and 50 edge-detail drill-ins from 12.7 s to 7.2 s (-43.4%). + Subgraph extraction is -19% to -21% end to end and -41% to -55% in the + microbenchmark. + +2. **Ambiguity early-bail in symbol resolution.** Strongly confirmed, and the + single biggest win in the suite. On the synthetic 10k-file repo (half the + modules defining the same hub names), symbol resolution drops from 8.43 s to + 0.79 s (-90.6%), taking the whole pipeline from 10.75 s to 3.07 s (-71.5%). + The microbenchmark isolates it: with 1000 files defining `__init__`/`new`/ + `get`/`run`/`handle`, resolving 10 000 references goes from 1.22 s to 3.8 ms. + `main` was cloning hundreds of `NodeId`s per reference and then discarding all + of them at the 5-candidate cap. The matching `unique_control` benchmark, same + volume with unique names, is unchanged (-0.9%) -- so nothing was traded away + for it. + +3. **Obstacle-indexed edge routing plus budget gates.** Confirmed. The index + alone is 2.8x-3.0x; the shipped redraw is -64.5% at 300n/200e, -82.7% at + 800n/500e and -100% at 1500n/1200e where the budget skips routing outright. + The 250/500-edge gates behave exactly as documented. + +4. **`into_iter` edge insertion.** Real but small: `add_edge_all_duplicates/1000` + is -35% and edge insertion in the harness is within noise (-1.7% at the large + size). It removes a clone per edge; edge insertion was never the bottleneck. + +## Contradicted or overstated + +5. **"~-22% payload" for the slim `ParseResult`. Not reproduced.** Measured + payload reduction is **-7.8%** (synthetic large), **-8.4%** (medium), **-9.8%** + (small) and **-9.1%** (the CodeCartographer repo itself). Dumping the `main` + payload for this repo and measuring per field explains why: + + | field | bytes | share of payload | + | --- | --- | --- | + | `children` | 103 362 | 14.7% | + | `id` | 90 610 | 12.9% | + | node-map keys | 84 025 | 11.9% | + | `span` | 81 003 | 11.5% | + | **`signature`** | **64 206** | **9.1%** | + | `parent` | 63 917 | 9.1% | + | `name` | 38 025 | 5.4% | + | everything else | ~79 000 | ~11% | + + So the docstring in `graph.rs` (and `docs/features/server_side_graph_state.md`) + calling `signature` "the single largest contributor to the payload" is wrong + for this repo: it is the fifth largest, behind `children`, `id`, the node-map + keys and `span`. The node id is in fact paid for **three times** -- as the map + key, as the `id` field, and again as each child's entry in its parent's + `children` array -- which together are 39.5% of the payload. That is where the + next payload win is, not in dropping more per-node fields. + +6. **"Serialization without the node-map deep clone" is a clear time win.** + Half right, and worth knowing. Building the `ParseResult` IS much cheaper + (-39.0% at the large size, -35.9% at small: no deep clone). But SERIALIZING + the slim borrowed form is **slower** -- +51.0% at the large size, +71.2% at + medium -- despite emitting fewer bytes. Net build+serialize is -8.8% (large), + -0.6% (medium), -15.4% (this repo): somewhere between neutral and modest, not + the step change the build-side number alone suggests. The plausible mechanism + is locality: `main` serializes a freshly written contiguous clone, the branch + walks the live node map and constructs a `SlimNode` per entry. Worth a look if + payload time ever matters; the memory saving (no second copy of the node map) + stands regardless. + +7. **Import resolution and symbol-table construction got slightly slower** on + the largest input (+12.9% and +6.3% respectively at the 10k-file size, though + -16% to -33% at the small size). This is a few hundred milliseconds against a + 7.7-second win elsewhere, and the direction flips with size, so it is most + likely allocator/cache behaviour at the new working-set size rather than a + real regression. Flagged rather than explained. + +## Where the numbers came from + +Raw output (harness JSON per size, criterion text for all three profiles, +frontend JSON) is not committed -- rerun `benchmarks/run_all.sh` to regenerate +it. The procedure is documented in `docs/features/benchmarking.md`. + +## Real repo: Megatron-LM (2026-08-11) + +The repo that originally motivated the performance work +(https://github.com/nvidia/megatron-lm, shallow clone): 1,279 parsed Python +files, 22,368 nodes, 59,056 edges, 149,219 raw references. `perf_harness` +release builds on both branches, sequential runs on the same machine, seed +20240611, **median of 3 runs per side**. Both sides produced byte-identical +graph statistics (same node/edge/resolution counts). + +| Metric | main | feat/perf-integration | delta | +| --- | --- | --- | --- | +| Full pipeline (scan -> parse -> resolve) | 922 ms | 912 ms | -1.2% (noise band) | +| Neighborhood battery (200 queries, depth 2) | 2,989 ms | 1,117 ms | **-62.6%** | +| Edge-detail battery (50 drill-ins) | 1,343 ms | 828 ms | **-38.3%** | +| Subgraph battery (24 render sets x 6 reps) | 1,831 ms | 1,531 ms | -16.4% | +| ParseResult build (steady mean) | 35.0 ms | 19.6 ms | -43.9% | +| ParseResult serialize (steady mean) | 18.7 ms | 26.8 ms | +43.0% | +| Payload bytes | 13.75 MB | 12.65 MB | -8.0% | + +Run-to-run spread (min-max of 3): neighborhood main 2,936-3,081 ms vs branch +1,047-1,124 ms (non-overlapping); full pipeline overlaps across sides +(798-958 ms), hence "noise band". + +Reading: at this size ingestion was never the bottleneck (~0.9 s either way) -- +Megatron-LM's pain is interactive, and the per-interaction backend paths are +16-63% faster. The serialize regression (finding 6) reproduces here; net +build+serialize is still -14%. The larger felt improvement should come from the +frontend changes this harness cannot measure (routing budget, one-rebuild +layouts, trigger policy): at 59k edges, symbol-view redraws previously entered +the unbounded O(E x N) routing regime measured in the frontend bench. diff --git a/benchmarks/gen_repo.py b/benchmarks/gen_repo.py new file mode 100644 index 0000000..bf92e16 --- /dev/null +++ b/benchmarks/gen_repo.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +"""Deterministically generate a synthetic Python (+ optional TypeScript) repo. + +The end-to-end harness (`crates/cc-core/examples/perf_harness.rs`) needs an +input repo that is (a) big enough for the pipeline costs to dominate noise and +(b) BYTE-IDENTICAL between two checkouts, so the same harness run on `main` and +on a feature branch is comparing the same work. Everything here is driven by a +single seeded `random.Random`, and the file order is fixed, so a given +(seed, files, depth, hub_fraction, ts_fraction) tuple always produces the same +tree. + +Shape knobs that matter for what the benchmarks measure: + +- `--files` / `--depth`: node and directory-chain counts, which drive scan cost, + the size of the child -> parent map, and how far edge endpoints have to be + lifted when containers are collapsed. +- `--hub-fraction`: the share of modules that define the SAME hub names + (`get`, `run`, `handle`, `new`, `__init__`). Those names are what push symbol + resolution into its ambiguous tiers, which is the path the early-bail change + in the perf work targets. At 0.0 every symbol is unique and resolution is + trivially cheap; at 0.6 a hub name has hundreds of global definitions. +- `--imports-per-file`: cross-module import + call density, i.e. edge count. + +Usage: + + python3 benchmarks/gen_repo.py --preset medium --out "$TMPDIR/cc-bench-medium" + python3 benchmarks/gen_repo.py --files 500 --depth 3 --hub-fraction 0.5 \ + --out /path/to/repo + +Never generates into the repository itself: `--out` is required (the presets +only pick sizes, not locations). +""" + +from __future__ import annotations + +import argparse +import random +import shutil +import sys +from dataclasses import dataclass +from pathlib import Path + +# Names deliberately shared by many modules. These are the ones that make a +# global symbol lookup return hundreds of candidates. +HUB_FUNCTIONS = ["run", "build", "handle", "process"] +HUB_METHODS = ["__init__", "get", "new", "update", "close"] + +PRESETS = { + "small": dict(files=200, depth=3, imports_per_file=3, hub_fraction=0.5), + "medium": dict(files=2000, depth=4, imports_per_file=4, hub_fraction=0.5), + "large": dict(files=10000, depth=5, imports_per_file=4, hub_fraction=0.5), +} + + +@dataclass(frozen=True) +class Module: + """One generated module: where it lives and whether it defines hub names.""" + + index: int + package: tuple[str, ...] + name: str + is_hub: bool + is_ts: bool + + @property + def rel_path(self) -> Path: + suffix = ".ts" if self.is_ts else ".py" + return Path(*self.package) / f"{self.name}{suffix}" + + @property + def dotted(self) -> str: + """Python import path (`pkg_0.pkg_3.mod_17`).""" + return ".".join([*self.package, self.name]) + + +def build_package_tree(rng: random.Random, depth: int, count: int) -> list[tuple[str, ...]]: + """Return `count` package paths, at most `depth` levels deep. + + Built breadth-first from a fixed branching factor so the directory tree is + bushy rather than a single chain: real repos have a handful of top-level + packages and progressively fewer deep ones. + """ + packages: list[tuple[str, ...]] = [()] + frontier: list[tuple[str, ...]] = [()] + branching = 4 + next_id = 0 + + while len(packages) < count and frontier: + parent = frontier.pop(0) + if len(parent) >= depth: + continue + for _ in range(branching): + if len(packages) >= count: + break + child = (*parent, f"pkg_{next_id}") + next_id += 1 + packages.append(child) + frontier.append(child) + + # Shuffle only the ORDER modules are assigned to packages, not the set, so + # the tree shape is stable while file placement still looks irregular. + rng.shuffle(packages) + return packages + + +def plan_modules( + rng: random.Random, files: int, depth: int, hub_fraction: float, ts_fraction: float +) -> list[Module]: + package_count = max(1, files // 6) + packages = build_package_tree(rng, depth, package_count) + + modules: list[Module] = [] + for i in range(files): + package = packages[i % len(packages)] + is_hub = rng.random() < hub_fraction + is_ts = rng.random() < ts_fraction + modules.append( + Module(index=i, package=package, name=f"mod_{i}", is_hub=is_hub, is_ts=is_ts) + ) + return modules + + +def pick_imports(rng: random.Random, modules: list[Module], me: Module, k: int) -> list[Module]: + """`k` distinct other modules for `me` to import, chosen deterministically.""" + if len(modules) <= 1 or k <= 0: + return [] + picked: list[Module] = [] + seen = {me.index} + for _ in range(k * 3): + if len(picked) >= k: + break + candidate = modules[rng.randrange(len(modules))] + if candidate.index in seen or candidate.is_ts != me.is_ts: + continue + seen.add(candidate.index) + picked.append(candidate) + return picked + + +def python_module_source(module: Module, imports: list[Module]) -> str: + i = module.index + lines: list[str] = [f'"""Generated module {module.dotted}."""', "", "import os", "import sys"] + + for dep in imports: + lines.append(f"from {dep.dotted} import Widget{dep.index}, make_{dep.index}") + lines.append("") + lines.append("") + + # Class with either hub method names or module-unique ones. + methods = HUB_METHODS if module.is_hub else [f"{m}_{i}" for m in HUB_METHODS] + lines.append(f"class Widget{i}:") + lines.append(f' """Widget defined by {module.dotted}."""') + lines.append("") + for method in methods: + if method == "__init__": + lines.append(" def __init__(self, name=None):") + lines.append(" self.name = name") + lines.append(" self.items = []") + elif method.startswith("__init__"): + # Unique-name variant of the constructor for non-hub modules. + lines.append(" def __init__(self, name=None):") + lines.append(" self.name = name") + lines.append(" self.items = []") + else: + lines.append(f" def {method}(self, value=None):") + lines.append(" if value is not None:") + lines.append(" self.items.append(value)") + lines.append(" return self.items") + lines.append("") + + # Free functions: hub-named for hub modules, unique otherwise. + functions = HUB_FUNCTIONS if module.is_hub else [f"{f}_{i}" for f in HUB_FUNCTIONS] + for func in functions: + lines.append(f"def {func}(source=None):") + lines.append(f" widget = Widget{i}(source)") + lines.append(" widget.items.append(source)") + lines.append(" return widget") + lines.append("") + + # A unique factory every importer can call, so imports produce call edges + # rather than dangling names. + lines.append(f"def make_{i}(source=None):") + lines.append(f" return Widget{i}(source)") + lines.append("") + + # Call sites into the imported modules: this is what makes the resolver do + # real work, and (for hub names) what drives it into the ambiguous tiers. + lines.append(f"def wire_{i}(payload):") + lines.append(" results = []") + for dep in imports: + lines.append(f" results.append(make_{dep.index}(payload))") + lines.append(f" results.append(Widget{dep.index}(payload))") + for func in HUB_FUNCTIONS: + lines.append(f" results.append({func}(payload))") + lines.append(" return results") + lines.append("") + + # Hub-name call sites. In a HUB module these resolve same-file (tier 1, the + # cheap path); everywhere else they hit the global tiers, where a name with + # hundreds of definitions is exactly the ambiguity the resolver has to bail + # out of. `--hub-fraction` therefore sets the ambiguous/cheap ratio. + lines.append(f"def dispatch_{i}(payload):") + lines.append(" out = []") + for func in HUB_FUNCTIONS: + lines.append(f" out.append({func}(payload))") + for dep in imports: + for method in ("get", "update", "close"): + lines.append(f" out.append(make_{dep.index}(payload).{method}(payload))") + lines.append(" return out") + lines.append("") + + return "\n".join(lines) + + +def typescript_module_source(module: Module, imports: list[Module]) -> str: + i = module.index + lines: list[str] = [f"// Generated module {module.dotted}.", ""] + + for dep in imports: + rel = relative_ts_import(module, dep) + lines.append(f'import {{ Widget{dep.index}, make{dep.index} }} from "{rel}";') + lines.append("") + + methods = HUB_METHODS if module.is_hub else [f"{m}_{i}" for m in HUB_METHODS] + lines.append(f"export interface Spec{i} {{") + lines.append(" name: string;") + lines.append(" items: string[];") + lines.append("}") + lines.append("") + lines.append(f"export class Widget{i} {{") + lines.append(" items: string[] = [];") + lines.append("") + for method in methods: + safe = method.replace("__init__", "init") + lines.append(f" {safe}(value?: string): string[] {{") + lines.append(" if (value) this.items.push(value);") + lines.append(" return this.items;") + lines.append(" }") + lines.append("") + lines.append("}") + lines.append("") + + functions = HUB_FUNCTIONS if module.is_hub else [f"{f}_{i}" for f in HUB_FUNCTIONS] + for func in functions: + lines.append(f"export function {func}(source: string): Widget{i} {{") + lines.append(f" const widget = new Widget{i}();") + lines.append(" widget.items.push(source);") + lines.append(" return widget;") + lines.append("}") + lines.append("") + + lines.append(f"export function make{i}(source: string): Widget{i} {{") + lines.append(f" const widget = new Widget{i}();") + lines.append(" widget.items.push(source);") + lines.append(" return widget;") + lines.append("}") + lines.append("") + + lines.append(f"export function wire{i}(payload: string): unknown[] {{") + lines.append(" const results: unknown[] = [];") + for dep in imports: + lines.append(f" results.push(make{dep.index}(payload));") + lines.append(f" results.push(new Widget{dep.index}());") + lines.append(" return results;") + lines.append("}") + lines.append("") + + return "\n".join(lines) + + +def relative_ts_import(module: Module, dep: Module) -> str: + """A `./`-prefixed relative specifier from `module` to `dep` (no extension).""" + from_dir = Path(*module.package) + to_path = Path(*dep.package) / dep.name + up = len(from_dir.parts) + rel = Path(*([".."] * up)) / to_path if up else to_path + text = rel.as_posix() + return text if text.startswith(".") else f"./{text}" + + +def generate( + out: Path, + files: int, + depth: int, + imports_per_file: int, + hub_fraction: float, + ts_fraction: float, + seed: int, + force: bool, +) -> dict[str, int]: + if out.exists(): + if not force: + raise SystemExit( + f"{out} already exists; pass --force to regenerate it from scratch" + ) + shutil.rmtree(out) + + rng = random.Random(seed) + modules = plan_modules(rng, files, depth, hub_fraction, ts_fraction) + + # A second generator for import edges, seeded off the same seed, so changing + # `--imports-per-file` does not perturb module placement. + edge_rng = random.Random(seed ^ 0x5EED) + + packages_written: set[tuple[str, ...]] = set() + hub_modules = 0 + ts_modules = 0 + + for module in modules: + target = out / module.rel_path + target.parent.mkdir(parents=True, exist_ok=True) + + # Every Python package level needs an __init__.py to be importable. + if not module.is_ts: + for level in range(len(module.package) + 1): + package = module.package[:level] + if package in packages_written: + continue + packages_written.add(package) + init = out / Path(*package) / "__init__.py" + init.parent.mkdir(parents=True, exist_ok=True) + init.write_text(f'"""Package {".".join(package) or "root"}."""\n') + + imports = pick_imports(edge_rng, modules, module, imports_per_file) + source = ( + typescript_module_source(module, imports) + if module.is_ts + else python_module_source(module, imports) + ) + target.write_text(source) + + hub_modules += int(module.is_hub) + ts_modules += int(module.is_ts) + + return { + "modules": len(modules), + "hub_modules": hub_modules, + "ts_modules": ts_modules, + "packages": len(packages_written), + } + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--out", required=True, type=Path, help="destination directory (must be outside the repo)") + parser.add_argument("--preset", choices=sorted(PRESETS), help="size preset: small/medium/large") + parser.add_argument("--files", type=int, help="number of modules to generate") + parser.add_argument("--depth", type=int, help="maximum package nesting depth") + parser.add_argument("--imports-per-file", type=int, help="cross-module imports per module") + parser.add_argument( + "--hub-fraction", + type=float, + help="share of modules defining the shared hub names (0.0-1.0)", + ) + parser.add_argument( + "--ts-fraction", + type=float, + default=0.0, + help="share of modules emitted as TypeScript instead of Python", + ) + parser.add_argument("--seed", type=int, default=20240611, help="RNG seed (default: 20240611)") + parser.add_argument("--force", action="store_true", help="delete --out first if it exists") + args = parser.parse_args(argv) + + settings = dict(PRESETS[args.preset]) if args.preset else dict(PRESETS["small"]) + for key in ("files", "depth", "imports_per_file", "hub_fraction"): + value = getattr(args, key) + if value is not None: + settings[key] = value + + stats = generate( + out=args.out, + files=int(settings["files"]), + depth=int(settings["depth"]), + imports_per_file=int(settings["imports_per_file"]), + hub_fraction=float(settings["hub_fraction"]), + ts_fraction=args.ts_fraction, + seed=args.seed, + force=args.force, + ) + + print( + f"generated {stats['modules']} modules " + f"({stats['hub_modules']} hub-named, {stats['ts_modules']} TypeScript) " + f"across {stats['packages']} packages into {args.out}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/benchmarks/run_all.sh b/benchmarks/run_all.sh new file mode 100644 index 0000000..02a1304 --- /dev/null +++ b/benchmarks/run_all.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# Run every benchmark layer once and drop the raw output in one directory. +# +# benchmarks/run_all.sh