codspeed.yml

# CodSpeed exec-harness targets (https://codspeed.io/docs).
#
# Same measurement philosophy as bench/check_regression.sh: each target is one
# whole-program run of the release interpreter under valgrind's CPU simulation,
# so the number is deterministic instruction/cycle counting, not wall-clock.
# What this adds over the in-repo Ir gate: a hosted history per commit,
# per-target differential flamegraphs on every PR, and a PR comment. The Ir
# gate (CI `bench` job) stays the MERGE gate — this is the profiler and the
# trend line, not a second gate.
#
# Tier is part of the target name: the interpreter JITs by default, so an
# unqualified target measures the JIT tier. The `vm-tier` targets pin
# EIGS_JIT_OFF=1 so an interpreter-only regression is visible on its own row
# (#1200: a probe that silently forced one tier covered only that tier).
# Measured 2026-09-20 under cachegrind, bench_dmg_shape: JIT on 1.20 G Ir,
# JIT off 2.22 G Ir — the JIT DOES engage under valgrind, both arms differ.
#
# The lane runs with cycle-estimation OFF (see the workflow): the headline
# is instructions, not the runner's cache model, because perf stat on the
# dev box refuted that model's 60% cache share (IPC 1.61, 1.6% LLC refs).
# Locally, match it: codspeed run -m simulation --cycle-estimation=false
#
# Run locally (after `codspeed auth login`):
#   make && codspeed run -m simulation --cycle-estimation=false   # every target
#   (CLI 5.3.1 has no per-target selector; the ids exist for the report only)
#
# Workloads: bench/*.eigs are the five Ir-gate workloads; the three tests/
# benches are the JIT's own shapes (CLAUDE.md "Benchmarks"), dmg_shape being
# the consumer stand-in for the DMG emulator's hot loop.
$schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json

benchmarks:
  # --- the five Ir-gate workloads (bench/) ---
  - name: "scalar_loop"
    id: scalar-loop
    exec: ./src/eigenscript bench/scalar_loop.eigs
  - name: "dict_ops"
    id: dict-ops
    exec: ./src/eigenscript bench/dict_ops.eigs
  - name: "string_build"
    id: string-build
    exec: ./src/eigenscript bench/string_build.eigs
  - name: "observed_loop"
    id: observed-loop
    exec: ./src/eigenscript bench/observed_loop.eigs
  - name: "unobserved_loop"
    id: unobserved-loop
    exec: ./src/eigenscript bench/unobserved_loop.eigs
  - name: "unobserved armed loop"
    id: unobserved-armed-loop
    exec: ./src/eigenscript bench/unobserved_armed_loop.eigs

  # --- the JIT's own shapes (tests/) ---
  - name: "bench_perf"
    id: bench-perf
    exec: ./src/eigenscript tests/bench_perf.eigs
  - name: "bench_dmg_shape"
    id: dmg-shape
    exec: ./src/eigenscript tests/bench_dmg_shape.eigs
  - name: "bench_idxset"
    id: idxset
    exec: ./src/eigenscript tests/bench_idxset.eigs

  # --- a REAL consumer: the DMG Game Boy emulator running Blargg's cpu_instrs ---
  # The in-repo dmg_shape bench flatters the JIT ~10x against the emulator it
  # names (measured 2026-09-21 under cachegrind: JIT removes 48% of the
  # instructions on bench_dmg_shape, 11.8% on DMG itself at 2 M cycles; the
  # fleet matrix on #1178 reads +5% wall). A per-PR verdict about the JIT
  # needs the consumer's shape, so the lane fetches DMG at the commit pinned
  # in .github/workflows/codspeed.yml into .codspeed-consumers/ (ignored).
  # Locally: run that workflow step by hand, or point the path at a checkout
  # -- and say which DMG commit you measured; the pin is the one in CI.
  - name: "dmg cpu_instrs 2M cycles"
    id: dmg-cpu-instrs
    exec: ./src/eigenscript .codspeed-consumers/DMG/dmg.eigs .codspeed-consumers/DMG/roms/cpu_instrs.gb --cycles 2000000
  - name: "dmg cpu_instrs 2M cycles (vm-tier)"
    id: dmg-cpu-instrs-vm
    exec: env EIGS_JIT_OFF=1 ./src/eigenscript .codspeed-consumers/DMG/dmg.eigs .codspeed-consumers/DMG/roms/cpu_instrs.gb --cycles 2000000

  # --- interpreter tier, JIT hard-disabled ---
  - name: "bench_dmg_shape (vm-tier)"
    id: dmg-shape-vm
    exec: env EIGS_JIT_OFF=1 ./src/eigenscript tests/bench_dmg_shape.eigs
  - name: "scalar_loop (vm-tier)"
    id: scalar-loop-vm
    exec: env EIGS_JIT_OFF=1 ./src/eigenscript bench/scalar_loop.eigs