From 068816aae943f3248e6fe6e224b88d6bad28fa2a Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 11:25:10 -0500 Subject: [PATCH 01/15] benchmark.sh: add --branch to build the compare binary from a git branch Co-authored-by: Cursor --- benchmark.sh | 100 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 6 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index e4f2a605..2f002beb 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -12,6 +12,7 @@ # ./benchmark.sh --build # ./benchmark.sh -- -n 8 -r # ./benchmark.sh --build --compare /path/to/other/dtest +# ./benchmark.sh --branch develop -- -n 8 # ./benchmark.sh --compare /path/to/other/dtest --epsilon 1 # ./benchmark.sh --repeats 5 -- -n 4 # REPEATS=3 ./benchmark.sh @@ -19,6 +20,7 @@ # Environment: # BRANCH Path to branch dtest (default: bazel-bin in this repo) # COMPARE Optional second dtest binary for comparison +# (or use --branch NAME to build the compare binary from a git branch) # HANDS_DIR Directory containing list*.txt files (default: ./hands) # REPEATS Runs per combination per binary (default: 1) # MAX_DEALS Include list10^n.txt files with 10^n <= N (default: 100) @@ -38,8 +40,31 @@ DETAILS="${DETAILS:-0}" EPSILON="${EPSILON:-0.5}" BUILD=0 REVERSE=0 +COMPARE_BRANCH="" DTEST_EXTRA=() +# Cleanup state (set later). The EXIT trap restores the original git branch if +# --branch switched away, and removes temp files. +RESULTS="" +ORIG_BRANCH="" +COMPARE_TMP="" + +cleanup() { + if [[ -n "$ORIG_BRANCH" ]]; then + local cur + cur="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ + || git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)" + if [[ -n "$cur" && "$cur" != "$ORIG_BRANCH" ]]; then + echo "Restoring git branch '$ORIG_BRANCH'..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" >/dev/null 2>&1 || true + fi + fi + [[ -n "$COMPARE_TMP" ]] && rm -f "$COMPARE_TMP" + [[ -n "$RESULTS" ]] && rm -f "$RESULTS" + return 0 +} +trap cleanup EXIT + SOLVERS=(solve calc) usage() { @@ -55,7 +80,9 @@ Options: --max-deals N Include list10^n.txt files with 10^n <= N (default: 100; env: MAX_DEALS) (alias: --max_deals) --build Build branch dtest only (bazel build //library/tests:dtest) - --branch PATH Branch dtest binary (default: $BRANCH) + --branch NAME Git branch to compare against: check it out, build dtest, save the + binary as the compare binary, restore the current branch, rebuild, + then run. Mutually exclusive with --compare. Requires a clean tree. --compare PATH Optional second dtest binary (summary; transient progress on tty) --details With --compare, keep per-run timing rows in final output --epsilon PCT With --compare, treat timings within PCT% as equal (default: 0.5; env: EPSILON) @@ -71,6 +98,8 @@ Examples: ./benchmark.sh --build ./benchmark.sh -- -n 8 ./benchmark.sh --repeats 3 -- -n 4 -r + ./benchmark.sh --branch develop + ./benchmark.sh --branch develop --repeats 3 -- -n 8 ./benchmark.sh --compare /path/to/dtest ./benchmark.sh --compare /path/to/dtest --details ./benchmark.sh --compare /path/to/dtest --epsilon 1 @@ -93,7 +122,7 @@ while [[ $# -gt 0 ]]; do ;; --branch) shift - BRANCH="${1:?missing value for --branch}" + COMPARE_BRANCH="${1:?missing value for --branch}" shift ;; --compare) @@ -151,11 +180,66 @@ if ! [[ "$EPSILON" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then exit 1 fi -if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" ]]; then - echo "error: --reverse requires --compare" >&2 +if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && -z "$COMPARE_BRANCH" ]]; then + echo "error: --reverse requires --compare or --branch" >&2 exit 1 fi +if [[ -n "$COMPARE_BRANCH" && -n "${COMPARE:-}" ]]; then + echo "error: --branch and --compare are mutually exclusive" >&2 + exit 1 +fi + +build_compare_from_branch() { + local dtest_rel="bazel-bin/library/tests/dtest" + + if ! git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "error: --branch requires a git work tree at $ROOT" >&2 + exit 1 + fi + + if ! git -C "$ROOT" rev-parse --verify --quiet "$COMPARE_BRANCH" >/dev/null; then + echo "error: --branch: unknown git ref '$COMPARE_BRANCH'" >&2 + exit 1 + fi + + if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then + echo "error: tracked changes present; commit or stash before using --branch" >&2 + exit 1 + fi + + ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ + || git -C "$ROOT" rev-parse HEAD)" + COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" + + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $COMPARE_BRANCH" >&2 + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + echo "DRY_RUN: cp -L $ROOT/$dtest_rel $COMPARE_TMP" >&2 + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + COMPARE="$COMPARE_TMP" + return 0 + fi + + echo "Building compare binary from '$COMPARE_BRANCH' (current: '$ORIG_BRANCH')..." >&2 + git -C "$ROOT" checkout "$COMPARE_BRANCH" + (cd "$ROOT" && bazel build //library/tests:dtest) + cp -L "$ROOT/$dtest_rel" "$COMPARE_TMP" + chmod +x "$COMPARE_TMP" + + echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" + (cd "$ROOT" && bazel build //library/tests:dtest) + + COMPARE="$COMPARE_TMP" +} + +if [[ -n "$COMPARE_BRANCH" ]]; then + build_compare_from_branch + BUILD=0 # build already done as part of the branch workflow +fi + select_hand_files() { is_power_of_10() { local n="$1" @@ -241,7 +325,7 @@ if git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then fi RESULTS="$(mktemp "${TMPDIR:-/tmp}/dds-benchmark.XXXXXX")" -trap 'rm -f "$RESULTS"' EXIT +# Removal handled by the cleanup() EXIT trap installed near the top. parse_dtest_output() { awk ' @@ -334,7 +418,11 @@ echo "DDS dtest benchmark" echo "===================" printf "%-12s %s\n" "branch:" "$BRANCH" if [[ -n "${COMPARE:-}" ]]; then - printf "%-12s %s\n" "compare:" "$COMPARE" + if [[ -n "$COMPARE_BRANCH" ]]; then + printf "%-12s %s\n" "compare:" "branch '$COMPARE_BRANCH' ($COMPARE)" + else + printf "%-12s %s\n" "compare:" "$COMPARE" + fi if [[ "$DETAILS" == "1" ]]; then printf "%-12s %s\n" "details:" "on" elif [[ -t 1 ]]; then From 91e2b59f816e1dd6c02b0d699e15d1d6ee65ba74 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 11:30:19 -0500 Subject: [PATCH 02/15] benchmark.sh: show branch names in summary header columns Co-authored-by: Cursor --- benchmark.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/benchmark.sh b/benchmark.sh index 2f002beb..8865606d 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -502,10 +502,22 @@ clear_transient_progress if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then echo + # Column headers default to the generic labels, but show the actual branch + # names when known: the current git branch for the branch binary, and the + # --branch name for the compare binary. Truncated to the 12-char column. + cmp_label="compare_avg" + if [[ -n "$COMPARE_BRANCH" ]]; then + cmp_label="${COMPARE_BRANCH:0:12}" + fi + br_label="branch_avg" + if [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then + br_label="${git_branch:0:12}" + fi + echo "Summary (branch vs compare, avg user ms)" echo "==============================================================================" printf "%-6s %-13s %12s %12s %10s %-15s\n" \ - "solver" "file" "compare_avg" "branch_avg" "cmp/branch" "note" + "solver" "file" "$cmp_label" "$br_label" "cmp/branch" "note" printf "%-6s %-13s %12s %12s %10s %-15s\n" \ "------" "-------------" "------------" "------------" "----------" "---------------" From 81af0ca431cbdbebc693707b25dd359ef70e5bc4 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 11:44:51 -0500 Subject: [PATCH 03/15] benchmark.sh: allow --branch twice to compare two branches Co-authored-by: Cursor --- benchmark.sh | 139 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 96 insertions(+), 43 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 8865606d..b2f514c4 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -40,7 +40,7 @@ DETAILS="${DETAILS:-0}" EPSILON="${EPSILON:-0.5}" BUILD=0 REVERSE=0 -COMPARE_BRANCH="" +BRANCH_NAMES=() DTEST_EXTRA=() # Cleanup state (set later). The EXIT trap restores the original git branch if @@ -48,6 +48,7 @@ DTEST_EXTRA=() RESULTS="" ORIG_BRANCH="" COMPARE_TMP="" +BRANCH_TMP="" cleanup() { if [[ -n "$ORIG_BRANCH" ]]; then @@ -60,6 +61,7 @@ cleanup() { fi fi [[ -n "$COMPARE_TMP" ]] && rm -f "$COMPARE_TMP" + [[ -n "$BRANCH_TMP" ]] && rm -f "$BRANCH_TMP" [[ -n "$RESULTS" ]] && rm -f "$RESULTS" return 0 } @@ -80,9 +82,11 @@ Options: --max-deals N Include list10^n.txt files with 10^n <= N (default: 100; env: MAX_DEALS) (alias: --max_deals) --build Build branch dtest only (bazel build //library/tests:dtest) - --branch NAME Git branch to compare against: check it out, build dtest, save the - binary as the compare binary, restore the current branch, rebuild, - then run. Mutually exclusive with --compare. Requires a clean tree. + --branch NAME Git branch to build and compare. Once: compare the current branch + against NAME. Twice (--branch A --branch B): compare A vs B and + ignore the current branch. Each branch is checked out, dtest is + built and its binary saved; the original branch is then restored. + Mutually exclusive with --compare. Requires a clean tree. --compare PATH Optional second dtest binary (summary; transient progress on tty) --details With --compare, keep per-run timing rows in final output --epsilon PCT With --compare, treat timings within PCT% as equal (default: 0.5; env: EPSILON) @@ -99,6 +103,7 @@ Examples: ./benchmark.sh -- -n 8 ./benchmark.sh --repeats 3 -- -n 4 -r ./benchmark.sh --branch develop + ./benchmark.sh --branch develop --branch opus-two-percent ./benchmark.sh --branch develop --repeats 3 -- -n 8 ./benchmark.sh --compare /path/to/dtest ./benchmark.sh --compare /path/to/dtest --details @@ -122,7 +127,7 @@ while [[ $# -gt 0 ]]; do ;; --branch) shift - COMPARE_BRANCH="${1:?missing value for --branch}" + BRANCH_NAMES+=("${1:?missing value for --branch}") shift ;; --compare) @@ -180,29 +185,54 @@ if ! [[ "$EPSILON" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then exit 1 fi -if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && -z "$COMPARE_BRANCH" ]]; then +num_branches=${#BRANCH_NAMES[@]} + +if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && "$num_branches" -eq 0 ]]; then echo "error: --reverse requires --compare or --branch" >&2 exit 1 fi -if [[ -n "$COMPARE_BRANCH" && -n "${COMPARE:-}" ]]; then +if [[ "$num_branches" -gt 0 && -n "${COMPARE:-}" ]]; then echo "error: --branch and --compare are mutually exclusive" >&2 exit 1 fi -build_compare_from_branch() { +if [[ "$num_branches" -gt 2 ]]; then + echo "error: --branch may be given at most twice (got $num_branches)" >&2 + exit 1 +fi + +# Check out $1, build dtest, and copy the binary to $2. +build_branch_binary() { + local name="$1" dest="$2" local dtest_rel="bazel-bin/library/tests/dtest" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $name" >&2 + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + echo "DRY_RUN: cp -L $ROOT/$dtest_rel $dest" >&2 + return 0 + fi + echo "Building dtest from '$name'..." >&2 + git -C "$ROOT" checkout "$name" + (cd "$ROOT" && bazel build //library/tests:dtest) + cp -L "$ROOT/$dtest_rel" "$dest" + chmod +x "$dest" +} +# With one --branch, compare the current branch against the named branch. +# With two, compare the two named branches and ignore the current branch. +setup_branches() { if ! git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then echo "error: --branch requires a git work tree at $ROOT" >&2 exit 1 fi - - if ! git -C "$ROOT" rev-parse --verify --quiet "$COMPARE_BRANCH" >/dev/null; then - echo "error: --branch: unknown git ref '$COMPARE_BRANCH'" >&2 - exit 1 - fi - + local name + for name in "${BRANCH_NAMES[@]}"; do + if ! git -C "$ROOT" rev-parse --verify --quiet "$name" >/dev/null; then + echo "error: --branch: unknown git ref '$name'" >&2 + exit 1 + fi + done if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then echo "error: tracked changes present; commit or stash before using --branch" >&2 exit 1 @@ -210,33 +240,39 @@ build_compare_from_branch() { ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ || git -C "$ROOT" rev-parse HEAD)" - COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" - if [[ "$DRY_RUN" == "1" ]]; then - echo "DRY_RUN: git -C $ROOT checkout $COMPARE_BRANCH" >&2 - echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 - echo "DRY_RUN: cp -L $ROOT/$dtest_rel $COMPARE_TMP" >&2 - echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 - echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + if [[ "$num_branches" -eq 1 ]]; then + COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" + build_branch_binary "${BRANCH_NAMES[0]}" "$COMPARE_TMP" + # Restore the current branch and rebuild it as the branch binary. + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + else + echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" + (cd "$ROOT" && bazel build //library/tests:dtest) + fi + COMPARE="$COMPARE_TMP" + else + # Two branches: build both, ignore the current branch's binary. + BRANCH_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-branch.XXXXXX")" + COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" + build_branch_binary "${BRANCH_NAMES[0]}" "$BRANCH_TMP" + build_branch_binary "${BRANCH_NAMES[1]}" "$COMPARE_TMP" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + else + echo "Restoring '$ORIG_BRANCH'..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" + fi + BRANCH="$BRANCH_TMP" COMPARE="$COMPARE_TMP" - return 0 fi - - echo "Building compare binary from '$COMPARE_BRANCH' (current: '$ORIG_BRANCH')..." >&2 - git -C "$ROOT" checkout "$COMPARE_BRANCH" - (cd "$ROOT" && bazel build //library/tests:dtest) - cp -L "$ROOT/$dtest_rel" "$COMPARE_TMP" - chmod +x "$COMPARE_TMP" - - echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 - git -C "$ROOT" checkout "$ORIG_BRANCH" - (cd "$ROOT" && bazel build //library/tests:dtest) - - COMPARE="$COMPARE_TMP" } -if [[ -n "$COMPARE_BRANCH" ]]; then - build_compare_from_branch +if [[ "$num_branches" -gt 0 ]]; then + setup_branches BUILD=0 # build already done as part of the branch workflow fi @@ -416,10 +452,25 @@ clear_transient_progress() { echo "DDS dtest benchmark" echo "===================" -printf "%-12s %s\n" "branch:" "$BRANCH" +# Branch names backing each binary, when --branch was used. With one --branch +# the branch binary is the current checkout; with two it is the first name. +branch_branch_name="" +compare_branch_name="" +if [[ "$num_branches" -eq 1 ]]; then + compare_branch_name="${BRANCH_NAMES[0]}" +elif [[ "$num_branches" -eq 2 ]]; then + branch_branch_name="${BRANCH_NAMES[0]}" + compare_branch_name="${BRANCH_NAMES[1]}" +fi + +if [[ -n "$branch_branch_name" ]]; then + printf "%-12s %s\n" "branch:" "branch '$branch_branch_name' ($BRANCH)" +else + printf "%-12s %s\n" "branch:" "$BRANCH" +fi if [[ -n "${COMPARE:-}" ]]; then - if [[ -n "$COMPARE_BRANCH" ]]; then - printf "%-12s %s\n" "compare:" "branch '$COMPARE_BRANCH' ($COMPARE)" + if [[ -n "$compare_branch_name" ]]; then + printf "%-12s %s\n" "compare:" "branch '$compare_branch_name' ($COMPARE)" else printf "%-12s %s\n" "compare:" "$COMPARE" fi @@ -506,15 +557,17 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then # names when known: the current git branch for the branch binary, and the # --branch name for the compare binary. Truncated to the 12-char column. cmp_label="compare_avg" - if [[ -n "$COMPARE_BRANCH" ]]; then - cmp_label="${COMPARE_BRANCH:0:12}" + if [[ -n "$compare_branch_name" ]]; then + cmp_label="${compare_branch_name:0:12}" fi br_label="branch_avg" - if [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then + if [[ -n "$branch_branch_name" ]]; then + br_label="${branch_branch_name:0:12}" + elif [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then br_label="${git_branch:0:12}" fi - echo "Summary (branch vs compare, avg user ms)" + echo "Summary (avg user ms)" echo "==============================================================================" printf "%-6s %-13s %12s %12s %10s %-15s\n" \ "solver" "file" "$cmp_label" "$br_label" "cmp/branch" "note" From 090f2122c2c8dc936111c4346f972b8287ee59c1 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 11:50:25 -0500 Subject: [PATCH 04/15] benchmark.sh: use branch names in summary note column Co-authored-by: Cursor --- benchmark.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index b2f514c4..6651462b 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -567,6 +567,19 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then br_label="${git_branch:0:12}" fi + # Names used in the "note" column (" faster"); fall back to the generic + # "branch"/"compare" when no branch name is known. + note_branch="branch" + if [[ -n "$branch_branch_name" ]]; then + note_branch="$branch_branch_name" + elif [[ -n "$compare_branch_name" && -n "$git_branch" && "$git_branch" != "unknown" ]]; then + note_branch="$git_branch" + fi + note_compare="compare" + if [[ -n "$compare_branch_name" ]]; then + note_compare="$compare_branch_name" + fi + echo "Summary (avg user ms)" echo "==============================================================================" printf "%-6s %-13s %12s %12s %10s %-15s\n" \ @@ -574,7 +587,8 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then printf "%-6s %-13s %12s %12s %10s %-15s\n" \ "------" "-------------" "------------" "------------" "----------" "---------------" - awk -F'\t' -v files="${FILES[*]}" -v epsilon_pct="$EPSILON" ' + awk -F'\t' -v files="${FILES[*]}" -v epsilon_pct="$EPSILON" \ + -v note_branch="$note_branch" -v note_compare="$note_compare" ' function within_epsilon(a, b, eps, hi, lo) { eps = epsilon_pct / 100 if (a > b) { hi = a; lo = b } else { hi = b; lo = a } @@ -607,9 +621,9 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then if (within_epsilon(u1, u2)) { note = "equal" } else if (cmp_branch >= 1) { - note = "branch faster" + note = note_branch " faster" } else { - note = "compare faster" + note = note_compare " faster" } sp = sprintf("%9.2fx", cmp_branch) printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", From ad65264777109ac9c7f506505d381ca07ad50749 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 11:55:45 -0500 Subject: [PATCH 05/15] benchmark.sh: hide transient per-run rows when repeats > 1 Co-authored-by: Cursor --- benchmark.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmark.sh b/benchmark.sh index 6651462b..162ab534 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -500,7 +500,11 @@ echo show_run_lines=1 TRANSIENT_PROGRESS=0 -if [[ -n "${COMPARE:-}" && "$DETAILS" != "1" ]]; then +# Per-run rows are detail: hide them from the final output (transient on a tty, +# suppressed otherwise) when comparing or when repeating, unless --details asks +# to keep them. With repeats > 1 the per-run rows are intermediate samples, so +# they are treated as transient too. +if [[ "$DETAILS" != "1" ]] && { [[ -n "${COMPARE:-}" ]] || (( REPEATS > 1 )); }; then show_run_lines=0 if [[ -t 1 ]]; then TRANSIENT_PROGRESS=1 From 3926cf6a0565278ccf737e6f24d083123bb548f5 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 14:23:13 -0500 Subject: [PATCH 06/15] benchmark.sh: add per-branch elapsed totals row with ratio and note Co-authored-by: Cursor --- benchmark.sh | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 162ab534..09fe292f 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -398,8 +398,11 @@ run_dtest() { return 0 fi + # Wrap with `time -p` so we can capture wall-clock elapsed for this run. Its + # "real/user/sys" lines go to the merged output but do not collide with the + # dtest lines parse_dtest_output looks for. local out - if ! out="$("${cmd[@]}" 2>&1)"; then + if ! out="$(/usr/bin/time -p "${cmd[@]}" 2>&1)"; then echo "error: dtest failed: ${cmd[*]}" >&2 echo "$out" >&2 exit 1 @@ -411,7 +414,10 @@ run_dtest() { if [[ "$parsed_user" == "NA" || "$parsed_sys" == "NA" ]]; then echo "warning: incomplete dtest timing output: ${cmd[*]}" >&2 fi - echo "$parsed" + local wall + wall="$(awk '/^real[[:space:]]/ { print $2; exit }' <<<"$out")" + [[ -z "$wall" ]] && wall="NA" + echo "$parsed $wall" } progress_lines=0 @@ -539,14 +545,14 @@ for solver in "${SOLVERS[@]}"; do continue fi - read -r user sys avg ratio < <(run_dtest "$bin" "$solver" "$hands") + read -r user sys avg ratio wall < <(run_dtest "$bin" "$solver" "$hands") if [[ "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ]]; then print_run_row "$solver" "$file" "$ver" "$user" "$sys" "$avg" "$ratio" "$run_label" fi - printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ - "$solver" "$file" "$ver" "$rep" "$user" "$sys" "$avg" "$ratio" \ + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ + "$solver" "$file" "$ver" "$rep" "$user" "$sys" "$avg" "$ratio" "$wall" \ >>"$RESULTS" done done @@ -603,9 +609,11 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then if ($3 == "compare") { s2[base] += $7 c2[base]++ + if ($9 != "NA") tw2 += $9 # total wall-clock elapsed, compare } else if ($3 == "branch") { s1[base] += $7 c1[base]++ + if ($9 != "NA") tw1 += $9 # total wall-clock elapsed, branch } } END { @@ -634,6 +642,22 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then solvers[si], filearr[fi], u2, u1, sp, note } } + + # Total elapsed (wall-clock seconds) summed per binary across all runs. + printf "%-6s %-13s %12s %12s %10s %-15s\n", + "------", "-------------", "------------", "------------", "----------", "---------------" + if (tw2 > 0 && tw1 > 0) { + tnote = "" + if (within_epsilon(tw1, tw2)) tnote = "equal" + else if (tw2 / tw1 >= 1) tnote = note_branch " faster" + else tnote = note_compare " faster" + tsp = sprintf("%9.2fx", tw2 / tw1) + printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", + "TOTAL", "elapsed (s)", tw2, tw1, tsp, tnote + } else { + printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", + "TOTAL", "elapsed (s)", tw2, tw1, "", "" + } } ' "$RESULTS" fi From c54f53105fb8022afbf7bedb577889c2ca2ef9b5 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 15:08:41 -0500 Subject: [PATCH 07/15] benchmark.sh: allow --branch with --compare to set branch binary Combining --branch NAME with --compare PATH now builds NAME as the branch binary and compares it against PATH, ignoring the current branch dtest. Co-authored-by: Cursor --- benchmark.sh | 49 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 09fe292f..dfa8bcc7 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -41,6 +41,7 @@ EPSILON="${EPSILON:-0.5}" BUILD=0 REVERSE=0 BRANCH_NAMES=() +COMPARE_GIVEN=0 DTEST_EXTRA=() # Cleanup state (set later). The EXIT trap restores the original git branch if @@ -84,10 +85,12 @@ Options: --build Build branch dtest only (bazel build //library/tests:dtest) --branch NAME Git branch to build and compare. Once: compare the current branch against NAME. Twice (--branch A --branch B): compare A vs B and - ignore the current branch. Each branch is checked out, dtest is - built and its binary saved; the original branch is then restored. - Mutually exclusive with --compare. Requires a clean tree. - --compare PATH Optional second dtest binary (summary; transient progress on tty) + ignore the current branch. With --compare PATH: build NAME as the + branch binary and compare it against PATH, ignoring the current + branch. Each branch is checked out, dtest is built and its binary + saved; the original branch is then restored. Requires a clean tree. + --compare PATH Second dtest binary (summary; transient progress on tty). May be + combined with a single --branch NAME (NAME backs the branch binary). --details With --compare, keep per-run timing rows in final output --epsilon PCT With --compare, treat timings within PCT% as equal (default: 0.5; env: EPSILON) --reverse With --compare, run compare before branch each repeat (default: branch first) @@ -104,6 +107,7 @@ Examples: ./benchmark.sh --repeats 3 -- -n 4 -r ./benchmark.sh --branch develop ./benchmark.sh --branch develop --branch opus-two-percent + ./benchmark.sh --branch opus-two-percent --compare /path/to/dtest ./benchmark.sh --branch develop --repeats 3 -- -n 8 ./benchmark.sh --compare /path/to/dtest ./benchmark.sh --compare /path/to/dtest --details @@ -133,6 +137,7 @@ while [[ $# -gt 0 ]]; do --compare) shift COMPARE="${1:?missing value for --compare}" + COMPARE_GIVEN=1 shift ;; --max-deals|--max_deals|-max-deals|-max_deals) @@ -185,6 +190,11 @@ if ! [[ "$EPSILON" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then exit 1 fi +# A compare binary supplied via the COMPARE env var behaves like --compare PATH. +if [[ "$COMPARE_GIVEN" == "0" && -n "${COMPARE:-}" ]]; then + COMPARE_GIVEN=1 +fi + num_branches=${#BRANCH_NAMES[@]} if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && "$num_branches" -eq 0 ]]; then @@ -192,8 +202,8 @@ if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && "$num_branches" -eq 0 ]]; then exit 1 fi -if [[ "$num_branches" -gt 0 && -n "${COMPARE:-}" ]]; then - echo "error: --branch and --compare are mutually exclusive" >&2 +if [[ "$num_branches" -gt 0 && "$COMPARE_GIVEN" == "1" && "$num_branches" -ne 1 ]]; then + echo "error: --compare accepts exactly one --branch (got $num_branches)" >&2 exit 1 fi @@ -219,8 +229,12 @@ build_branch_binary() { chmod +x "$dest" } -# With one --branch, compare the current branch against the named branch. -# With two, compare the two named branches and ignore the current branch. +# Branch-mode binary selection: +# --branch NAME : branch = current checkout, compare = build(NAME) +# --branch NAME --compare PATH : branch = build(NAME), compare = PATH +# (the current branch's dtest is ignored) +# --branch A --branch B : branch = build(A), compare = build(B) +# (the current branch's dtest is ignored) setup_branches() { if ! git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then echo "error: --branch requires a git work tree at $ROOT" >&2 @@ -241,7 +255,19 @@ setup_branches() { ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ || git -C "$ROOT" rev-parse HEAD)" - if [[ "$num_branches" -eq 1 ]]; then + if [[ "$COMPARE_GIVEN" == "1" ]]; then + # --branch NAME --compare PATH: build NAME as the branch binary and keep the + # user-supplied compare path. The current branch's dtest is not used. + BRANCH_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-branch.XXXXXX")" + build_branch_binary "${BRANCH_NAMES[0]}" "$BRANCH_TMP" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + else + echo "Restoring '$ORIG_BRANCH'..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" + fi + BRANCH="$BRANCH_TMP" + elif [[ "$num_branches" -eq 1 ]]; then COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" build_branch_binary "${BRANCH_NAMES[0]}" "$COMPARE_TMP" # Restore the current branch and rebuild it as the branch binary. @@ -462,7 +488,10 @@ echo "===================" # the branch binary is the current checkout; with two it is the first name. branch_branch_name="" compare_branch_name="" -if [[ "$num_branches" -eq 1 ]]; then +if [[ "$num_branches" -eq 1 && "$COMPARE_GIVEN" == "1" ]]; then + # --branch NAME --compare PATH: NAME backs the branch binary; compare is a path. + branch_branch_name="${BRANCH_NAMES[0]}" +elif [[ "$num_branches" -eq 1 ]]; then compare_branch_name="${BRANCH_NAMES[0]}" elif [[ "$num_branches" -eq 2 ]]; then branch_branch_name="${BRANCH_NAMES[0]}" From 4d8ed6bc3604b4c77b768726171e74350cb29857 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 16:26:24 -0500 Subject: [PATCH 08/15] Reject duplicate --repeats in benchmark.sh A second --repeats silently overwrote the first; error out instead. Co-authored-by: Cursor --- benchmark.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/benchmark.sh b/benchmark.sh index dfa8bcc7..c3a942d9 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -42,6 +42,7 @@ BUILD=0 REVERSE=0 BRANCH_NAMES=() COMPARE_GIVEN=0 +REPEATS_GIVEN=0 DTEST_EXTRA=() # Cleanup state (set later). The EXIT trap restores the original git branch if @@ -125,6 +126,11 @@ while [[ $# -gt 0 ]]; do exit 0 ;; --repeats) + if (( REPEATS_GIVEN )); then + echo "error: --repeats may be given only once" >&2 + exit 1 + fi + REPEATS_GIVEN=1 shift REPEATS="${1:?missing value for --repeats}" shift From 7b926b15137035b93315ac15f6f5d4c96c48de8d Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 16:32:20 -0500 Subject: [PATCH 09/15] Always show benchmark summary; gate detail rows on --details The summary was only printed in --compare mode, so a single binary with --repeats > 1 cleared its transient per-run rows and showed nothing. Add a single-binary summary (avg user ms per solver/file plus total elapsed) for the non-compare case so a summary is always shown. Also show the per-run detail table only with --details. Without it the rows are transient progress on a tty and suppressed otherwise. Co-authored-by: Cursor --- benchmark.sh | 57 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index c3a942d9..a43af64d 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -539,17 +539,16 @@ if ((${#DTEST_EXTRA[@]} > 0)); then fi echo -show_run_lines=1 +show_run_lines=0 TRANSIENT_PROGRESS=0 -# Per-run rows are detail: hide them from the final output (transient on a tty, -# suppressed otherwise) when comparing or when repeating, unless --details asks -# to keep them. With repeats > 1 the per-run rows are intermediate samples, so -# they are treated as transient too. -if [[ "$DETAILS" != "1" ]] && { [[ -n "${COMPARE:-}" ]] || (( REPEATS > 1 )); }; then - show_run_lines=0 - if [[ -t 1 ]]; then - TRANSIENT_PROGRESS=1 - fi +# Per-run rows are detail: keep them in the final output only with --details. +# Otherwise show them transiently as progress on a tty (cleared before the +# summary) and suppress them entirely when output is not a tty. The summary is +# always shown regardless. +if [[ "$DETAILS" == "1" ]]; then + show_run_lines=1 +elif [[ -t 1 ]]; then + TRANSIENT_PROGRESS=1 fi if [[ "$DRY_RUN" != "1" && ( "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ) ]]; then @@ -597,7 +596,7 @@ done clear_transient_progress if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then - echo + echo # compare summary (two binaries) # Column headers default to the generic labels, but show the actual branch # names when known: the current git branch for the branch binary, and the # --branch name for the compare binary. Truncated to the 12-char column. @@ -695,6 +694,42 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then } } ' "$RESULTS" +elif [[ "$DRY_RUN" != "1" ]]; then + echo # single-binary summary (no --compare) + br_label="branch_avg" + if [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then + br_label="${git_branch:0:12}" + fi + + echo "Summary (avg user ms)" + echo "============================================================" + printf "%-6s %-13s %12s %6s\n" "solver" "file" "$br_label" "runs" + printf "%-6s %-13s %12s %6s\n" \ + "------" "-------------" "------------" "------" + + awk -F'\t' -v files="${FILES[*]}" ' + $3 == "branch" { + base = $1 SUBSEP $2 + s[base] += $7 + c[base]++ + if ($9 != "NA") tw += $9 # total wall-clock elapsed + } + END { + split("solve calc", solvers, " ") + nfiles = split(files, filearr, " ") + for (si = 1; si <= 2; si++) { + for (fi = 1; fi <= nfiles; fi++) { + base = solvers[si] SUBSEP filearr[fi] + if (!(base in c)) continue + printf "%-6s %-13s %12.2f %6d\n", + solvers[si], filearr[fi], s[base] / c[base], c[base] + } + } + printf "%-6s %-13s %12s %6s\n", + "------", "-------------", "------------", "------" + printf "%-6s %-13s %12.2f %6s\n", "TOTAL", "elapsed (s)", tw, "" + } + ' "$RESULTS" fi echo From 63aa81039fc9a83e86be493414edebe0cc68caaa Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 16:47:08 -0500 Subject: [PATCH 10/15] Suppress per-run detail rows entirely without --details The detail rows were shown as transient progress on a tty and then erased with cursor-up escapes. With many repeats the rows scroll off-screen, so the erase cannot reach them and they persist. Show per-run rows only with --details and drop the transient progress table; the summary is always shown. Co-authored-by: Cursor --- benchmark.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index a43af64d..c0bae34a 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -541,14 +541,11 @@ echo show_run_lines=0 TRANSIENT_PROGRESS=0 -# Per-run rows are detail: keep them in the final output only with --details. -# Otherwise show them transiently as progress on a tty (cleared before the -# summary) and suppress them entirely when output is not a tty. The summary is -# always shown regardless. +# Per-run rows are detail: show them only with --details. Without it they are +# suppressed entirely (no transient progress table, which could otherwise scroll +# off-screen and leave residue). The summary is always shown regardless. if [[ "$DETAILS" == "1" ]]; then show_run_lines=1 -elif [[ -t 1 ]]; then - TRANSIENT_PROGRESS=1 fi if [[ "$DRY_RUN" != "1" && ( "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ) ]]; then From 79e8dec07779259cf5582adbe94df96a34bbc2be Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 17:17:13 -0500 Subject: [PATCH 11/15] Hide build output unless --details git checkout + bazel output is build noise. Capture it to a log and surface it only on failure (or with --details); the short "Building..." labels remain as progress markers. ANSI save/restore (DECSC/DECRC) cannot erase this output reliably: bazel drives its own cursor save/restore for its progress display, clobbering the saved position, so a later restore+clear erases nothing. Capturing to a log is robust and works whether or not stderr is a tty. Co-authored-by: Cursor --- benchmark.sh | 59 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index c0bae34a..6a7e0eb4 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -2,10 +2,11 @@ # Benchmark dtest performance on one or two binaries. # # Runs all combinations of solver (solve, calc) and hand file -# (list100/1000/…/1), largest files first. With --compare, prints summary only -# unless --details; without --compare, prints per-run rows. With --compare and a -# tty, per-run rows appear during the run then are cleared before the summary. -# Does not pass dtest options unless given after "--" (see below). +# (list100/1000/…/1), largest files first. Always prints a summary. Per-run +# timing rows and build (git/bazel) output are shown only with --details; +# otherwise build output is captured to a log (surfaced only on failure) and +# per-run rows are suppressed. Does not pass dtest options unless given after +# "--" (see below). # # Usage: # ./benchmark.sh @@ -25,7 +26,7 @@ # REPEATS Runs per combination per binary (default: 1) # MAX_DEALS Include list10^n.txt files with 10^n <= N (default: 100) # DRY_RUN If 1, print commands only -# DETAILS If 1 with --compare, keep per-run rows in output (default: transient on tty) +# DETAILS If 1, keep per-run rows and build output (default: 0, summary only) # EPSILON With --compare, max % diff to treat branch/compare as equal (default: 0.5) set -euo pipefail @@ -51,6 +52,7 @@ RESULTS="" ORIG_BRANCH="" COMPARE_TMP="" BRANCH_TMP="" +BUILD_LOG="" cleanup() { if [[ -n "$ORIG_BRANCH" ]]; then @@ -64,6 +66,7 @@ cleanup() { fi [[ -n "$COMPARE_TMP" ]] && rm -f "$COMPARE_TMP" [[ -n "$BRANCH_TMP" ]] && rm -f "$BRANCH_TMP" + [[ -n "$BUILD_LOG" ]] && rm -f "$BUILD_LOG" [[ -n "$RESULTS" ]] && rm -f "$RESULTS" return 0 } @@ -75,8 +78,8 @@ usage() { cat <"$BUILD_LOG" 2>&1; then + cat "$BUILD_LOG" >&2 + return 1 + fi +} + build_branch_binary() { local name="$1" dest="$2" local dtest_rel="bazel-bin/library/tests/dtest" @@ -229,8 +256,7 @@ build_branch_binary() { return 0 fi echo "Building dtest from '$name'..." >&2 - git -C "$ROOT" checkout "$name" - (cd "$ROOT" && bazel build //library/tests:dtest) + run_build checkout_and_build "$name" cp -L "$ROOT/$dtest_rel" "$dest" chmod +x "$dest" } @@ -270,7 +296,7 @@ setup_branches() { echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 else echo "Restoring '$ORIG_BRANCH'..." >&2 - git -C "$ROOT" checkout "$ORIG_BRANCH" + run_build git -C "$ROOT" checkout "$ORIG_BRANCH" fi BRANCH="$BRANCH_TMP" elif [[ "$num_branches" -eq 1 ]]; then @@ -282,8 +308,7 @@ setup_branches() { echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 else echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 - git -C "$ROOT" checkout "$ORIG_BRANCH" - (cd "$ROOT" && bazel build //library/tests:dtest) + run_build checkout_and_build "$ORIG_BRANCH" fi COMPARE="$COMPARE_TMP" else @@ -296,7 +321,7 @@ setup_branches() { echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 else echo "Restoring '$ORIG_BRANCH'..." >&2 - git -C "$ROOT" checkout "$ORIG_BRANCH" + run_build git -C "$ROOT" checkout "$ORIG_BRANCH" fi BRANCH="$BRANCH_TMP" COMPARE="$COMPARE_TMP" @@ -353,7 +378,7 @@ if [[ "$BUILD" == "1" ]]; then echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 else echo "Building //library/tests:dtest..." >&2 - (cd "$ROOT" && bazel build //library/tests:dtest) + run_build bazel_dtest fi fi @@ -516,9 +541,7 @@ if [[ -n "${COMPARE:-}" ]]; then printf "%-12s %s\n" "compare:" "$COMPARE" fi if [[ "$DETAILS" == "1" ]]; then - printf "%-12s %s\n" "details:" "on" - elif [[ -t 1 ]]; then - printf "%-12s %s\n" "details:" "transient (cleared before summary)" + printf "%-12s %s\n" "details:" "on (per-run rows + build output)" else printf "%-12s %s\n" "details:" "off (summary only)" fi From 5857d562f2f7050837d6ac2f669d92bd80bc44d9 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 17:35:19 -0500 Subject: [PATCH 12/15] Allow --branch . to mean the current branch Resolve a "." branch argument to the current branch (or HEAD when detached) before ref validation, so it can be compared without naming it explicitly. Co-authored-by: Cursor --- benchmark.sh | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 6a7e0eb4..9550e19b 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -87,12 +87,13 @@ Options: --max-deals N Include list10^n.txt files with 10^n <= N (default: 100; env: MAX_DEALS) (alias: --max_deals) --build Build branch dtest only (bazel build //library/tests:dtest) - --branch NAME Git branch to build and compare. Once: compare the current branch - against NAME. Twice (--branch A --branch B): compare A vs B and - ignore the current branch. With --compare PATH: build NAME as the - branch binary and compare it against PATH, ignoring the current - branch. Each branch is checked out, dtest is built and its binary - saved; the original branch is then restored. Requires a clean tree. + --branch NAME Git branch to build and compare ("." means the current branch). + Once: compare the current branch against NAME. Twice (--branch A + --branch B): compare A vs B and ignore the current branch. With + --compare PATH: build NAME as the branch binary and compare it + against PATH, ignoring the current branch. Each branch is checked + out, dtest is built and its binary saved; the original branch is + then restored. Requires a clean tree. --compare PATH Second dtest binary (summary; transient progress on tty). May be combined with a single --branch NAME (NAME backs the branch binary). --details Keep per-run timing rows and build (git/bazel) output @@ -272,6 +273,18 @@ setup_branches() { echo "error: --branch requires a git work tree at $ROOT" >&2 exit 1 fi + + ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ + || git -C "$ROOT" rev-parse HEAD)" + + # "." is shorthand for the current branch. + local i + for i in "${!BRANCH_NAMES[@]}"; do + if [[ "${BRANCH_NAMES[$i]}" == "." ]]; then + BRANCH_NAMES[$i]="$ORIG_BRANCH" + fi + done + local name for name in "${BRANCH_NAMES[@]}"; do if ! git -C "$ROOT" rev-parse --verify --quiet "$name" >/dev/null; then @@ -284,9 +297,6 @@ setup_branches() { exit 1 fi - ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ - || git -C "$ROOT" rev-parse HEAD)" - if [[ "$COMPARE_GIVEN" == "1" ]]; then # --branch NAME --compare PATH: build NAME as the branch binary and keep the # user-supplied compare path. The current branch's dtest is not used. From 065af9a33b8799a0b5bde802fa46f4f932842924 Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 17:52:03 -0500 Subject: [PATCH 13/15] Show live per-run rows during runs, hidden before the summary Display the script's per-run timing rows as progress while the benchmark runs. On a tty without --details they are shown on the alternate screen and discarded when we switch back to the main screen just before the summary, so they never clutter the final output regardless of how much scrolls. With --details they stay on the main screen; off a tty they are suppressed (summary only). dtest's own output is captured, not shown. Replaces the old transient per-run rows, whose ANSI line-clearing could not erase content that had scrolled off-screen. Co-authored-by: Cursor --- benchmark.sh | 64 +++++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 9550e19b..eea85f82 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -55,6 +55,12 @@ BRANCH_TMP="" BUILD_LOG="" cleanup() { + # Leave the alternate screen first so any restore/error messages and the shell + # prompt land on the normal screen. + if [[ "${ALT_SCREEN_ACTIVE:-0}" == "1" ]]; then + printf '\033[?1049l' >/dev/tty 2>/dev/null || true + ALT_SCREEN_ACTIVE=0 + fi if [[ -n "$ORIG_BRANCH" ]]; then local cur cur="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ @@ -468,6 +474,7 @@ run_dtest() { # Wrap with `time -p` so we can capture wall-clock elapsed for this run. Its # "real/user/sys" lines go to the merged output but do not collide with the # dtest lines parse_dtest_output looks for. + # local out if ! out="$(/usr/bin/time -p "${cmd[@]}" 2>&1)"; then echo "error: dtest failed: ${cmd[*]}" >&2 @@ -487,8 +494,6 @@ run_dtest() { echo "$parsed $wall" } -progress_lines=0 -TRANSIENT_PROGRESS=0 show_run_lines=1 print_run_table_header() { @@ -496,31 +501,11 @@ print_run_table_header() { "solver" "file" "ver" "user_ms" "sys_ms" "avg_user" "ratio" "run" printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ "------" "-------------" "-------" "--------" "--------" "----------" "------" "---" - if [[ "$TRANSIENT_PROGRESS" == "1" ]]; then - progress_lines=$((progress_lines + 2)) - fi } print_run_row() { printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" - if [[ "$TRANSIENT_PROGRESS" == "1" ]]; then - progress_lines=$((progress_lines + 1)) - fi -} - -clear_transient_progress() { - if [[ "$TRANSIENT_PROGRESS" != "1" || $progress_lines -le 0 ]]; then - return - fi - # Cursor rests on a blank line below the last row; erase it, then each table line. - # Do not move up after clearing the topmost line (would hit the header above). - printf '\033[2K' - local i - for (( i = 0; i < progress_lines; i++ )); do - printf '\033[1A\033[2K' - done - progress_lines=0 } echo "DDS dtest benchmark" @@ -572,16 +557,28 @@ if ((${#DTEST_EXTRA[@]} > 0)); then fi echo +# The per-run rows are the script's live progress. With --details they are kept +# in the final output. Without --details, on a tty, they are shown on the +# alternate screen so the user sees progress, then discarded when we switch back +# to the main screen just before the summary; off a tty they are suppressed +# (summary only), since there is nothing to hide them. show_run_lines=0 -TRANSIENT_PROGRESS=0 -# Per-run rows are detail: show them only with --details. Without it they are -# suppressed entirely (no transient progress table, which could otherwise scroll -# off-screen and leave residue). The summary is always shown regardless. -if [[ "$DETAILS" == "1" ]]; then - show_run_lines=1 +ALT_SCREEN=0 +if [[ "$DRY_RUN" != "1" ]]; then + if [[ "$DETAILS" == "1" ]]; then + show_run_lines=1 + elif [[ -t 1 ]]; then + show_run_lines=1 + ALT_SCREEN=1 + fi fi -if [[ "$DRY_RUN" != "1" && ( "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ) ]]; then +if [[ "$ALT_SCREEN" == "1" ]]; then + printf '\033[?1049h\033[H\033[2J' >/dev/tty # enter alt screen, home, clear + ALT_SCREEN_ACTIVE=1 +fi + +if [[ "$DRY_RUN" != "1" && "$show_run_lines" == "1" ]]; then print_run_table_header fi @@ -611,7 +608,7 @@ for solver in "${SOLVERS[@]}"; do read -r user sys avg ratio wall < <(run_dtest "$bin" "$solver" "$hands") - if [[ "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ]]; then + if [[ "$show_run_lines" == "1" ]]; then print_run_row "$solver" "$file" "$ver" "$user" "$sys" "$avg" "$ratio" "$run_label" fi @@ -623,7 +620,12 @@ for solver in "${SOLVERS[@]}"; do done done -clear_transient_progress +# Return to the normal screen, discarding the live dtest output, before the +# summary so the final output is just the header and the summary. +if [[ "$ALT_SCREEN" == "1" ]]; then + printf '\033[?1049l' >/dev/tty + ALT_SCREEN_ACTIVE=0 +fi if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then echo # compare summary (two binaries) From 7e2bcf8771186c372fa0f399db687ff0876ed2ea Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Sun, 28 Jun 2026 22:45:11 -0500 Subject: [PATCH 14/15] Treat untracked files as non-clean in --branch check The cleanliness check for --branch used --untracked-files=no, so untracked files passed the check but could still block git checkout ("would be overwritten"). Include untracked files so the check matches the documented "clean tree" requirement. Co-authored-by: Cursor --- benchmark.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index eea85f82..3df030e2 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -298,8 +298,10 @@ setup_branches() { exit 1 fi done - if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no)" ]]; then - echo "error: tracked changes present; commit or stash before using --branch" >&2 + # Untracked files can also block a checkout ("would be overwritten"), so treat + # any working tree change (tracked or untracked) as non-clean. + if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then + echo "error: working tree not clean; commit, stash, or remove changes (tracked or untracked) before using --branch" >&2 exit 1 fi From 590b145c15b9428463b30fd54315f1186f7891de Mon Sep 17 00:00:00 2001 From: Adam Wildavsky Date: Mon, 29 Jun 2026 04:46:03 +0100 Subject: [PATCH 15/15] Use `command time -p` for portability Hard-coding /usr/bin/time reduces portability (some environments only provide time as a shell keyword or at a different path). Using command time -p keeps the portable format while avoiding an absolute path dependency. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark.sh b/benchmark.sh index eea85f82..0bd9372b 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -476,7 +476,7 @@ run_dtest() { # dtest lines parse_dtest_output looks for. # local out - if ! out="$(/usr/bin/time -p "${cmd[@]}" 2>&1)"; then + if ! out="$(command time -p "${cmd[@]}" 2>&1)"; then echo "error: dtest failed: ${cmd[*]}" >&2 echo "$out" >&2 exit 1