diff --git a/benchmark.sh b/benchmark.sh index e4f2a605..22d136a9 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -2,16 +2,18 @@ # Benchmark dtest performance on one or two binaries. # # Runs all combinations of solver (solve, calc) and hand file -# (list100/1000/…/1), largest files first. With --compare, prints summary only -# unless --details; without --compare, prints per-run rows. With --compare and a -# tty, per-run rows appear during the run then are cleared before the summary. -# Does not pass dtest options unless given after "--" (see below). +# (list100/1000/…/1), largest files first. Always prints a summary. Per-run +# timing rows and build (git/bazel) output are shown only with --details; +# otherwise build output is captured to a log (surfaced only on failure) and +# per-run rows are suppressed. Does not pass dtest options unless given after +# "--" (see below). # # Usage: # ./benchmark.sh # ./benchmark.sh --build # ./benchmark.sh -- -n 8 -r # ./benchmark.sh --build --compare /path/to/other/dtest +# ./benchmark.sh --branch develop -- -n 8 # ./benchmark.sh --compare /path/to/other/dtest --epsilon 1 # ./benchmark.sh --repeats 5 -- -n 4 # REPEATS=3 ./benchmark.sh @@ -19,11 +21,12 @@ # Environment: # BRANCH Path to branch dtest (default: bazel-bin in this repo) # COMPARE Optional second dtest binary for comparison +# (or use --branch NAME to build the compare binary from a git branch) # HANDS_DIR Directory containing list*.txt files (default: ./hands) # REPEATS Runs per combination per binary (default: 1) # MAX_DEALS Include list10^n.txt files with 10^n <= N (default: 100) # DRY_RUN If 1, print commands only -# DETAILS If 1 with --compare, keep per-run rows in output (default: transient on tty) +# DETAILS If 1, keep per-run rows and build output (default: 0, summary only) # EPSILON With --compare, max % diff to treat branch/compare as equal (default: 0.5) set -euo pipefail @@ -38,16 +41,51 @@ DETAILS="${DETAILS:-0}" EPSILON="${EPSILON:-0.5}" BUILD=0 REVERSE=0 +BRANCH_NAMES=() +COMPARE_GIVEN=0 +REPEATS_GIVEN=0 DTEST_EXTRA=() +# Cleanup state (set later). The EXIT trap restores the original git branch if +# --branch switched away, and removes temp files. +RESULTS="" +ORIG_BRANCH="" +COMPARE_TMP="" +BRANCH_TMP="" +BUILD_LOG="" + +cleanup() { + # Leave the alternate screen first so any restore/error messages and the shell + # prompt land on the normal screen. + if [[ "${ALT_SCREEN_ACTIVE:-0}" == "1" ]]; then + printf '\033[?1049l' >/dev/tty 2>/dev/null || true + ALT_SCREEN_ACTIVE=0 + fi + if [[ -n "$ORIG_BRANCH" ]]; then + local cur + cur="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ + || git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)" + if [[ -n "$cur" && "$cur" != "$ORIG_BRANCH" ]]; then + echo "Restoring git branch '$ORIG_BRANCH'..." >&2 + git -C "$ROOT" checkout "$ORIG_BRANCH" >/dev/null 2>&1 || true + fi + fi + [[ -n "$COMPARE_TMP" ]] && rm -f "$COMPARE_TMP" + [[ -n "$BRANCH_TMP" ]] && rm -f "$BRANCH_TMP" + [[ -n "$BUILD_LOG" ]] && rm -f "$BUILD_LOG" + [[ -n "$RESULTS" ]] && rm -f "$RESULTS" + return 0 +} +trap cleanup EXIT + SOLVERS=(solve calc) usage() { cat <&2 + exit 1 + fi + REPEATS_GIVEN=1 shift REPEATS="${1:?missing value for --repeats}" shift ;; --branch) shift - BRANCH="${1:?missing value for --branch}" + BRANCH_NAMES+=("${1:?missing value for --branch}") shift ;; --compare) shift COMPARE="${1:?missing value for --compare}" + COMPARE_GIVEN=1 shift ;; --max-deals|--max_deals|-max-deals|-max_deals) @@ -151,11 +206,151 @@ if ! [[ "$EPSILON" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then exit 1 fi -if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" ]]; then - echo "error: --reverse requires --compare" >&2 +# A compare binary supplied via the COMPARE env var behaves like --compare PATH. +if [[ "$COMPARE_GIVEN" == "0" && -n "${COMPARE:-}" ]]; then + COMPARE_GIVEN=1 +fi + +num_branches=${#BRANCH_NAMES[@]} + +if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" && "$num_branches" -eq 0 ]]; then + echo "error: --reverse requires --compare or --branch" >&2 + exit 1 +fi + +if [[ "$num_branches" -gt 0 && "$COMPARE_GIVEN" == "1" && "$num_branches" -ne 1 ]]; then + echo "error: --compare accepts exactly one --branch (got $num_branches)" >&2 exit 1 fi +if [[ "$num_branches" -gt 2 ]]; then + echo "error: --branch may be given at most twice (got $num_branches)" >&2 + exit 1 +fi + +# Check out $1, build dtest, and copy the binary to $2. +# Build output (git checkout + bazel) is noise unless --details was given. +# Trying to show it live and erase it afterward with ANSI does not work: bazel +# drives its own cursor save/restore for its progress display, clobbering any +# saved position, so a restore+clear erases nothing. Instead, capture the +# output to a log and surface it only on failure (or with --details). bazel sees +# a non-tty here and emits plain, line-based output. The short "Building..." +# labels are kept as progress markers. +bazel_dtest() { ( cd "$ROOT" && bazel build //library/tests:dtest ); } +checkout_and_build() { git -C "$ROOT" checkout "$1" && bazel_dtest; } + +run_build() { + if [[ "$DETAILS" == "1" ]]; then + "$@" + return + fi + if [[ -z "$BUILD_LOG" ]]; then + BUILD_LOG="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-build.XXXXXX")" + fi + if ! "$@" >"$BUILD_LOG" 2>&1; then + cat "$BUILD_LOG" >&2 + return 1 + fi +} + +build_branch_binary() { + local name="$1" dest="$2" + local dtest_rel="bazel-bin/library/tests/dtest" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $name" >&2 + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + echo "DRY_RUN: cp -L $ROOT/$dtest_rel $dest" >&2 + return 0 + fi + echo "Building dtest from '$name'..." >&2 + run_build checkout_and_build "$name" + cp -L "$ROOT/$dtest_rel" "$dest" + chmod +x "$dest" +} + +# Branch-mode binary selection: +# --branch NAME : branch = current checkout, compare = build(NAME) +# --branch NAME --compare PATH : branch = build(NAME), compare = PATH +# (the current branch's dtest is ignored) +# --branch A --branch B : branch = build(A), compare = build(B) +# (the current branch's dtest is ignored) +setup_branches() { + if ! git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "error: --branch requires a git work tree at $ROOT" >&2 + exit 1 + fi + + ORIG_BRANCH="$(git -C "$ROOT" symbolic-ref --quiet --short HEAD 2>/dev/null \ + || git -C "$ROOT" rev-parse HEAD)" + + # "." is shorthand for the current branch. + local i + for i in "${!BRANCH_NAMES[@]}"; do + if [[ "${BRANCH_NAMES[$i]}" == "." ]]; then + BRANCH_NAMES[$i]="$ORIG_BRANCH" + fi + done + + local name + for name in "${BRANCH_NAMES[@]}"; do + if ! git -C "$ROOT" rev-parse --verify --quiet "$name" >/dev/null; then + echo "error: --branch: unknown git ref '$name'" >&2 + exit 1 + fi + done + # Untracked files can also block a checkout ("would be overwritten"), so treat + # any working tree change (tracked or untracked) as non-clean. + if [[ -n "$(git -C "$ROOT" status --porcelain --untracked-files=normal)" ]]; then + echo "error: working tree not clean; commit, stash, or remove changes (tracked or untracked) before using --branch" >&2 + exit 1 + fi + + if [[ "$COMPARE_GIVEN" == "1" ]]; then + # --branch NAME --compare PATH: build NAME as the branch binary and keep the + # user-supplied compare path. The current branch's dtest is not used. + BRANCH_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-branch.XXXXXX")" + build_branch_binary "${BRANCH_NAMES[0]}" "$BRANCH_TMP" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + else + echo "Restoring '$ORIG_BRANCH'..." >&2 + run_build git -C "$ROOT" checkout "$ORIG_BRANCH" + fi + BRANCH="$BRANCH_TMP" + elif [[ "$num_branches" -eq 1 ]]; then + COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" + build_branch_binary "${BRANCH_NAMES[0]}" "$COMPARE_TMP" + # Restore the current branch and rebuild it as the branch binary. + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 + else + echo "Restoring '$ORIG_BRANCH' and rebuilding..." >&2 + run_build checkout_and_build "$ORIG_BRANCH" + fi + COMPARE="$COMPARE_TMP" + else + # Two branches: build both, ignore the current branch's binary. + BRANCH_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-branch.XXXXXX")" + COMPARE_TMP="$(mktemp "${TMPDIR:-/tmp}/dds-dtest-compare.XXXXXX")" + build_branch_binary "${BRANCH_NAMES[0]}" "$BRANCH_TMP" + build_branch_binary "${BRANCH_NAMES[1]}" "$COMPARE_TMP" + if [[ "$DRY_RUN" == "1" ]]; then + echo "DRY_RUN: git -C $ROOT checkout $ORIG_BRANCH" >&2 + else + echo "Restoring '$ORIG_BRANCH'..." >&2 + run_build git -C "$ROOT" checkout "$ORIG_BRANCH" + fi + BRANCH="$BRANCH_TMP" + COMPARE="$COMPARE_TMP" + fi +} + +if [[ "$num_branches" -gt 0 ]]; then + setup_branches + BUILD=0 # build already done as part of the branch workflow +fi + select_hand_files() { is_power_of_10() { local n="$1" @@ -201,7 +396,7 @@ if [[ "$BUILD" == "1" ]]; then echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2 else echo "Building //library/tests:dtest..." >&2 - (cd "$ROOT" && bazel build //library/tests:dtest) + run_build bazel_dtest fi fi @@ -241,7 +436,7 @@ if git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then fi RESULTS="$(mktemp "${TMPDIR:-/tmp}/dds-benchmark.XXXXXX")" -trap 'rm -f "$RESULTS"' EXIT +# Removal handled by the cleanup() EXIT trap installed near the top. parse_dtest_output() { awk ' @@ -278,8 +473,12 @@ run_dtest() { return 0 fi + # Wrap with `time -p` so we can capture wall-clock elapsed for this run. Its + # "real/user/sys" lines go to the merged output but do not collide with the + # dtest lines parse_dtest_output looks for. + # local out - if ! out="$("${cmd[@]}" 2>&1)"; then + if ! out="$(command time -p "${cmd[@]}" 2>&1)"; then echo "error: dtest failed: ${cmd[*]}" >&2 echo "$out" >&2 exit 1 @@ -291,11 +490,12 @@ run_dtest() { if [[ "$parsed_user" == "NA" || "$parsed_sys" == "NA" ]]; then echo "warning: incomplete dtest timing output: ${cmd[*]}" >&2 fi - echo "$parsed" + local wall + wall="$(awk '/^real[[:space:]]/ { print $2; exit }' <<<"$out")" + [[ -z "$wall" ]] && wall="NA" + echo "$parsed $wall" } -progress_lines=0 -TRANSIENT_PROGRESS=0 show_run_lines=1 print_run_table_header() { @@ -303,42 +503,42 @@ print_run_table_header() { "solver" "file" "ver" "user_ms" "sys_ms" "avg_user" "ratio" "run" printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ "------" "-------------" "-------" "--------" "--------" "----------" "------" "---" - if [[ "$TRANSIENT_PROGRESS" == "1" ]]; then - progress_lines=$((progress_lines + 2)) - fi } print_run_row() { printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \ "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" - if [[ "$TRANSIENT_PROGRESS" == "1" ]]; then - progress_lines=$((progress_lines + 1)) - fi -} - -clear_transient_progress() { - if [[ "$TRANSIENT_PROGRESS" != "1" || $progress_lines -le 0 ]]; then - return - fi - # Cursor rests on a blank line below the last row; erase it, then each table line. - # Do not move up after clearing the topmost line (would hit the header above). - printf '\033[2K' - local i - for (( i = 0; i < progress_lines; i++ )); do - printf '\033[1A\033[2K' - done - progress_lines=0 } echo "DDS dtest benchmark" echo "===================" -printf "%-12s %s\n" "branch:" "$BRANCH" +# Branch names backing each binary, when --branch was used. With one --branch +# the branch binary is the current checkout; with two it is the first name. +branch_branch_name="" +compare_branch_name="" +if [[ "$num_branches" -eq 1 && "$COMPARE_GIVEN" == "1" ]]; then + # --branch NAME --compare PATH: NAME backs the branch binary; compare is a path. + branch_branch_name="${BRANCH_NAMES[0]}" +elif [[ "$num_branches" -eq 1 ]]; then + compare_branch_name="${BRANCH_NAMES[0]}" +elif [[ "$num_branches" -eq 2 ]]; then + branch_branch_name="${BRANCH_NAMES[0]}" + compare_branch_name="${BRANCH_NAMES[1]}" +fi + +if [[ -n "$branch_branch_name" ]]; then + printf "%-12s %s\n" "branch:" "branch '$branch_branch_name' ($BRANCH)" +else + printf "%-12s %s\n" "branch:" "$BRANCH" +fi if [[ -n "${COMPARE:-}" ]]; then - printf "%-12s %s\n" "compare:" "$COMPARE" + if [[ -n "$compare_branch_name" ]]; then + printf "%-12s %s\n" "compare:" "branch '$compare_branch_name' ($COMPARE)" + else + printf "%-12s %s\n" "compare:" "$COMPARE" + fi if [[ "$DETAILS" == "1" ]]; then - printf "%-12s %s\n" "details:" "on" - elif [[ -t 1 ]]; then - printf "%-12s %s\n" "details:" "transient (cleared before summary)" + printf "%-12s %s\n" "details:" "on (per-run rows + build output)" else printf "%-12s %s\n" "details:" "off (summary only)" fi @@ -359,16 +559,28 @@ if ((${#DTEST_EXTRA[@]} > 0)); then fi echo -show_run_lines=1 -TRANSIENT_PROGRESS=0 -if [[ -n "${COMPARE:-}" && "$DETAILS" != "1" ]]; then - show_run_lines=0 - if [[ -t 1 ]]; then - TRANSIENT_PROGRESS=1 +# The per-run rows are the script's live progress. With --details they are kept +# in the final output. Without --details, on a tty, they are shown on the +# alternate screen so the user sees progress, then discarded when we switch back +# to the main screen just before the summary; off a tty they are suppressed +# (summary only), since there is nothing to hide them. +show_run_lines=0 +ALT_SCREEN=0 +if [[ "$DRY_RUN" != "1" ]]; then + if [[ "$DETAILS" == "1" ]]; then + show_run_lines=1 + elif [[ -t 1 ]]; then + show_run_lines=1 + ALT_SCREEN=1 fi fi -if [[ "$DRY_RUN" != "1" && ( "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ) ]]; then +if [[ "$ALT_SCREEN" == "1" ]]; then + printf '\033[?1049h\033[H\033[2J' >/dev/tty # enter alt screen, home, clear + ALT_SCREEN_ACTIVE=1 +fi + +if [[ "$DRY_RUN" != "1" && "$show_run_lines" == "1" ]]; then print_run_table_header fi @@ -396,32 +608,65 @@ for solver in "${SOLVERS[@]}"; do continue fi - read -r user sys avg ratio < <(run_dtest "$bin" "$solver" "$hands") + read -r user sys avg ratio wall < <(run_dtest "$bin" "$solver" "$hands") - if [[ "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ]]; then + if [[ "$show_run_lines" == "1" ]]; then print_run_row "$solver" "$file" "$ver" "$user" "$sys" "$avg" "$ratio" "$run_label" fi - printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ - "$solver" "$file" "$ver" "$rep" "$user" "$sys" "$avg" "$ratio" \ + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ + "$solver" "$file" "$ver" "$rep" "$user" "$sys" "$avg" "$ratio" "$wall" \ >>"$RESULTS" done done done done -clear_transient_progress +# Return to the normal screen, discarding the live dtest output, before the +# summary so the final output is just the header and the summary. +if [[ "$ALT_SCREEN" == "1" ]]; then + printf '\033[?1049l' >/dev/tty + ALT_SCREEN_ACTIVE=0 +fi if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then - echo - echo "Summary (branch vs compare, avg user ms)" + echo # compare summary (two binaries) + # Column headers default to the generic labels, but show the actual branch + # names when known: the current git branch for the branch binary, and the + # --branch name for the compare binary. Truncated to the 12-char column. + cmp_label="compare_avg" + if [[ -n "$compare_branch_name" ]]; then + cmp_label="${compare_branch_name:0:12}" + fi + br_label="branch_avg" + if [[ -n "$branch_branch_name" ]]; then + br_label="${branch_branch_name:0:12}" + elif [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then + br_label="${git_branch:0:12}" + fi + + # Names used in the "note" column (" faster"); fall back to the generic + # "branch"/"compare" when no branch name is known. + note_branch="branch" + if [[ -n "$branch_branch_name" ]]; then + note_branch="$branch_branch_name" + elif [[ -n "$compare_branch_name" && -n "$git_branch" && "$git_branch" != "unknown" ]]; then + note_branch="$git_branch" + fi + note_compare="compare" + if [[ -n "$compare_branch_name" ]]; then + note_compare="$compare_branch_name" + fi + + echo "Summary (avg user ms)" echo "==============================================================================" printf "%-6s %-13s %12s %12s %10s %-15s\n" \ - "solver" "file" "compare_avg" "branch_avg" "cmp/branch" "note" + "solver" "file" "$cmp_label" "$br_label" "cmp/branch" "note" printf "%-6s %-13s %12s %12s %10s %-15s\n" \ "------" "-------------" "------------" "------------" "----------" "---------------" - awk -F'\t' -v files="${FILES[*]}" -v epsilon_pct="$EPSILON" ' + awk -F'\t' -v files="${FILES[*]}" -v epsilon_pct="$EPSILON" \ + -v note_branch="$note_branch" -v note_compare="$note_compare" ' function within_epsilon(a, b, eps, hi, lo) { eps = epsilon_pct / 100 if (a > b) { hi = a; lo = b } else { hi = b; lo = a } @@ -432,9 +677,11 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then if ($3 == "compare") { s2[base] += $7 c2[base]++ + if ($9 != "NA") tw2 += $9 # total wall-clock elapsed, compare } else if ($3 == "branch") { s1[base] += $7 c1[base]++ + if ($9 != "NA") tw1 += $9 # total wall-clock elapsed, branch } } END { @@ -454,15 +701,67 @@ if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then if (within_epsilon(u1, u2)) { note = "equal" } else if (cmp_branch >= 1) { - note = "branch faster" + note = note_branch " faster" } else { - note = "compare faster" + note = note_compare " faster" } sp = sprintf("%9.2fx", cmp_branch) printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", solvers[si], filearr[fi], u2, u1, sp, note } } + + # Total elapsed (wall-clock seconds) summed per binary across all runs. + printf "%-6s %-13s %12s %12s %10s %-15s\n", + "------", "-------------", "------------", "------------", "----------", "---------------" + if (tw2 > 0 && tw1 > 0) { + tnote = "" + if (within_epsilon(tw1, tw2)) tnote = "equal" + else if (tw2 / tw1 >= 1) tnote = note_branch " faster" + else tnote = note_compare " faster" + tsp = sprintf("%9.2fx", tw2 / tw1) + printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", + "TOTAL", "elapsed (s)", tw2, tw1, tsp, tnote + } else { + printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n", + "TOTAL", "elapsed (s)", tw2, tw1, "", "" + } + } + ' "$RESULTS" +elif [[ "$DRY_RUN" != "1" ]]; then + echo # single-binary summary (no --compare) + br_label="branch_avg" + if [[ -n "$git_branch" && "$git_branch" != "unknown" ]]; then + br_label="${git_branch:0:12}" + fi + + echo "Summary (avg user ms)" + echo "============================================================" + printf "%-6s %-13s %12s %6s\n" "solver" "file" "$br_label" "runs" + printf "%-6s %-13s %12s %6s\n" \ + "------" "-------------" "------------" "------" + + awk -F'\t' -v files="${FILES[*]}" ' + $3 == "branch" { + base = $1 SUBSEP $2 + s[base] += $7 + c[base]++ + if ($9 != "NA") tw += $9 # total wall-clock elapsed + } + END { + split("solve calc", solvers, " ") + nfiles = split(files, filearr, " ") + for (si = 1; si <= 2; si++) { + for (fi = 1; fi <= nfiles; fi++) { + base = solvers[si] SUBSEP filearr[fi] + if (!(base in c)) continue + printf "%-6s %-13s %12.2f %6d\n", + solvers[si], filearr[fi], s[base] / c[base], c[base] + } + } + printf "%-6s %-13s %12s %6s\n", + "------", "-------------", "------------", "------" + printf "%-6s %-13s %12.2f %6s\n", "TOTAL", "elapsed (s)", tw, "" } ' "$RESULTS" fi