benchmark.sh

#!/usr/bin/env bash
# Benchmark dtest performance on one or two binaries.
#
# Runs all combinations of solver (solve, calc) and hand file
# (list100/1000/…/1), largest files first. With --compare, prints summary only
# unless --details; without --compare, prints per-run rows. With --compare and a
# tty, per-run rows appear during the run then are cleared before the summary.
# Does not pass dtest options unless given after "--" (see below).
#
# Usage:
#   ./benchmark.sh
#   ./benchmark.sh --build
#   ./benchmark.sh -- -n 8 -r
#   ./benchmark.sh --build --compare /path/to/other/dtest
#   ./benchmark.sh --compare /path/to/other/dtest --epsilon 1
#   ./benchmark.sh --repeats 5 -- -n 4
#   REPEATS=3 ./benchmark.sh
#
# Environment:
#   BRANCH     Path to branch dtest (default: bazel-bin in this repo)
#   COMPARE    Optional second dtest binary for comparison
#   HANDS_DIR  Directory containing list*.txt files (default: ./hands)
#   REPEATS    Runs per combination per binary (default: 1)
#   MAX_DEALS  Include list10^n.txt files with 10^n <= N (default: 100)
#   DRY_RUN    If 1, print commands only
#   DETAILS    If 1 with --compare, keep per-run rows in output (default: transient on tty)
#   EPSILON    With --compare, max % diff to treat branch/compare as equal (default: 0.5)

set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BRANCH="${BRANCH:-$ROOT/bazel-bin/library/tests/dtest}"
HANDS_DIR="${HANDS_DIR:-$ROOT/hands}"
REPEATS="${REPEATS:-1}"
MAX_DEALS="${MAX_DEALS:-100}"
DRY_RUN="${DRY_RUN:-0}"
DETAILS="${DETAILS:-0}"
EPSILON="${EPSILON:-0.5}"
BUILD=0
REVERSE=0
DTEST_EXTRA=()

SOLVERS=(solve calc)

usage() {
  cat <<EOF
Usage: $(basename "$0") [OPTIONS]

Benchmark dtest across solver/file combinations. With --compare, prints a summary
by default; use --details for per-run rows.

Options:
  -h, --help          Show this help
  --repeats N         Runs per combination per binary (default: 1; env: REPEATS)
  --max-deals N       Include list10^n.txt files with 10^n <= N (default: 100; env: MAX_DEALS)
                      (alias: --max_deals)
  --build             Build branch dtest only (bazel build //library/tests:dtest)
  --branch PATH       Branch dtest binary (default: $BRANCH)
  --compare PATH      Optional second dtest binary (summary; transient progress on tty)
  --details           With --compare, keep per-run timing rows in final output
  --epsilon PCT       With --compare, treat timings within PCT% as equal (default: 0.5; env: EPSILON)
  --reverse           With --compare, run compare before branch each repeat (default: branch first)
  --                  End benchmark options; remaining args are passed to dtest
                      (e.g. -- -n 8 -r for 8 threads and slow-board report)

Environment:
  BRANCH, COMPARE, HANDS_DIR, REPEATS, MAX_DEALS, DRY_RUN, DETAILS, EPSILON

Examples:
  ./benchmark.sh
  ./benchmark.sh --build
  ./benchmark.sh -- -n 8
  ./benchmark.sh --repeats 3 -- -n 4 -r
  ./benchmark.sh --compare /path/to/dtest
  ./benchmark.sh --compare /path/to/dtest --details
  ./benchmark.sh --compare /path/to/dtest --epsilon 1
  ./benchmark.sh --compare /path/to/dtest --reverse
  ./benchmark.sh --repeats 5 --compare /path/to/dtest
  DRY_RUN=1 ./benchmark.sh
EOF
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    -h|--help)
      usage
      exit 0
      ;;
    --repeats)
      shift
      REPEATS="${1:?missing value for --repeats}"
      shift
      ;;
    --branch)
      shift
      BRANCH="${1:?missing value for --branch}"
      shift
      ;;
    --compare)
      shift
      COMPARE="${1:?missing value for --compare}"
      shift
      ;;
    --max-deals|--max_deals|-max-deals|-max_deals)
      shift
      MAX_DEALS="${1:?missing value for --max-deals}"
      shift
      ;;
    --build)
      BUILD=1
      shift
      ;;
    --reverse)
      REVERSE=1
      shift
      ;;
    --details)
      DETAILS=1
      shift
      ;;
    --epsilon)
      shift
      EPSILON="${1:?missing value for --epsilon}"
      shift
      ;;
    --)
      shift
      DTEST_EXTRA=("$@")
      break
      ;;
    *)
      echo "Unknown option: $1" >&2
      usage >&2
      exit 1
      ;;
  esac
done

if ! [[ "$MAX_DEALS" =~ ^[0-9]+$ ]] || (( MAX_DEALS < 1 )); then
  echo "error: max_deals must be a positive integer (got: $MAX_DEALS)" >&2
  exit 1
fi

if ! [[ "$REPEATS" =~ ^[0-9]+$ ]] || (( REPEATS < 1 )); then
  echo "error: repeats must be a positive integer (got: $REPEATS)" >&2
  exit 1
fi

if ! [[ "$EPSILON" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
  echo "error: epsilon must be a non-negative number (got: $EPSILON)" >&2
  exit 1
fi

if [[ "$REVERSE" == "1" && -z "${COMPARE:-}" ]]; then
  echo "error: --reverse requires --compare" >&2
  exit 1
fi

select_hand_files() {
  is_power_of_10() {
    local n="$1"
    (( n >= 1 )) || return 1
    while (( n > 1 )); do
      (( n % 10 == 0 )) || return 1
      n=$(( n / 10 ))
    done
    return 0
  }

  local -a candidates=()
  local path base count

  shopt -s nullglob
  for path in "$HANDS_DIR"/list*.txt; do
    base="${path##*/}"
    if [[ "$base" =~ ^list([0-9]+)\.txt$ ]]; then
      count="${BASH_REMATCH[1]}"
      if is_power_of_10 "$count" && (( count <= MAX_DEALS )); then
        candidates+=("${count}:${base}")
      fi
    fi
  done
  shopt -u nullglob

  if ((${#candidates[@]} == 0)); then
    echo "error: no list10^n.txt files with 10^n <= $MAX_DEALS in $HANDS_DIR" >&2
    exit 1
  fi

  FILES=()
  local item
  while IFS= read -r item; do
    FILES+=("${item#*:}")
  done < <(printf '%s\n' "${candidates[@]}" | sort -t: -k1,1rn)
}

select_hand_files

if [[ "$BUILD" == "1" ]]; then
  if [[ "$DRY_RUN" == "1" ]]; then
    echo "DRY_RUN: (cd $ROOT && bazel build //library/tests:dtest)" >&2
  else
    echo "Building //library/tests:dtest..." >&2
    (cd "$ROOT" && bazel build //library/tests:dtest)
  fi
fi

if [[ "$DRY_RUN" != "1" ]]; then
  if [[ ! -x "$BRANCH" ]]; then
    echo "error: branch binary not found or not executable: $BRANCH" >&2
    echo "hint: bazel build //library/tests:dtest" >&2
    exit 1
  fi

  if [[ -n "${COMPARE:-}" && ! -x "$COMPARE" ]]; then
    echo "error: compare binary not found or not executable: $COMPARE" >&2
    exit 1
  fi
fi

BIN_PAIRS=("branch:$BRANCH")
if [[ -n "${COMPARE:-}" ]]; then
  if [[ "$REVERSE" == "1" ]]; then
    BIN_PAIRS=("compare:$COMPARE" "branch:$BRANCH")
  else
    BIN_PAIRS=("branch:$BRANCH" "compare:$COMPARE")
  fi
fi
num_bins=${#BIN_PAIRS[@]}

for f in "${FILES[@]}"; do
  if [[ ! -f "$HANDS_DIR/$f" ]]; then
    echo "error: hand file not found: $HANDS_DIR/$f" >&2
    exit 1
  fi
done

git_branch="unknown"
if git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
  git_branch="$(git -C "$ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)"
fi

RESULTS="$(mktemp "${TMPDIR:-/tmp}/dds-benchmark.XXXXXX")"
trap 'rm -f "$RESULTS"' EXIT

parse_dtest_output() {
  awk '
    /^Number of hands/ { hands = $NF }
    /^User time \(ms\)/ { user = ($NF == "zero" ? 0 : $NF) }
    /^Sys time \(ms\)/  { sys = ($NF == "zero" ? 0 : $NF) }
    /^Avg user time \(ms\)/ { avg = ($NF == "zero" ? 0 : $NF) }
    /^Ratio[[:space:]]/ { ratio = $NF }
    END {
      if (user == "") user = "NA"
      if (sys == "") sys = "NA"
      if (avg == "") {
        if (user == 0) avg = 0
        else if (hands != "" && user != "NA" && hands > 0) avg = user / hands
        else avg = "NA"
      }
      if (ratio == "") ratio = "NA"
      print user, sys, avg, ratio
    }
  '
}

run_dtest() {
  local binary="$1"
  local solver="$2"
  local hands="$3"
  local -a cmd=("$binary" -f "$hands" -s "$solver")
  if ((${#DTEST_EXTRA[@]} > 0)); then
    cmd+=("${DTEST_EXTRA[@]}")
  fi

  if [[ "$DRY_RUN" == "1" ]]; then
    echo "DRY_RUN: ${cmd[*]}" >&2
    return 0
  fi

  local out
  if ! out="$("${cmd[@]}" 2>&1)"; then
    echo "error: dtest failed: ${cmd[*]}" >&2
    echo "$out" >&2
    exit 1
  fi
  local parsed
  parsed="$(parse_dtest_output <<<"$out")"
  local parsed_user parsed_sys
  read -r parsed_user parsed_sys _ _ <<<"$parsed"
  if [[ "$parsed_user" == "NA" || "$parsed_sys" == "NA" ]]; then
    echo "warning: incomplete dtest timing output: ${cmd[*]}" >&2
  fi
  echo "$parsed"
}

progress_lines=0
TRANSIENT_PROGRESS=0
show_run_lines=1

print_run_table_header() {
  printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \
    "solver" "file" "ver" "user_ms" "sys_ms" "avg_user" "ratio" "run"
  printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \
    "------" "-------------" "-------" "--------" "--------" "----------" "------" "---"
  if [[ "$TRANSIENT_PROGRESS" == "1" ]]; then
    progress_lines=$((progress_lines + 2))
  fi
}

print_run_row() {
  printf "%-6s %-13s %7s %8s %8s %10s %6s %s\n" \
    "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8"
  if [[ "$TRANSIENT_PROGRESS" == "1" ]]; then
    progress_lines=$((progress_lines + 1))
  fi
}

clear_transient_progress() {
  if [[ "$TRANSIENT_PROGRESS" != "1" || $progress_lines -le 0 ]]; then
    return
  fi
  # Cursor rests on a blank line below the last row; erase it, then each table line.
  # Do not move up after clearing the topmost line (would hit the header above).
  printf '\033[2K'
  local i
  for (( i = 0; i < progress_lines; i++ )); do
    printf '\033[1A\033[2K'
  done
  progress_lines=0
}

echo "DDS dtest benchmark"
echo "==================="
printf "%-12s %s\n" "branch:" "$BRANCH"
if [[ -n "${COMPARE:-}" ]]; then
  printf "%-12s %s\n" "compare:" "$COMPARE"
  if [[ "$DETAILS" == "1" ]]; then
    printf "%-12s %s\n" "details:" "on"
  elif [[ -t 1 ]]; then
    printf "%-12s %s\n" "details:" "transient (cleared before summary)"
  else
    printf "%-12s %s\n" "details:" "off (summary only)"
  fi
  if [[ "$REVERSE" == "1" ]]; then
    printf "%-12s %s\n" "run order:" "interleaved compare, branch"
  else
    printf "%-12s %s\n" "run order:" "interleaved branch, compare"
  fi
  printf "%-12s %s\n" "epsilon:" "${EPSILON}%"
fi
printf "%-12s %s\n" "hands dir:" "$HANDS_DIR"
printf "%-12s %s\n" "max_deals:" "$MAX_DEALS"
printf "%-12s %s\n" "files:" "${FILES[*]}"
printf "%-12s %s\n" "git branch:" "$git_branch"
printf "%-12s %s\n" "repeats:" "$REPEATS"
if ((${#DTEST_EXTRA[@]} > 0)); then
  printf "%-12s %s\n" "dtest args:" "${DTEST_EXTRA[*]}"
fi
echo

show_run_lines=1
TRANSIENT_PROGRESS=0
if [[ -n "${COMPARE:-}" && "$DETAILS" != "1" ]]; then
  show_run_lines=0
  if [[ -t 1 ]]; then
    TRANSIENT_PROGRESS=1
  fi
fi

if [[ "$DRY_RUN" != "1" && ( "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ) ]]; then
  print_run_table_header
fi

total_runs=$(( ${#SOLVERS[@]} * ${#FILES[@]} * num_bins * REPEATS ))
run_no=0

for solver in "${SOLVERS[@]}"; do
  for file in "${FILES[@]}"; do
    hands="$HANDS_DIR/$file"

    for (( rep = 1; rep <= REPEATS; rep++ )); do
      if [[ "$REPEATS" -gt 1 ]]; then
        run_label="${rep}/${REPEATS}"
      else
        run_label="1/1"
      fi

      for pair in "${BIN_PAIRS[@]}"; do
        ver="${pair%%:*}"
        bin="${pair#*:}"
        run_no=$((run_no + 1))

        if [[ "$DRY_RUN" == "1" ]]; then
          run_dtest "$bin" "$solver" "$hands"
          continue
        fi

        read -r user sys avg ratio < <(run_dtest "$bin" "$solver" "$hands")

        if [[ "$show_run_lines" == "1" || "$TRANSIENT_PROGRESS" == "1" ]]; then
          print_run_row "$solver" "$file" "$ver" "$user" "$sys" "$avg" "$ratio" "$run_label"
        fi

        printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \
          "$solver" "$file" "$ver" "$rep" "$user" "$sys" "$avg" "$ratio" \
          >>"$RESULTS"
      done
    done
  done
done

clear_transient_progress

if [[ -n "${COMPARE:-}" && "$DRY_RUN" != "1" ]]; then
  echo
  echo "Summary (branch vs compare, avg user ms)"
  echo "=============================================================================="
  printf "%-6s %-13s %12s %12s %10s %-15s\n" \
    "solver" "file" "compare_avg" "branch_avg" "cmp/branch" "note"
  printf "%-6s %-13s %12s %12s %10s %-15s\n" \
    "------" "-------------" "------------" "------------" "----------" "---------------"

  awk -F'\t' -v files="${FILES[*]}" -v epsilon_pct="$EPSILON" '
    function within_epsilon(a, b,    eps, hi, lo) {
      eps = epsilon_pct / 100
      if (a > b) { hi = a; lo = b } else { hi = b; lo = a }
      return (hi <= 0 || (hi - lo) / hi <= eps)
    }
    {
      base = $1 SUBSEP $2
      if ($3 == "compare") {
        s2[base] += $7
        c2[base]++
      } else if ($3 == "branch") {
        s1[base] += $7
        c1[base]++
      }
    }
    END {
      split("solve calc", solvers, " ")
      nfiles = split(files, filearr, " ")

      for (si = 1; si <= 2; si++) {
        for (fi = 1; fi <= nfiles; fi++) {
          base = solvers[si] SUBSEP filearr[fi]
          if (!(base in c2) || !(base in c1)) continue
          # Every member of s1, c1, s2, and c2 should be positive.
          # If not, it will be due to rounding to zero. To fix, update
          # TestTimer.cpp to accumulate microseconds rather than milliseconds. 
          u2 = s2[base] / c2[base]
          u1 = s1[base] / c1[base]
          cmp_branch = u2 / u1
          if (within_epsilon(u1, u2)) {
            note = "equal"
          } else if (cmp_branch >= 1) {
            note = "branch faster"
          } else {
            note = "compare faster"
          }
          sp = sprintf("%9.2fx", cmp_branch)
          printf "%-6s %-13s %12.2f %12.2f %10s %-15s\n",
            solvers[si], filearr[fi], u2, u1, sp, note
        }
      }
    }
  ' "$RESULTS"
fi

echo
if [[ "$DRY_RUN" == "1" ]]; then
  echo "DRY_RUN: $total_runs dtest invocations (not run)."
else
  echo "Completed $run_no runs ($total_runs expected)."
fi