benchmark.sh

#!/usr/bin/env bash
set -euo pipefail

SOLVER_NAME=""
SCENARIO_LIMIT=""
EFFORT=""
SAMPLE_TIMEOUT=""
SAMPLE_NUMBERS=""
INCLUDE_ASSIGNABLE=false
DATASET="dataset01"
DEFAULT_SOLVER_NAME="AutoroutingPipelineSolver9_PreloadedTraceGraph"
PIPELINE_ID=""
NETWORKED_COLD_HOT=false

resolve_pipeline_solver_name() {
  case "$1" in
    1) echo "AutoroutingPipeline1_OriginalUnravel" ;;
    2) echo "AutoroutingPipelineSolver2_PortPointPathing" ;;
    3) echo "AutoroutingPipelineSolver3_HgPortPointPathing" ;;
    4) echo "AutoroutingPipelineSolver4" ;;
    5) echo "AutoroutingPipelineSolver5" ;;
    6) echo "AutoroutingPipelineSolver6" ;;
    7) echo "AutoroutingPipelineSolver7_MultiGraph" ;;
    9) echo "AutoroutingPipelineSolver9_PreloadedTraceGraph" ;;
    9net|9NET) echo "AutoroutingPipelineSolver9_Networked" ;;
    10) echo "AutoroutingPipelineSolver10_BgaFanout" ;;
    krt|KRT) echo "KrtAutoroutingPipelineSolver" ;;
    *)
      echo "Unknown pipeline: $1" >&2
      exit 1
      ;;
  esac
}

default_concurrency() {
  getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || echo 4
}

CONCURRENCY_WAS_SET=false
if [ -n "${BENCHMARK_CONCURRENCY:-}" ]; then
  CONCURRENCY="$BENCHMARK_CONCURRENCY"
  CONCURRENCY_WAS_SET=true
else
  CONCURRENCY="$(default_concurrency)"
fi

get_solvers() {
  INCLUDE_ASSIGNABLE="$INCLUDE_ASSIGNABLE" bun --eval '
    import { readFileSync } from "node:fs"
    import { join } from "node:path"

    // Use autorouter-pipelines/index.ts as the source of truth for benchmarkable solvers
    const pipelinesIndex = readFileSync(join(process.cwd(), "lib", "autorouter-pipelines", "index.ts"), "utf8")
    const pipelineNames = new Set()
    for (const match of pipelinesIndex.matchAll(/export\s*\{([\s\S]*?)\}\s*from/g)) {
      const exportEntries = match[1].split(",").map((entry) => entry.trim()).filter(Boolean)
      for (const entry of exportEntries) {
        const localName = entry.split(/\s+as\s+/)[0]?.trim()
        if (localName) pipelineNames.add(localName)
      }
    }

    // Resolve aliases from lib/index.ts
    const libIndex = readFileSync(join(process.cwd(), "lib", "index.ts"), "utf8")
    const solvers = [...pipelineNames].flatMap(name => {
      const aliasMatches = [...libIndex.matchAll(new RegExp(name + "\\s+as\\s+(\\w+)", "g"))].map(match => match[1])
      return [name, ...aliasMatches]
    })
    const uniqueSolvers = [...new Set(solvers)]

    const includeAssignable = process.env.INCLUDE_ASSIGNABLE === "true"
    const filtered = includeAssignable ? uniqueSolvers : uniqueSolvers.filter(name => !name.includes("Assignable"))
    filtered.push("KrtAutoroutingPipelineSolver")

    console.log(filtered.join("\n"))
  ' 2>/dev/null || true
}

print_help() {
  cat <<'EOF'
Usage:
  ./benchmark.sh [solver-name|all] [scenario-limit] [--concurrency N] [--effort N] [--sample-timeout DURATION] [--sample N] [--dataset NAME] [--include-assignable]
  ./benchmark.sh [--solver NAME] [--pipeline ID] [--limit N] [--concurrency N] [--effort N] [--sample-timeout DURATION] [--sample N] [--dataset NAME] [--include-assignable]

Options:
  --solver NAME        Run only one solver (same as first positional arg)
  --pipeline ID        Run a pipeline alias (1-7, 9, 9net, 10, or krt). 9net runs cold then hot against hd-cache2.
  --limit N            Run only first N scenarios (same as second positional arg)
  --scenario-limit N   Backward-compatible alias for --limit
  --concurrency N      Number of Bun workers used per solver, or "auto"
  --effort N           Override scenario effort multiplier
  --sample-timeout D   Override per-sample timeout directly; otherwise timeout is 300s + 60s * effort
  --sample N           Run one 1-based sample number from the dataset order
  --sample-numbers L   Run comma-separated 1-based sample numbers
  --dataset NAME       Dataset to benchmark: 1/dataset01 (default), zdwiel, 5/srj05, 11/srj11, 12/srj12, 13/srj13, 14/srj14, 15/srj15, 16/srj16, 18/srj18, 19/srj19, 20/srj20, 21/srj21, 23/srj23, 24/srj24, 27/srj27, 28/srj28, or 29/srj29
  --include-assignable Include assignable pipelines (excluded by default)
  -h, --help           Show this help

Defaults:
  Running ./benchmark.sh with no parameters benchmarks only AutoroutingPipelineSolver9_PreloadedTraceGraph.
  Use "all" to benchmark every available solver.

Examples:
  ./benchmark.sh
  ./benchmark.sh AutoroutingPipelineSolver9_PreloadedTraceGraph
  ./benchmark.sh all 20 --concurrency auto
  ./benchmark.sh --solver AutoroutingPipelineSolver9_PreloadedTraceGraph --effort 2
  ./benchmark.sh --solver AutoroutingPipelineSolver9_PreloadedTraceGraph --sample-timeout 90s
  ./benchmark.sh --pipeline 10 --dataset 29 --sample 1
  ./benchmark.sh --pipeline 10 --dataset 29 --limit 5
  ./benchmark.sh --sample-numbers 120,139,148 --concurrency 8 --sample-timeout 120s
  ./benchmark.sh --solver AutoroutingPipelineSolver9_PreloadedTraceGraph --scenario-limit 20
  ./benchmark.sh --solver AutoroutingPipelineSolver9_PreloadedTraceGraph --dataset zdwiel --scenario-limit 20
  ./benchmark.sh --pipeline 4
  ./benchmark.sh --pipeline 5
  ./benchmark.sh --pipeline 6
  ./benchmark.sh --pipeline 7
  ./benchmark.sh --pipeline 9
  ./benchmark.sh --pipeline 9net --dataset 18
  ./benchmark.sh --pipeline 10 --dataset 29
  ./benchmark.sh --pipeline krt
  ./benchmark.sh --solver AutoroutingPipelineSolver9_PreloadedTraceGraph --dataset srj05 --scenario-limit 20
  ./benchmark.sh --dataset 11 --scenario-limit 20
  ./benchmark.sh --dataset 12 --scenario-limit 10
  ./benchmark.sh --dataset 13
  ./benchmark.sh --dataset 14
  ./benchmark.sh --dataset 15
  ./benchmark.sh --dataset 16
  ./benchmark.sh --dataset 18
  ./benchmark.sh --dataset 19
  ./benchmark.sh --dataset 20
  ./benchmark.sh --dataset 21
  ./benchmark.sh --dataset 23
  ./benchmark.sh --dataset 24
  ./benchmark.sh --dataset 27
  ./benchmark.sh --dataset 28
  ./benchmark.sh --dataset 29 --pipeline 10
  ./benchmark.sh --include-assignable
EOF

  SOLVERS="$(get_solvers)"
  if [ -n "$SOLVERS" ]; then
    echo ""
    echo "Available solvers:"
    while IFS= read -r solver; do
      [ -n "$solver" ] && echo "  - $solver"
    done <<EOF
$SOLVERS
EOF
  fi
}

if [ "${1:-}" != "" ] && [[ "${1}" != --* ]]; then
  SOLVER_NAME="$1"
  shift
fi

if [ "${1:-}" != "" ] && [[ "${1}" != --* ]]; then
  SCENARIO_LIMIT="$1"
  shift
fi

while [ "$#" -gt 0 ]; do
  case "$1" in
    -h|--help)
      print_help
      exit 0
      ;;
    --solver)
      SOLVER_NAME="${2:-}"
      shift 2
      ;;
    --pipeline)
      PIPELINE_ID="${2:-}"
      shift 2
      ;;
    --limit|--scenario-limit)
      SCENARIO_LIMIT="${2:-}"
      shift 2
      ;;
    --concurrency)
      CONCURRENCY="${2:-}"
      CONCURRENCY_WAS_SET=true
      if [ "$CONCURRENCY" = "auto" ]; then
        CONCURRENCY="$(default_concurrency)"
      fi
      shift 2
      ;;
    --effort)
      EFFORT="${2:-}"
      shift 2
      ;;
    --sample-timeout)
      SAMPLE_TIMEOUT="${2:-}"
      shift 2
      ;;
    --sample|--sample-numbers)
      SAMPLE_NUMBERS="${2:-}"
      shift 2
      ;;
    --dataset)
      DATASET="${2:-}"
      shift 2
      ;;
    --include-assignable)
      INCLUDE_ASSIGNABLE=true
      shift
      ;;
    *)
      echo "Unknown argument: $1"
      echo "Run ./benchmark.sh --help for usage"
      exit 1
      ;;
  esac
done

if [ -n "$PIPELINE_ID" ]; then
  SOLVER_NAME="$(resolve_pipeline_solver_name "$PIPELINE_ID")"
  case "$PIPELINE_ID" in
    9net|9NET) NETWORKED_COLD_HOT=true ;;
  esac
fi

if [ "$NETWORKED_COLD_HOT" = true ]; then
  if [ -n "$EFFORT" ] && [ "$EFFORT" != "1" ]; then
    echo "Pipeline 9net only supports --effort 1" >&2
    exit 1
  fi
  if [ -z "${HD_CACHE2_CACHE_VERSION:-}" ]; then
    echo "Pipeline 9net requires HD_CACHE2_CACHE_VERSION" >&2
    exit 1
  fi
fi

if [ "$CONCURRENCY_WAS_SET" = false ] && [ "$SOLVER_NAME" = "AutoroutingPipelineSolver7_MultiGraph" ]; then
  case "$DATASET" in
    18|srj18|dataset18|dataset-srj18)
      if [ "$CONCURRENCY" -gt 4 ]; then
        CONCURRENCY=4
      fi
      ;;
  esac
fi

CMD=(bun "scripts/benchmark/index.ts" "--concurrency" "$CONCURRENCY")

if [ "$NETWORKED_COLD_HOT" = true ]; then
  CMD+=("--networked-cold-hot")
fi

if [ -z "$SOLVER_NAME" ]; then
  SOLVER_NAME="$DEFAULT_SOLVER_NAME"
fi

if [ -n "$SOLVER_NAME" ] && [ "$SOLVER_NAME" != "_" ] && [ "$SOLVER_NAME" != "all" ]; then
  CMD+=("--solver" "$SOLVER_NAME")
fi

if [ -n "$SCENARIO_LIMIT" ]; then
  CMD+=("--scenario-limit" "$SCENARIO_LIMIT")
fi

if [ -n "$EFFORT" ]; then
  CMD+=("--effort" "$EFFORT")
fi

if [ -n "$SAMPLE_TIMEOUT" ]; then
  CMD+=("--sample-timeout" "$SAMPLE_TIMEOUT")
fi

if [ -n "$SAMPLE_NUMBERS" ]; then
  CMD+=("--sample-numbers" "$SAMPLE_NUMBERS")
fi

if [ -n "$DATASET" ]; then
  CMD+=("--dataset" "$DATASET")
fi

if [ "$INCLUDE_ASSIGNABLE" != true ]; then
  CMD+=("--exclude-assignable")
fi

"${CMD[@]}"