try to improve benchmarking

Author: claydugo

.bashrc

@@ -52,6 +52,7 @@ path_prepend "$HOME/.cargo/bin"
 path_prepend "/usr/lib/qt6/bin"
 path_prepend "$HOME/.pixi/bin"
 path_prepend "$BUN_INSTALL/bin"
+path_prepend "$HOME/go/bin"
 path_prepend "$HOME/.local/bin"
 
 if hash nvim 2>/dev/null; then

.github/scripts/benchmark-bash.sh

@@ -3,30 +3,37 @@
 set -eo pipefail
 
 OUTPUT=${1:-benchmark-bash-result.json}
-ITERATIONS=${ITERATIONS:-20}
+WARMUPS=${WARMUPS:-3}
+RUNS=${RUNS:-30}
 
 tmpdir=$(mktemp -d)
 trap 'rm -rf "$tmpdir"' EXIT
 
 hyperfine \
-    --warmup 3 \
-    --runs "$ITERATIONS" \
+    --warmup "$WARMUPS" \
+    --runs "$RUNS" \
     --shell=none \
     --style=basic \
-    --export-csv "$tmpdir/result.csv" \
+    --export-json "$tmpdir/result.json" \
     'bash -i -c exit'
 
-mean_seconds=$(awk -F, 'NR==2 { gsub(/"/, "", $2); print $2 }' "$tmpdir/result.csv")
-mean_ms=$(awk -v m="$mean_seconds" 'BEGIN { printf "%.3f", m * 1000 }')
+read -r median_ms stddev_ms min_ms <<<"$(python3 - "$tmpdir/result.json" <<'PY'
+import json, sys
+with open(sys.argv[1]) as f:
+    r = json.load(f)["results"][0]
+print(f"{r['median']*1000:.3f} {r['stddev']*1000:.3f} {r['min']*1000:.3f}")
+PY
+)"
 
-echo "Mean: ${mean_ms}ms"
+echo "Median: ${median_ms}ms (stddev ${stddev_ms}ms, min ${min_ms}ms)"
 
 cat > "$OUTPUT" <<EOF
 [
   {
     "name": "bash startup time",
     "unit": "ms",
-    "value": $mean_ms
+    "value": $median_ms,
+    "range": "±$stddev_ms"
   }
 ]
 EOF

.github/scripts/benchmark-nvim.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+OUTPUT=${1:-benchmark-result.json}
+WARMUPS=${WARMUPS:-3}
+RUNS=${RUNS:-30}
+
+tmpdir=$(mktemp -d)
+trap 'rm -rf "$tmpdir"' EXIT
+
+hyperfine \
+    --warmup "$WARMUPS" \
+    --runs "$RUNS" \
+    --shell=none \
+    --style=basic \
+    --export-json "$tmpdir/result.json" \
+    'nvim --headless +qa'
+
+read -r median_ms stddev_ms min_ms <<<"$(python3 - "$tmpdir/result.json" <<'PY'
+import json, sys
+with open(sys.argv[1]) as f:
+    r = json.load(f)["results"][0]
+print(f"{r['median']*1000:.3f} {r['stddev']*1000:.3f} {r['min']*1000:.3f}")
+PY
+)"
+
+echo "Median: ${median_ms}ms (stddev ${stddev_ms}ms, min ${min_ms}ms)"
+
+cat > "$OUTPUT" <<EOF
+[
+  {
+    "name": "nvim startup time",
+    "unit": "ms",
+    "value": $median_ms,
+    "range": "±$stddev_ms"
+  }
+]
+EOF
+echo "Wrote $OUTPUT"

.github/scripts/benchmark.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Compare a benchmark result against a rolling-median baseline of recent runs.
+
+History format is compatible with the github-action-benchmark
+``benchmark-data.json`` layout so the file stays easy to inspect by hand.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def median(values: list[float]) -> float:
+    s = sorted(values)
+    n = len(s)
+    return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("current", type=Path, help="Path to current result JSON")
+    p.add_argument("history_dir", type=Path, help="Directory holding benchmark-data.json")
+    p.add_argument("suite", help="Benchmark suite name (history key)")
+    p.add_argument("--record", action="store_true", help="Append current result to history")
+    p.add_argument("--window", type=int, default=10, help="Recent entries to median (default: 10)")
+    p.add_argument("--threshold", type=float, default=1.5, help="Regression ratio (default: 1.5)")
+    p.add_argument("--max-entries", type=int, default=200, help="Cap stored history (default: 200)")
+    args = p.parse_args()
+
+    args.history_dir.mkdir(parents=True, exist_ok=True)
+    history_path = args.history_dir / "benchmark-data.json"
+
+    current = json.loads(args.current.read_text())[0]
+    value = current["value"]
+    unit = current["unit"]
+
+    if history_path.exists():
+        history = json.loads(history_path.read_text())
+    else:
+        history = {"entries": {}}
+
+    entries = history["entries"].setdefault(args.suite, [])
+    recent = entries[-args.window:]
+
+    print(f"Suite: {args.suite}")
+    print(f"Current: {value}{unit}")
+
+    if recent:
+        baseline = median([e["benches"][0]["value"] for e in recent])
+        ratio = value / baseline
+        print(f"Baseline (median of last {len(recent)}): {baseline:.3f}{unit}")
+        print(f"Ratio: {ratio:.2f}x (regression threshold {args.threshold}x)")
+        if ratio > args.threshold:
+            print(
+                f"::warning title=Benchmark regression::{args.suite}: "
+                f"{value:.2f}{unit} is {ratio:.2f}x the median of the last "
+                f"{len(recent)} runs ({baseline:.2f}{unit})"
+            )
+    else:
+        print("Baseline: (no history yet — recording without comparison)")
+
+    if args.record:
+        entries.append({
+            "commit": {"id": os.environ.get("GITHUB_SHA", "")},
+            "date": int(datetime.datetime.now().timestamp() * 1000),
+            "tool": "customSmallerIsBetter",
+            "benches": [{"name": current["name"], "value": value, "unit": unit}],
+        })
+        history["entries"][args.suite] = entries[-args.max_entries:]
+        history_path.write_text(json.dumps(history, indent=2))
+        print(f"Recorded entry ({len(history['entries'][args.suite])} total) to {history_path}")
+    else:
+        print("Not recording (--record not set)")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/compare-benchmark.py

@@ -20,7 +20,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # full history needed to scan past commits
+          fetch-depth: 0 # full history needed to scan past commits
 
       - name: Run gg-shield
         uses: GitGuardian/ggshield-action@v1

.github/workflows/gitguardian.yml

@@ -9,17 +9,19 @@ on:
       - '.bashrc'
       - '.aliases'
       - '.github/workflows/install-dotfiles.yml'
-      - '.github/scripts/benchmark.sh'
+      - '.github/scripts/benchmark-nvim.sh'
       - '.github/scripts/benchmark-bash.sh'
+      - '.github/scripts/compare-benchmark.py'
   pull_request:
     paths:
       - 'scripts/bootstrap.sh'
       - '.config/nvim/**'
       - '.bashrc'
       - '.aliases'
       - '.github/workflows/install-dotfiles.yml'
-      - '.github/scripts/benchmark.sh'
+      - '.github/scripts/benchmark-nvim.sh'
       - '.github/scripts/benchmark-bash.sh'
+      - '.github/scripts/compare-benchmark.py'
   schedule:
     - cron: '0 3 * * 6'
   workflow_dispatch:
@@ -46,7 +48,7 @@ jobs:
       - name: Install prerequisites
         run: |
           apt-get update
-          apt-get install -y git curl sudo tree xz-utils fontconfig gcc make
+          apt-get install -y git curl sudo tree xz-utils fontconfig gcc make python3
 
       - name: Create test user
         run: |
@@ -282,7 +284,7 @@ jobs:
           su - clay -c '
             export PATH="$HOME/.pixi/bin:$PATH"
             cd ~/dotfiles
-            bash .github/scripts/benchmark.sh /tmp/result.json
+            bash .github/scripts/benchmark-nvim.sh /tmp/result.json
             cat /tmp/result.json
           '
 
@@ -303,34 +305,21 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-benchmark-history-
 
-      - name: Compare against benchmark history (alert on regression)
-        uses: benchmark-action/github-action-benchmark@v1
-        with:
-          name: nvim startup (Ubuntu)
-          tool: customSmallerIsBetter
-          output-file-path: /tmp/result.json
-          external-data-json-path: /tmp/cache/benchmark-data.json
-          alert-threshold: '150%'
-          comment-on-alert: true
-          fail-on-alert: false
-          alert-comment-cc-users: '@claydugo'
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Compare bash startup against history (alert on regression)
-        uses: benchmark-action/github-action-benchmark@v1
-        with:
-          name: bash startup (Ubuntu)
-          tool: customSmallerIsBetter
-          output-file-path: /tmp/bash-result.json
-          external-data-json-path: /tmp/cache/benchmark-data.json
-          alert-threshold: '150%'
-          comment-on-alert: true
-          fail-on-alert: false
-          alert-comment-cc-users: '@claydugo'
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Compare against rolling baseline (nvim + bash)
+        env:
+          RECORD: ${{ github.event_name == 'push' && '--record' || '' }}
+          GITHUB_SHA: ${{ github.sha }}
+        run: |
+          chown -R clay:clay /tmp/cache 2>/dev/null || true
+          su - clay -c "
+            export GITHUB_SHA='${GITHUB_SHA}'
+            cd ~/dotfiles
+            python3 .github/scripts/compare-benchmark.py /tmp/result.json /tmp/cache 'nvim startup (Ubuntu)' ${RECORD}
+            python3 .github/scripts/compare-benchmark.py /tmp/bash-result.json /tmp/cache 'bash startup (Ubuntu)' ${RECORD}
+          "
 
       - name: Save benchmark history
-        if: always()
+        if: github.event_name == 'push'
         uses: actions/cache/save@v4
         with:
           path: /tmp/cache
@@ -458,7 +447,7 @@ jobs:
         run: |
           export PATH="$HOME/.pixi/bin:$PATH"
           cd "$HOME/dotfiles"
-          bash .github/scripts/benchmark.sh /tmp/result.json
+          bash .github/scripts/benchmark-nvim.sh /tmp/result.json
           cat /tmp/result.json
 
       - name: Measure bash startup
@@ -476,34 +465,16 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-benchmark-history-
 
-      - name: Compare against benchmark history (alert on regression)
-        uses: benchmark-action/github-action-benchmark@v1
-        with:
-          name: nvim startup (macOS)
-          tool: customSmallerIsBetter
-          output-file-path: /tmp/result.json
-          external-data-json-path: /tmp/cache/benchmark-data.json
-          alert-threshold: '150%'
-          comment-on-alert: true
-          fail-on-alert: false
-          alert-comment-cc-users: '@claydugo'
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Compare bash startup against history (alert on regression)
-        uses: benchmark-action/github-action-benchmark@v1
-        with:
-          name: bash startup (macOS)
-          tool: customSmallerIsBetter
-          output-file-path: /tmp/bash-result.json
-          external-data-json-path: /tmp/cache/benchmark-data.json
-          alert-threshold: '150%'
-          comment-on-alert: true
-          fail-on-alert: false
-          alert-comment-cc-users: '@claydugo'
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Compare against rolling baseline (nvim + bash)
+        env:
+          RECORD: ${{ github.event_name == 'push' && '--record' || '' }}
+        run: |
+          cd "$HOME/dotfiles"
+          python3 .github/scripts/compare-benchmark.py /tmp/result.json /tmp/cache 'nvim startup (macOS)' ${RECORD}
+          python3 .github/scripts/compare-benchmark.py /tmp/bash-result.json /tmp/cache 'bash startup (macOS)' ${RECORD}
 
       - name: Save benchmark history
-        if: always()
+        if: github.event_name == 'push'
         uses: actions/cache/save@v4
         with:
           path: /tmp/cache

.github/workflows/install-dotfiles.yml

@@ -31,6 +31,13 @@ jobs:
           chmod +x selene
           sudo mv selene /usr/local/bin/
 
+      - name: Install yamlfmt
+        run: |
+          YAMLFMT_VERSION=0.21.0
+          curl -fsSL "https://github.com/google/yamlfmt/releases/download/v${YAMLFMT_VERSION}/yamlfmt_${YAMLFMT_VERSION}_Linux_x86_64.tar.gz" \
+            | sudo tar -xz -C /usr/local/bin yamlfmt
+          sudo chmod +x /usr/local/bin/yamlfmt
+
       - name: Discover shell scripts (by shebang or shellcheck directive)
         id: discover
         run: |
@@ -85,3 +92,7 @@ jobs:
       - name: selene
         if: always()
         run: selene .
+
+      - name: yamlfmt --lint
+        if: always()
+        run: yamlfmt -lint

.github/workflows/lint.yml

@@ -26,7 +26,7 @@ jobs:
 
       - name: Commit and push
         if: steps.changes.outputs.changed == 'true'
-        run: |
+        run: |-
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
           git add README.md