chg: add benchmark tool

Author: nicokant

.github/workflows/benchmark.yml

@@ -0,0 +1,70 @@
+name: Benchmark
+
+on:
+  pull_request:
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6.4.3
+
+      - name: Install dependencies (PR branch)
+        run: uv sync --frozen
+
+      - name: Benchmark current (PR branch)
+        run: >
+          uv run pytest
+          src/gps_logger_parser/tests/test_benchmark.py
+          --benchmark-json=bench_current.json
+          --benchmark-min-rounds=5
+          -q
+
+      - name: Save current benchmark
+        run: cp bench_current.json /tmp/bench_current.json
+
+      - name: Checkout base branch
+        run: git checkout ${{ github.event.pull_request.base.sha }}
+
+      - name: Install dependencies (base branch)
+        run: uv sync
+
+      - name: Benchmark baseline (base branch)
+        id: baseline
+        run: >
+          uv run pytest
+          src/gps_logger_parser/tests/test_benchmark.py
+          --benchmark-json=bench_baseline.json
+          --benchmark-min-rounds=5
+          -q
+        continue-on-error: true
+
+      - name: Checkout PR branch again
+        run: git checkout ${{ github.event.pull_request.head.sha }}
+
+      - name: Restore current benchmark
+        run: cp /tmp/bench_current.json bench_current.json
+
+      - name: Generate comparison report
+        run: |
+          if [ -f bench_baseline.json ]; then
+            uv run python scripts/benchmark_compare.py bench_baseline.json bench_current.json > benchmark_report.md
+          else
+            echo "No baseline benchmark available (benchmark test may not exist on base branch)." > benchmark_report.md
+          fi
+
+      - name: Post benchmark comment
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body-path: benchmark_report.md
+          comment-tag: benchmark-report

pyproject.toml

@@ -12,7 +12,9 @@ dev = [
   "mkdocs-material>=9.7.1",
   "mkdocs-typer2>=0.1.6",
   "ty>=0.0.3",
-  "coverage>=7.13.5"
+  "coverage>=7.13.5",
+  "pytest-benchmark>=5.2.3",
+  "pytest-memray>=1.8.0"
 ]
 
 [project]

scripts/benchmark_compare.py

@@ -0,0 +1,136 @@
+"""Compare two pytest-benchmark JSON result files and output a Markdown report.
+
+Usage:
+    python scripts/benchmark_compare.py <baseline.json> <current.json>
+
+The report is printed to stdout and is suitable for posting as a GitHub PR
+comment.  The script always exits 0 — it is informational only.
+"""
+
+import json
+import pathlib
+import sys
+
+REGRESSION_THRESHOLD_PCT = 20  # warn marker above this percentage
+
+
+def load_benchmarks(path: str) -> dict[str, dict]:
+    """Return a dict mapping test name -> stats from a pytest-benchmark JSON."""
+    with pathlib.Path(path).open() as file:
+        data = json.load(file)
+    return {bench["name"]: bench["stats"] for bench in data["benchmarks"]}
+
+
+def short_name(full_name: str) -> str:
+    """Extract the parametrize ID from a full test name."""
+    start = full_name.find("[")
+    end = full_name.rfind("]")
+    if start != -1 and end != -1:
+        return full_name[start + 1 : end]
+    return full_name
+
+
+def format_table(
+    rows: list[tuple[str, float, float, float]],
+) -> list[str]:
+    """Format comparison rows into a Markdown table.
+
+    Each row is (name, baseline_ms, current_ms, change_pct).
+    """
+    lines = [
+        "| Test | Baseline (ms) | Current (ms) | Change |",
+        "|---|--:|--:|--:|",
+    ]
+    for name, baseline_ms, current_ms, change_pct in rows:
+        warn = " :warning:" if change_pct > REGRESSION_THRESHOLD_PCT else ""
+        lines.append(
+            f"| {name} | {baseline_ms:.2f} | {current_ms:.2f} "
+            f"| {change_pct:+.1f}%{warn} |"
+        )
+    return lines
+
+
+def compare(
+    baseline: dict[str, dict],
+    current: dict[str, dict],
+    name_filter: str,
+) -> tuple[list[tuple[str, float, float, float]], float]:
+    """Compare benchmarks whose names contain *name_filter*.
+
+    Returns (rows, average_change_pct).
+    """
+    rows: list[tuple[str, float, float, float]] = []
+    for name in sorted(baseline):
+        if name_filter not in name:
+            continue
+        if name not in current:
+            continue
+        baseline_ms = baseline[name]["mean"] * 1000
+        current_ms = current[name]["mean"] * 1000
+        change_pct = ((current_ms - baseline_ms) / baseline_ms) * 100
+        rows.append((short_name(name), baseline_ms, current_ms, change_pct))
+
+    average = sum(r[3] for r in rows) / len(rows) if rows else 0.0
+    return rows, average
+
+
+def main() -> None:
+    if len(sys.argv) != 3:
+        print(f"Usage: {sys.argv[0]} <baseline.json> <current.json>", file=sys.stderr)
+        sys.exit(1)
+
+    baseline = load_benchmarks(sys.argv[1])
+    current = load_benchmarks(sys.argv[2])
+
+    output: list[str] = ["## Benchmark Comparison", ""]
+
+    # --- Detection ---
+    detect_rows, detect_avg = compare(baseline, current, "test_bench_detect[")
+    # Exclude harmonize tests that also contain "detect"
+    detect_rows = [r for r in detect_rows if "harmonize" not in r[0]]
+    if detect_rows:
+        # Recalculate average after filtering
+        detect_avg = (
+            sum(r[3] for r in detect_rows) / len(detect_rows) if detect_rows else 0.0
+        )
+        output.append("### Detection (`detect_file`)")
+        output.append("")
+        output.extend(format_table(detect_rows))
+        output.append("")
+        output.append(f"**Average: {detect_avg:+.1f}%**")
+        output.append("")
+
+    # --- Full pipeline ---
+    harmonize_rows, harmonize_avg = compare(
+        baseline, current, "test_bench_detect_and_harmonize["
+    )
+    if harmonize_rows:
+        output.append("### Full Pipeline (`detect_file` + `as_table`)")
+        output.append("")
+        output.extend(format_table(harmonize_rows))
+        output.append("")
+        output.append(f"**Average: {harmonize_avg:+.1f}%**")
+        output.append("")
+
+    # --- Summary ---
+    if not detect_rows and not harmonize_rows:
+        output.append(
+            "No matching benchmarks found in both baseline and current results."
+        )
+    else:
+        regressions = [
+            r for r in (detect_rows + harmonize_rows) if r[3] > REGRESSION_THRESHOLD_PCT
+        ]
+        if regressions:
+            output.append(
+                f":warning: **{len(regressions)} test(s) regressed "
+                f"by more than {REGRESSION_THRESHOLD_PCT}%**"
+            )
+        else:
+            output.append("No significant regressions detected.")
+
+    print("\n".join(output))
+
+
+if __name__ == "__main__":
+    main()

src/gps_logger_parser/tests/test_benchmark.py

@@ -0,0 +1,42 @@
+import pathlib
+
+import pytest
+import yaml
+
+from ..parser import detect_file
+
+TESTS_DATA_PATH = pathlib.Path("tests")
+TEST_CONFIG_PATH = TESTS_DATA_PATH / "config.yaml"
+
+CONFIG = yaml.safe_load(TEST_CONFIG_PATH.open("r"))
+
+test_files = [
+    pytest.param(
+        TESTS_DATA_PATH / "files" / filename,
+        conf,
+        id=filename,
+    )
+    for filename, conf in CONFIG.get("files", {}).items()
+    if conf.get("skip", False) is not True
+    and (TESTS_DATA_PATH / "files" / filename).exists()
+]
+
+
+@pytest.mark.parametrize("path,config", test_files)
+def test_bench_detect(benchmark, path, config):
+    """Benchmark detect_file() — the parser detection loop."""
+    result = benchmark(detect_file, path)
+    assert result.DATATYPE == config["type"]
+
+
+@pytest.mark.parametrize("path,config", test_files)
+def test_bench_detect_and_harmonize(benchmark, path, config):
+    """Benchmark detect_file() + as_table() — full pipeline."""
+
+    def detect_and_harmonize():
+        parser_instance = detect_file(path)
+        return parser_instance.as_table()
+
+    table = benchmark(detect_and_harmonize)
+    assert table
+    assert "_original_data" in table.column_names