benchmarks.yml

name: Benchmarks

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
    types: [opened, synchronize, reopened, labeled, unlabeled]
  merge_group:
    types: [checks_requested]
  schedule:
    - cron: "41 4 * * 3"
  workflow_dispatch:

concurrency:
  group: benchmarks-${{ github.workflow }}-${{ github.ref }}
  # A newer PR commit supersedes stale evidence; Benchmark Validation still
  # runs after dependency cancellation and rejects missing evidence.
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
  actions: read
  contents: read

jobs:
  plan:
    name: Plan Benchmarks
    runs-on: ubuntu-latest
    timeout-minutes: 8
    outputs:
      run-benchmark-check: ${{ steps.plan.outputs.run-benchmark-check }}
      run-benchmark-record-schema: ${{ steps.plan.outputs.run-benchmark-record-schema }}
      run-benchmark-inventory: ${{ steps.plan.outputs.run-benchmark-inventory }}
      run-benchmark-host-validation: ${{ steps.plan.outputs.run-benchmark-host-validation }}
      benchmark-host-validation-matrix: ${{ steps.plan.outputs.benchmark-host-validation-matrix }}
      run-benchmark-oracle: ${{ steps.plan.outputs.run-benchmark-oracle }}
      benchmark-oracle-matrix: ${{ steps.plan.outputs.benchmark-oracle-matrix }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          version: "0.11.28"
      - name: Restore benchmark timing history
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: .benchmark-timings.json
          key: benchmark-timings-unmatched
          restore-keys: benchmark-timings-
      - id: plan
        name: Classify benchmark changes
        env:
          EVENT_NAME: ${{ github.event_name }}
          BASE_SHA: >-
            ${{ github.event.pull_request.base.sha ||
                github.event.merge_group.base_sha ||
                github.event.before }}
          HEAD_SHA: >-
            ${{ github.event.pull_request.head.sha ||
                github.event.merge_group.head_sha ||
                github.sha }}
          FORCE_FULL: ${{ contains(github.event.pull_request.labels.*.name, 'ci:benchmark-full') }}
        run: |
          set -eu
          planner_args=(--event "$EVENT_NAME" --head "$HEAD_SHA")
          changed_paths=()
          case "$EVENT_NAME" in
            schedule|workflow_dispatch)
              planner_args=(--event "$EVENT_NAME" --force-full --head "$HEAD_SHA")
              ;;
            pull_request|merge_group|push)
              if [ "$FORCE_FULL" = "true" ] || [ -z "$BASE_SHA" ] || [[ "$BASE_SHA" =~ ^0+$ ]]; then
                planner_args=(--event "$EVENT_NAME" --force-full --head "$HEAD_SHA")
              else
                planner_args=(--event "$EVENT_NAME" --base "$BASE_SHA" --head "$HEAD_SHA")
                mapfile -d '' changed_paths < <(
                  git diff --name-only -z "$BASE_SHA" "$HEAD_SHA"
                )
              fi
              ;;
            *) exit 2 ;;
          esac
          if [ -f .benchmark-timings.json ]; then
            planner_args+=(--timings-file .benchmark-timings.json)
          fi
          plan_dir="$RUNNER_TEMP/jacobian-benchmark-plan"
          mkdir -p "$plan_dir"
          uvx --from harbor==0.20.0 --with tomli-w==1.2.0 \
            python .github/scripts/plan-benchmarks \
            "${planner_args[@]}" --output "$plan_dir/plan.json" \
            --github-output "$GITHUB_OUTPUT" -- "${changed_paths[@]}"
          printf '%s\n' "${changed_paths[@]}" > "$plan_dir/changed-paths.txt"
          {
            echo "## Benchmark plan"
            echo
            echo '```json'
            cat "$plan_dir/plan.json"
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"
      - name: Upload benchmark plan
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: benchmark-plan
          path: ${{ runner.temp }}/jacobian-benchmark-plan/plan.json
          retention-days: 14
      - name: Prepare benchmark pytest timing hint
        if: steps.plan.outputs.run-benchmark-host-validation == 'true'
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          .github/scripts/manage-test-timings prepare \
            --suite benchmark \
            --output .ci/benchmark-test-durations.json
      - name: Upload benchmark pytest timing hint
        if: steps.plan.outputs.run-benchmark-host-validation == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: benchmark-test-durations-input
          path: .ci/benchmark-test-durations.json
          include-hidden-files: true
          retention-days: 1

  static:
    name: Benchmark Static Quality
    needs: plan
    if: needs.plan.outputs.run-benchmark-check == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          version: "0.11.28"
      - name: Check benchmark Python, verifier, and validation code
        run: uv run --locked python tools/check_benchmark_static.py

  contracts:
    name: Benchmark Contracts, Adapters, Records & Digests
    needs: plan
    if: needs.plan.outputs.run-benchmark-record-schema == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          version: "0.11.28"
      - name: Validate benchmark contracts, adapters, schemas, records, and digests
        run: make harbor-contracts harbor-adapter-checks

  host_validation:
    name: Host Verifiers (${{ matrix.name }})
    needs: plan
    if: needs.plan.outputs.run-benchmark-host-validation == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        include: ${{ fromJSON(needs.plan.outputs.benchmark-host-validation-matrix) }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          version: "0.11.28"
      - name: Download benchmark timing hint
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: benchmark-test-durations-input
          path: .ci
      - name: Download benchmark plan
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: benchmark-plan
          path: ${{ runner.temp }}/benchmark-plan
      - name: Run selected host-side verifier regressions
        env:
          HOST_ENTRY: ${{ toJSON(matrix) }}
        run: |
          uv run --locked python -m benchmarks.tooling.host_validation run-entry \
            --entry-json "$HOST_ENTRY" \
            --plan "${{ runner.temp }}/benchmark-plan/plan.json" \
            --execution-sha "${{ github.sha }}" \
            --timings .ci/benchmark-test-durations.json \
            --durations-output "${{ runner.temp }}/benchmark-host-timing/durations/benchmark-test-durations.json" \
            --receipt-dir "${{ runner.temp }}/benchmark-host-timing" \
            --total-workers 8 --max-parallel 4 --store-durations
      - name: Upload host timing receipt
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: benchmark-host-timing-${{ matrix.name }}
          path: ${{ runner.temp }}/benchmark-host-timing/**/pytest-receipt.json
          if-no-files-found: error
          retention-days: 90
      - name: Upload benchmark shard durations
        if: matrix.splits > 0 && !cancelled()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        continue-on-error: true
        with:
          name: benchmark-duration-shard-${{ matrix.group }}
          path: ${{ runner.temp }}/benchmark-host-timing/durations/benchmark-test-durations.json
          include-hidden-files: true
          retention-days: 1

  inventory:
    name: Benchmark Inventory Validation
    needs: plan
    if: needs.plan.outputs.run-benchmark-inventory == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 25
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          version: "0.11.28"
      - name: Build and validate the content-bound benchmark inventory
        run: make benchmark-inventory

  oracle:
    name: Benchmark Oracle (${{ matrix.dataset }}/${{ matrix.shard }})
    needs: [plan, static, contracts]
    if: >-
      ${{ always() &&
      needs.plan.result == 'success' &&
      needs.static.result == 'success' &&
      needs.plan.outputs.run-benchmark-oracle == 'true' &&
      needs.contracts.result == 'success' }}
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        include: ${{ fromJSON(needs.plan.outputs.benchmark-oracle-matrix) }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          version: "0.11.28"
      - name: Run exact task Oracle
        env:
          DATASET: ${{ matrix.dataset }}
          TASKS: ${{ join(matrix.tasks, ' ') }}
        run: make harbor-oracle-task DATASET="$DATASET" TASKS="$TASKS"
      - name: Upload Oracle evidence
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: benchmark-oracle-${{ matrix.dataset }}-${{ matrix.shard }}
          path: |
            benchmarks/results/${{ matrix.dataset }}-oracle
            benchmarks/results/${{ matrix.dataset }}-oracle/jacobian-augmented-task-digests.*.json
            benchmarks/results/${{ matrix.dataset }}-oracle/**/oracle-evidence.json
          if-no-files-found: ignore
          retention-days: ${{ github.event_name == 'pull_request' && 14 || 90 }}

  validation:
    name: Benchmark Validation
    # This required aggregate must run after cancellation so the validator can
    # turn cancelled dependencies into a failing check instead of a skipped,
    # branch-protection-passing job.
    if: ${{ always() }}
    needs: [plan, static, contracts, host_validation, inventory, oracle]
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - name: Download benchmark plan
        if: needs.plan.result == 'success'
        continue-on-error: true
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: benchmark-plan
          path: ${{ runner.temp }}/benchmark-validation/plan
      - name: Download host shard receipts
        if: needs.plan.outputs.run-benchmark-host-validation == 'true'
        continue-on-error: true
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: benchmark-host-timing-*
          path: ${{ runner.temp }}/benchmark-validation/host
      - name: Download host timing input
        if: needs.plan.outputs.run-benchmark-host-validation == 'true'
        continue-on-error: true
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: benchmark-test-durations-input
          path: ${{ runner.temp }}/benchmark-validation/timing
      - name: Require planned benchmark validation
        env:
          PLAN_RESULT: ${{ needs.plan.result }}
          PLAN_CHECK: ${{ needs.plan.outputs.run-benchmark-check }}
          RECORD_SCHEMA_FLAG: ${{ needs.plan.outputs.run-benchmark-record-schema }}
          INVENTORY_FLAG: ${{ needs.plan.outputs.run-benchmark-inventory }}
          HOST_VALIDATION_FLAG: ${{ needs.plan.outputs.run-benchmark-host-validation }}
          ORACLE_FLAG: ${{ needs.plan.outputs.run-benchmark-oracle }}
          STATIC_RESULT: ${{ needs.static.result }}
          CONTRACTS_RESULT: ${{ needs.contracts.result }}
          HOST_VALIDATION_RESULT: ${{ needs.host_validation.result }}
          INVENTORY_RESULT: ${{ needs.inventory.result }}
          ORACLE_RESULT: ${{ needs.oracle.result }}
        run: |
          python -m benchmarks.tooling.benchmark_validation \
            --plan-result "$PLAN_RESULT" \
            --plan "${{ runner.temp }}/benchmark-validation/plan/plan.json" \
            --execution-sha "${{ github.sha }}" \
            --receipt-root "${{ runner.temp }}/benchmark-validation/host" \
            --timings "${{ runner.temp }}/benchmark-validation/timing/benchmark-test-durations.json" \
            --lane "static:${PLAN_CHECK:-false}:${STATIC_RESULT:-cancelled}" \
            --lane "contracts:${RECORD_SCHEMA_FLAG:-false}:${CONTRACTS_RESULT:-cancelled}" \
            --lane "host-validation:${HOST_VALIDATION_FLAG:-false}:${HOST_VALIDATION_RESULT:-cancelled}" \
            --lane "inventory:${INVENTORY_FLAG:-false}:${INVENTORY_RESULT:-cancelled}" \
            --lane "oracle:${ORACLE_FLAG:-false}:${ORACLE_RESULT:-cancelled}"

  timings:
    name: Publish Benchmark Timings
    needs: [plan, host_validation, oracle, validation]
    if: >-
      ${{ always() &&
      needs.plan.outputs.run-benchmark-host-validation == 'true' &&
      needs.host_validation.result == 'success' &&
      needs.validation.result == 'success' &&
      (needs.plan.outputs.run-benchmark-oracle != 'true' ||
       needs.oracle.result == 'success') }}
    runs-on: ubuntu-latest
    timeout-minutes: 8
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          version: "0.11.28"
      - name: Restore previous timing history
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: .benchmark-timings.json
          key: benchmark-timings-unmatched
          restore-keys: benchmark-timings-
      - name: Download host timing receipts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: benchmark-host-timing-*
          path: timing-artifacts
      - name: Download benchmark shard durations
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        continue-on-error: true
        with:
          pattern: benchmark-duration-shard-*
          path: .ci/shards
      - name: Download Oracle artifacts
        if: needs.plan.outputs.run-benchmark-oracle == 'true'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: benchmark-oracle-*
          path: timing-artifacts
      - name: Collect and compare shard timings
        env:
          ORACLE_MATRIX: ${{ needs.plan.outputs.benchmark-oracle-matrix }}
          HOST_MATRIX: ${{ needs.plan.outputs.benchmark-host-validation-matrix }}
        run: |
          uv run --locked python -m benchmarks.tooling.benchmark_timings \
            --root timing-artifacts \
            --output .benchmark-timings.json \
            --previous .benchmark-timings.json \
            --oracle-matrix-json "$ORACLE_MATRIX" \
            --host-matrix-json "$HOST_MATRIX" \
            --report-output benchmark-timing-report.json \
            --summary-output "$GITHUB_STEP_SUMMARY"
      - id: benchmark-durations
        name: Merge complete full-suite pytest timings
        run: |
          set -eu
          inputs=()
          for shard in 1 2 3 4; do
            path=".ci/shards/benchmark-duration-shard-${shard}/benchmark-test-durations.json"
            if [ ! -f "$path" ]; then
              echo "complete=false" >> "$GITHUB_OUTPUT"
              exit 0
            fi
            inputs+=(--input "$path")
          done
          .github/scripts/manage-test-timings merge \
            --suite benchmark \
            "${inputs[@]}" \
            --output .ci/benchmark-test-durations.json \
            --source-sha "${{ github.sha }}" \
            --python-version "3.12"
          echo "complete=true" >> "$GITHUB_OUTPUT"
      - name: Upload timing evidence
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: benchmark-timings-${{ github.sha }}
          path: |
            .benchmark-timings.json
            benchmark-timing-report.json
          if-no-files-found: error
          retention-days: 90
      - name: Publish benchmark pytest timing hint
        if: steps.benchmark-durations.outputs.complete == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: benchmark-test-durations
          path: .ci/benchmark-test-durations.json
          include-hidden-files: true
          retention-days: 90
      - name: Save timing history for future main-branch plans
        if: github.ref == 'refs/heads/main'
        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          path: .benchmark-timings.json
          key: benchmark-timings-${{ github.run_id }}