feat(math): add Latin squares domain with check and transversal search (#1887) #4194
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| types: [opened, synchronize, reopened, labeled, unlabeled] | |
| merge_group: | |
| types: [checks_requested] | |
| schedule: | |
| - cron: "41 4 * * 3" | |
| workflow_dispatch: | |
| concurrency: | |
| group: benchmarks-${{ github.workflow }}-${{ github.ref }} | |
| # A newer PR commit supersedes stale evidence; Benchmark Validation still | |
| # runs after dependency cancellation and rejects missing evidence. | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| actions: read | |
| contents: read | |
| jobs: | |
| plan: | |
| name: Plan Benchmarks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 8 | |
| outputs: | |
| run-benchmark-check: ${{ steps.plan.outputs.run-benchmark-check }} | |
| run-benchmark-record-schema: ${{ steps.plan.outputs.run-benchmark-record-schema }} | |
| run-benchmark-inventory: ${{ steps.plan.outputs.run-benchmark-inventory }} | |
| run-benchmark-host-validation: ${{ steps.plan.outputs.run-benchmark-host-validation }} | |
| benchmark-host-validation-matrix: ${{ steps.plan.outputs.benchmark-host-validation-matrix }} | |
| run-benchmark-oracle: ${{ steps.plan.outputs.run-benchmark-oracle }} | |
| benchmark-oracle-matrix: ${{ steps.plan.outputs.benchmark-oracle-matrix }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| with: | |
| version: "0.11.28" | |
| - name: Restore benchmark timing history | |
| uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: .benchmark-timings.json | |
| key: benchmark-timings-unmatched | |
| restore-keys: benchmark-timings- | |
| - id: plan | |
| name: Classify benchmark changes | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| BASE_SHA: >- | |
| ${{ github.event.pull_request.base.sha || | |
| github.event.merge_group.base_sha || | |
| github.event.before }} | |
| HEAD_SHA: >- | |
| ${{ github.event.pull_request.head.sha || | |
| github.event.merge_group.head_sha || | |
| github.sha }} | |
| FORCE_FULL: ${{ contains(github.event.pull_request.labels.*.name, 'ci:benchmark-full') }} | |
| run: | | |
| set -eu | |
| planner_args=(--event "$EVENT_NAME" --head "$HEAD_SHA") | |
| changed_paths=() | |
| case "$EVENT_NAME" in | |
| schedule|workflow_dispatch) | |
| planner_args=(--event "$EVENT_NAME" --force-full --head "$HEAD_SHA") | |
| ;; | |
| pull_request|merge_group|push) | |
| if [ "$FORCE_FULL" = "true" ] || [ -z "$BASE_SHA" ] || [[ "$BASE_SHA" =~ ^0+$ ]]; then | |
| planner_args=(--event "$EVENT_NAME" --force-full --head "$HEAD_SHA") | |
| else | |
| planner_args=(--event "$EVENT_NAME" --base "$BASE_SHA" --head "$HEAD_SHA") | |
| mapfile -d '' changed_paths < <( | |
| git diff --name-only -z "$BASE_SHA" "$HEAD_SHA" | |
| ) | |
| fi | |
| ;; | |
| *) exit 2 ;; | |
| esac | |
| if [ -f .benchmark-timings.json ]; then | |
| planner_args+=(--timings-file .benchmark-timings.json) | |
| fi | |
| plan_dir="$RUNNER_TEMP/jacobian-benchmark-plan" | |
| mkdir -p "$plan_dir" | |
| uvx --from harbor==0.20.0 --with tomli-w==1.2.0 \ | |
| python .github/scripts/plan-benchmarks \ | |
| "${planner_args[@]}" --output "$plan_dir/plan.json" \ | |
| --github-output "$GITHUB_OUTPUT" -- "${changed_paths[@]}" | |
| printf '%s\n' "${changed_paths[@]}" > "$plan_dir/changed-paths.txt" | |
| { | |
| echo "## Benchmark plan" | |
| echo | |
| echo '```json' | |
| cat "$plan_dir/plan.json" | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload benchmark plan | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-plan | |
| path: ${{ runner.temp }}/jacobian-benchmark-plan/plan.json | |
| retention-days: 14 | |
| - name: Prepare benchmark pytest timing hint | |
| if: steps.plan.outputs.run-benchmark-host-validation == 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| .github/scripts/manage-test-timings prepare \ | |
| --suite benchmark \ | |
| --output .ci/benchmark-test-durations.json | |
| - name: Upload benchmark pytest timing hint | |
| if: steps.plan.outputs.run-benchmark-host-validation == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-test-durations-input | |
| path: .ci/benchmark-test-durations.json | |
| include-hidden-files: true | |
| retention-days: 1 | |
| static: | |
| name: Benchmark Static Quality | |
| needs: plan | |
| if: needs.plan.outputs.run-benchmark-check == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| with: | |
| version: "0.11.28" | |
| - name: Check benchmark Python, verifier, and validation code | |
| run: uv run --locked python tools/check_benchmark_static.py | |
| contracts: | |
| name: Benchmark Contracts, Adapters, Records & Digests | |
| needs: plan | |
| if: needs.plan.outputs.run-benchmark-record-schema == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| with: | |
| version: "0.11.28" | |
| - name: Validate benchmark contracts, adapters, schemas, records, and digests | |
| run: make harbor-contracts harbor-adapter-checks | |
| host_validation: | |
| name: Host Verifiers (${{ matrix.name }}) | |
| needs: plan | |
| if: needs.plan.outputs.run-benchmark-host-validation == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| include: ${{ fromJSON(needs.plan.outputs.benchmark-host-validation-matrix) }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| with: | |
| version: "0.11.28" | |
| - name: Download benchmark timing hint | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: benchmark-test-durations-input | |
| path: .ci | |
| - name: Download benchmark plan | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: benchmark-plan | |
| path: ${{ runner.temp }}/benchmark-plan | |
| - name: Run selected host-side verifier regressions | |
| env: | |
| HOST_ENTRY: ${{ toJSON(matrix) }} | |
| run: | | |
| uv run --locked python -m benchmarks.tooling.host_validation run-entry \ | |
| --entry-json "$HOST_ENTRY" \ | |
| --plan "${{ runner.temp }}/benchmark-plan/plan.json" \ | |
| --execution-sha "${{ github.sha }}" \ | |
| --timings .ci/benchmark-test-durations.json \ | |
| --durations-output "${{ runner.temp }}/benchmark-host-timing/durations/benchmark-test-durations.json" \ | |
| --receipt-dir "${{ runner.temp }}/benchmark-host-timing" \ | |
| --total-workers 8 --max-parallel 4 --store-durations | |
| - name: Upload host timing receipt | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-host-timing-${{ matrix.name }} | |
| path: ${{ runner.temp }}/benchmark-host-timing/**/pytest-receipt.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Upload benchmark shard durations | |
| if: matrix.splits > 0 && !cancelled() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| continue-on-error: true | |
| with: | |
| name: benchmark-duration-shard-${{ matrix.group }} | |
| path: ${{ runner.temp }}/benchmark-host-timing/durations/benchmark-test-durations.json | |
| include-hidden-files: true | |
| retention-days: 1 | |
| inventory: | |
| name: Benchmark Inventory Validation | |
| needs: plan | |
| if: needs.plan.outputs.run-benchmark-inventory == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 25 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| with: | |
| version: "0.11.28" | |
| - name: Build and validate the content-bound benchmark inventory | |
| run: make benchmark-inventory | |
| oracle: | |
| name: Benchmark Oracle (${{ matrix.dataset }}/${{ matrix.shard }}) | |
| needs: [plan, static, contracts] | |
| if: >- | |
| ${{ always() && | |
| needs.plan.result == 'success' && | |
| needs.static.result == 'success' && | |
| needs.plan.outputs.run-benchmark-oracle == 'true' && | |
| needs.contracts.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| include: ${{ fromJSON(needs.plan.outputs.benchmark-oracle-matrix) }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| with: | |
| version: "0.11.28" | |
| - name: Run exact task Oracle | |
| env: | |
| DATASET: ${{ matrix.dataset }} | |
| TASKS: ${{ join(matrix.tasks, ' ') }} | |
| run: make harbor-oracle-task DATASET="$DATASET" TASKS="$TASKS" | |
| - name: Upload Oracle evidence | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-oracle-${{ matrix.dataset }}-${{ matrix.shard }} | |
| path: | | |
| benchmarks/results/${{ matrix.dataset }}-oracle | |
| benchmarks/results/${{ matrix.dataset }}-oracle/jacobian-augmented-task-digests.*.json | |
| benchmarks/results/${{ matrix.dataset }}-oracle/**/oracle-evidence.json | |
| if-no-files-found: ignore | |
| retention-days: ${{ github.event_name == 'pull_request' && 14 || 90 }} | |
| validation: | |
| name: Benchmark Validation | |
| # This required aggregate must run after cancellation so the validator can | |
| # turn cancelled dependencies into a failing check instead of a skipped, | |
| # branch-protection-passing job. | |
| if: ${{ always() }} | |
| needs: [plan, static, contracts, host_validation, inventory, oracle] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - name: Download benchmark plan | |
| if: needs.plan.result == 'success' | |
| continue-on-error: true | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: benchmark-plan | |
| path: ${{ runner.temp }}/benchmark-validation/plan | |
| - name: Download host shard receipts | |
| if: needs.plan.outputs.run-benchmark-host-validation == 'true' | |
| continue-on-error: true | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: benchmark-host-timing-* | |
| path: ${{ runner.temp }}/benchmark-validation/host | |
| - name: Download host timing input | |
| if: needs.plan.outputs.run-benchmark-host-validation == 'true' | |
| continue-on-error: true | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: benchmark-test-durations-input | |
| path: ${{ runner.temp }}/benchmark-validation/timing | |
| - name: Require planned benchmark validation | |
| env: | |
| PLAN_RESULT: ${{ needs.plan.result }} | |
| PLAN_CHECK: ${{ needs.plan.outputs.run-benchmark-check }} | |
| RECORD_SCHEMA_FLAG: ${{ needs.plan.outputs.run-benchmark-record-schema }} | |
| INVENTORY_FLAG: ${{ needs.plan.outputs.run-benchmark-inventory }} | |
| HOST_VALIDATION_FLAG: ${{ needs.plan.outputs.run-benchmark-host-validation }} | |
| ORACLE_FLAG: ${{ needs.plan.outputs.run-benchmark-oracle }} | |
| STATIC_RESULT: ${{ needs.static.result }} | |
| CONTRACTS_RESULT: ${{ needs.contracts.result }} | |
| HOST_VALIDATION_RESULT: ${{ needs.host_validation.result }} | |
| INVENTORY_RESULT: ${{ needs.inventory.result }} | |
| ORACLE_RESULT: ${{ needs.oracle.result }} | |
| run: | | |
| python -m benchmarks.tooling.benchmark_validation \ | |
| --plan-result "$PLAN_RESULT" \ | |
| --plan "${{ runner.temp }}/benchmark-validation/plan/plan.json" \ | |
| --execution-sha "${{ github.sha }}" \ | |
| --receipt-root "${{ runner.temp }}/benchmark-validation/host" \ | |
| --timings "${{ runner.temp }}/benchmark-validation/timing/benchmark-test-durations.json" \ | |
| --lane "static:${PLAN_CHECK:-false}:${STATIC_RESULT:-cancelled}" \ | |
| --lane "contracts:${RECORD_SCHEMA_FLAG:-false}:${CONTRACTS_RESULT:-cancelled}" \ | |
| --lane "host-validation:${HOST_VALIDATION_FLAG:-false}:${HOST_VALIDATION_RESULT:-cancelled}" \ | |
| --lane "inventory:${INVENTORY_FLAG:-false}:${INVENTORY_RESULT:-cancelled}" \ | |
| --lane "oracle:${ORACLE_FLAG:-false}:${ORACLE_RESULT:-cancelled}" | |
| timings: | |
| name: Publish Benchmark Timings | |
| needs: [plan, host_validation, oracle, validation] | |
| if: >- | |
| ${{ always() && | |
| needs.plan.outputs.run-benchmark-host-validation == 'true' && | |
| needs.host_validation.result == 'success' && | |
| needs.validation.result == 'success' && | |
| (needs.plan.outputs.run-benchmark-oracle != 'true' || | |
| needs.oracle.result == 'success') }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 8 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| with: | |
| version: "0.11.28" | |
| - name: Restore previous timing history | |
| uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: .benchmark-timings.json | |
| key: benchmark-timings-unmatched | |
| restore-keys: benchmark-timings- | |
| - name: Download host timing receipts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: benchmark-host-timing-* | |
| path: timing-artifacts | |
| - name: Download benchmark shard durations | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| continue-on-error: true | |
| with: | |
| pattern: benchmark-duration-shard-* | |
| path: .ci/shards | |
| - name: Download Oracle artifacts | |
| if: needs.plan.outputs.run-benchmark-oracle == 'true' | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: benchmark-oracle-* | |
| path: timing-artifacts | |
| - name: Collect and compare shard timings | |
| env: | |
| ORACLE_MATRIX: ${{ needs.plan.outputs.benchmark-oracle-matrix }} | |
| HOST_MATRIX: ${{ needs.plan.outputs.benchmark-host-validation-matrix }} | |
| run: | | |
| uv run --locked python -m benchmarks.tooling.benchmark_timings \ | |
| --root timing-artifacts \ | |
| --output .benchmark-timings.json \ | |
| --previous .benchmark-timings.json \ | |
| --oracle-matrix-json "$ORACLE_MATRIX" \ | |
| --host-matrix-json "$HOST_MATRIX" \ | |
| --report-output benchmark-timing-report.json \ | |
| --summary-output "$GITHUB_STEP_SUMMARY" | |
| - id: benchmark-durations | |
| name: Merge complete full-suite pytest timings | |
| run: | | |
| set -eu | |
| inputs=() | |
| for shard in 1 2 3 4; do | |
| path=".ci/shards/benchmark-duration-shard-${shard}/benchmark-test-durations.json" | |
| if [ ! -f "$path" ]; then | |
| echo "complete=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| inputs+=(--input "$path") | |
| done | |
| .github/scripts/manage-test-timings merge \ | |
| --suite benchmark \ | |
| "${inputs[@]}" \ | |
| --output .ci/benchmark-test-durations.json \ | |
| --source-sha "${{ github.sha }}" \ | |
| --python-version "3.12" | |
| echo "complete=true" >> "$GITHUB_OUTPUT" | |
| - name: Upload timing evidence | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-timings-${{ github.sha }} | |
| path: | | |
| .benchmark-timings.json | |
| benchmark-timing-report.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Publish benchmark pytest timing hint | |
| if: steps.benchmark-durations.outputs.complete == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-test-durations | |
| path: .ci/benchmark-test-durations.json | |
| include-hidden-files: true | |
| retention-days: 90 | |
| - name: Save timing history for future main-branch plans | |
| if: github.ref == 'refs/heads/main' | |
| uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: .benchmark-timings.json | |
| key: benchmark-timings-${{ github.run_id }} |