eval: add offline trajectory value comparison #1735
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| types: [opened, synchronize, reopened, edited, labeled, unlabeled] | |
| merge_group: | |
| types: [checks_requested] | |
| schedule: | |
| - cron: "41 4 * * 3" | |
| workflow_dispatch: | |
| concurrency: | |
| group: benchmarks-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| actions: read | |
| contents: read | |
| jobs: | |
| plan: | |
| name: Plan Benchmarks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 8 | |
| outputs: | |
| benchmark-plan-version: ${{ steps.plan.outputs.benchmark-plan-version }} | |
| benchmark-plan-event: ${{ steps.plan.outputs.benchmark-plan-event }} | |
| benchmark-plan-base-sha: ${{ steps.plan.outputs.benchmark-plan-base-sha }} | |
| benchmark-plan-head-sha: ${{ steps.plan.outputs.benchmark-plan-head-sha }} | |
| benchmark-planner-digest: ${{ steps.plan.outputs.benchmark-planner-digest }} | |
| benchmark-topology-digest: ${{ steps.plan.outputs.benchmark-topology-digest }} | |
| benchmark-plan-mode: ${{ steps.plan.outputs.benchmark-plan-mode }} | |
| run-benchmark-check: ${{ steps.plan.outputs.run-benchmark-check }} | |
| run-benchmark-record-schema: ${{ steps.plan.outputs.run-benchmark-record-schema }} | |
| run-benchmark-prospective-digest: ${{ steps.plan.outputs.run-benchmark-prospective-digest }} | |
| run-benchmark-inventory: ${{ steps.plan.outputs.run-benchmark-inventory }} | |
| run-benchmark-host-validation: ${{ steps.plan.outputs.run-benchmark-host-validation }} | |
| benchmark-host-validation-matrix: ${{ steps.plan.outputs.benchmark-host-validation-matrix }} | |
| run-benchmark-oracle: ${{ steps.plan.outputs.run-benchmark-oracle }} | |
| benchmark-oracle-scope: ${{ steps.plan.outputs.benchmark-oracle-scope }} | |
| benchmark-oracle-matrix: ${{ steps.plan.outputs.benchmark-oracle-matrix }} | |
| benchmark-plan-reasons: ${{ steps.plan.outputs.benchmark-plan-reasons }} | |
| benchmark-plan-receipt-digest: ${{ steps.plan.outputs.benchmark-plan-receipt-digest }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 | |
| - name: Restore benchmark timing history | |
| uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: .benchmark-timings.json | |
| key: benchmark-timings-unmatched | |
| restore-keys: benchmark-timings- | |
| - id: plan | |
| name: Classify benchmark changes | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| BASE_SHA: >- | |
| ${{ github.event.pull_request.base.sha || | |
| github.event.merge_group.base_sha || | |
| github.event.before }} | |
| HEAD_SHA: >- | |
| ${{ github.event.pull_request.head.sha || | |
| github.event.merge_group.head_sha || | |
| github.sha }} | |
| FORCE_FULL: ${{ contains(github.event.pull_request.labels.*.name, 'ci:benchmark-full') }} | |
| run: | | |
| set -eu | |
| planner_args=(--event "$EVENT_NAME" --head "$HEAD_SHA") | |
| changed_paths=() | |
| case "$EVENT_NAME" in | |
| schedule|workflow_dispatch) | |
| planner_args=(--event "$EVENT_NAME" --force-full --head "$HEAD_SHA") | |
| ;; | |
| pull_request|merge_group|push) | |
| if [ "$FORCE_FULL" = "true" ] || [ -z "$BASE_SHA" ] || [[ "$BASE_SHA" =~ ^0+$ ]]; then | |
| planner_args=(--event "$EVENT_NAME" --force-full --head "$HEAD_SHA") | |
| else | |
| planner_args=(--event "$EVENT_NAME" --base "$BASE_SHA" --head "$HEAD_SHA") | |
| mapfile -d '' changed_paths < <( | |
| git diff --name-only -z "$BASE_SHA" "$HEAD_SHA" | |
| ) | |
| fi | |
| ;; | |
| *) exit 2 ;; | |
| esac | |
| if [ -f .benchmark-timings.json ]; then | |
| planner_args+=(--timings-file .benchmark-timings.json) | |
| fi | |
| plan_output=$( | |
| uvx --from harbor==0.20.0 --with tomli-w==1.2.0 \ | |
| python .github/scripts/plan-benchmarks \ | |
| "${planner_args[@]}" -- "${changed_paths[@]}" | |
| ) | |
| printf '%s\n' "$plan_output" | python .github/scripts/validate-benchmark-plan | |
| printf '%s\n' "$plan_output" >> "$GITHUB_OUTPUT" | |
| plan_dir="$RUNNER_TEMP/jacobian-benchmark-plan" | |
| mkdir -p "$plan_dir" | |
| printf '%s\n' "$plan_output" > "$plan_dir/plan.txt" | |
| printf '%s\n' "${changed_paths[@]}" > "$plan_dir/changed-paths.txt" | |
| python .github/scripts/emit-plan-receipt \ | |
| --kind benchmark \ | |
| --event "$EVENT_NAME" \ | |
| --base "$BASE_SHA" \ | |
| --head "$HEAD_SHA" \ | |
| --planner .github/scripts/plan-benchmarks \ | |
| --config .github/scripts/_ci_paths.py \ | |
| --config .github/scripts/manage-test-timings \ | |
| --config .github/scripts/validate-benchmark-plan \ | |
| --config benchmarks/registry.toml \ | |
| --config benchmarks/environment-profiles.toml \ | |
| --config .github/workflows/benchmarks.yml \ | |
| --config Makefile \ | |
| --config tools/check_benchmark_adapters.py \ | |
| --config tools/check_benchmark_contracts.py \ | |
| --config tools/check_benchmark_static.py \ | |
| --config tools/pytest_lifecycle.py \ | |
| --config tools/check_harbor_dataset.py \ | |
| --config tools/sync_harbor_verifier_support.py \ | |
| --config benchmarks/tooling/benchmark_validation.py \ | |
| --config benchmarks/tooling/host_validation.py \ | |
| --config benchmarks/tooling/validation_plan.py \ | |
| --plan-file "$plan_dir/plan.txt" \ | |
| --paths-file "$plan_dir/changed-paths.txt" \ | |
| --output "$plan_dir/plan-receipt.json" | |
| echo "benchmark-plan-receipt-digest=$(python -c 'import json,sys; print(json.load(open(sys.argv[1]))["receipt_digest"])' "$plan_dir/plan-receipt.json")" >> "$GITHUB_OUTPUT" | |
| { | |
| echo "## Benchmark plan" | |
| echo | |
| echo '```text' | |
| echo "$plan_output" | |
| echo '```' | |
| echo "Plan receipt: \`$(python -c 'import json,sys; print(json.load(open(sys.argv[1]))["receipt_digest"])' "$plan_dir/plan-receipt.json")\`" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload benchmark plan receipt | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-plan-receipt | |
| path: ${{ runner.temp }}/jacobian-benchmark-plan/plan-receipt.json | |
| retention-days: 14 | |
| - name: Prepare benchmark pytest timing hint | |
| if: steps.plan.outputs.run-benchmark-host-validation == 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| .github/scripts/manage-test-timings prepare \ | |
| --suite benchmark \ | |
| --output .ci/benchmark-test-durations.json | |
| - name: Upload benchmark pytest timing hint | |
| if: steps.plan.outputs.run-benchmark-host-validation == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-test-durations-input | |
| path: .ci/benchmark-test-durations.json | |
| include-hidden-files: true | |
| retention-days: 1 | |
| static: | |
| name: Benchmark Static Quality | |
| needs: plan | |
| if: needs.plan.outputs.run-benchmark-check == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 | |
| - name: Check benchmark Python, verifier, and validation code | |
| run: uv run --locked python tools/check_benchmark_static.py | |
| contracts: | |
| name: Benchmark Contracts, Adapters, Records & Digests | |
| needs: plan | |
| if: needs.plan.outputs.run-benchmark-record-schema == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 | |
| - name: Validate benchmark contracts, adapters, schemas, records, and digests | |
| run: make harbor-contracts harbor-adapter-checks | |
| host_validation: | |
| name: Host Verifiers (${{ matrix.name }}) | |
| needs: plan | |
| if: needs.plan.outputs.run-benchmark-host-validation == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| include: ${{ fromJSON(needs.plan.outputs.benchmark-host-validation-matrix) }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 | |
| - name: Download benchmark timing hint | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: benchmark-test-durations-input | |
| path: .ci | |
| - name: Download benchmark plan receipt | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: benchmark-plan-receipt | |
| path: ${{ runner.temp }}/benchmark-plan | |
| - name: Run selected host-side verifier regressions | |
| env: | |
| HOST_ENTRY: ${{ toJSON(matrix) }} | |
| run: | | |
| uv run --locked python -m benchmarks.tooling.host_validation run-entry \ | |
| --entry-json "$HOST_ENTRY" \ | |
| --plan-receipt "${{ runner.temp }}/benchmark-plan/plan-receipt.json" \ | |
| --execution-sha "${{ github.sha }}" \ | |
| --timings .ci/benchmark-test-durations.json \ | |
| --durations-output "${{ runner.temp }}/benchmark-host-timing/durations/benchmark-test-durations.json" \ | |
| --receipt-dir "${{ runner.temp }}/benchmark-host-timing" \ | |
| --total-workers 8 --max-parallel 4 --store-durations | |
| - name: Upload host timing receipt | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-host-timing-${{ matrix.name }} | |
| path: ${{ runner.temp }}/benchmark-host-timing/**/pytest-receipt.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Upload benchmark shard durations | |
| if: matrix.splits > 0 && !cancelled() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| continue-on-error: true | |
| with: | |
| name: benchmark-duration-shard-${{ matrix.group }} | |
| path: ${{ runner.temp }}/benchmark-host-timing/durations/benchmark-test-durations.json | |
| include-hidden-files: true | |
| retention-days: 1 | |
| inventory: | |
| name: Benchmark Inventory Validation | |
| needs: plan | |
| if: needs.plan.outputs.run-benchmark-inventory == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 25 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 | |
| - name: Build and validate the content-bound benchmark inventory | |
| run: make benchmark-inventory | |
| oracle: | |
| name: Benchmark Oracle (${{ matrix.dataset }}/${{ matrix.shard }}) | |
| needs: [plan, static, contracts] | |
| if: >- | |
| ${{ always() && | |
| needs.plan.result == 'success' && | |
| needs.static.result == 'success' && | |
| needs.plan.outputs.run-benchmark-oracle == 'true' && | |
| needs.contracts.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| include: ${{ fromJSON(needs.plan.outputs.benchmark-oracle-matrix) }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 | |
| - name: Run exact task Oracle | |
| env: | |
| DATASET: ${{ matrix.dataset }} | |
| TASKS: ${{ join(matrix.tasks, ' ') }} | |
| run: make harbor-oracle-task DATASET="$DATASET" TASKS="$TASKS" | |
| - name: Upload Oracle evidence | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-oracle-${{ matrix.dataset }}-${{ matrix.shard }} | |
| path: | | |
| benchmarks/results/${{ matrix.dataset }}-oracle | |
| benchmarks/results/${{ matrix.dataset }}-oracle/jacobian-augmented-task-digests.json | |
| benchmarks/results/${{ matrix.dataset }}-oracle/**/oracle-evidence.json | |
| if-no-files-found: ignore | |
| retention-days: ${{ github.event_name == 'pull_request' && 14 || 90 }} | |
| validation: | |
| name: Benchmark Validation | |
| if: ${{ always() }} | |
| needs: [plan, static, contracts, host_validation, inventory, oracle] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - name: Download benchmark plan receipt | |
| if: needs.plan.result == 'success' | |
| continue-on-error: true | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: benchmark-plan-receipt | |
| path: ${{ runner.temp }}/benchmark-validation/plan | |
| - name: Download host shard receipts | |
| if: needs.plan.outputs.run-benchmark-host-validation == 'true' | |
| continue-on-error: true | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: benchmark-host-timing-* | |
| path: ${{ runner.temp }}/benchmark-validation/host | |
| - name: Download host timing input | |
| if: needs.plan.outputs.run-benchmark-host-validation == 'true' | |
| continue-on-error: true | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: benchmark-test-durations-input | |
| path: ${{ runner.temp }}/benchmark-validation/timing | |
| - name: Require planned benchmark validation | |
| env: | |
| PLAN_RESULT: ${{ needs.plan.result }} | |
| PLAN_CHECK: ${{ needs.plan.outputs.run-benchmark-check }} | |
| RECORD_SCHEMA_FLAG: ${{ needs.plan.outputs.run-benchmark-record-schema }} | |
| PROSPECTIVE_DIGEST_FLAG: ${{ needs.plan.outputs.run-benchmark-prospective-digest }} | |
| INVENTORY_FLAG: ${{ needs.plan.outputs.run-benchmark-inventory }} | |
| HOST_VALIDATION_FLAG: ${{ needs.plan.outputs.run-benchmark-host-validation }} | |
| ORACLE_FLAG: ${{ needs.plan.outputs.run-benchmark-oracle }} | |
| STATIC_RESULT: ${{ needs.static.result }} | |
| CONTRACTS_RESULT: ${{ needs.contracts.result }} | |
| HOST_VALIDATION_RESULT: ${{ needs.host_validation.result }} | |
| INVENTORY_RESULT: ${{ needs.inventory.result }} | |
| ORACLE_RESULT: ${{ needs.oracle.result }} | |
| run: | | |
| python -m benchmarks.tooling.benchmark_validation \ | |
| --plan-result "$PLAN_RESULT" \ | |
| --plan-receipt "${{ runner.temp }}/benchmark-validation/plan/plan-receipt.json" \ | |
| --execution-sha "${{ github.sha }}" \ | |
| --receipt-root "${{ runner.temp }}/benchmark-validation/host" \ | |
| --timings "${{ runner.temp }}/benchmark-validation/timing/benchmark-test-durations.json" \ | |
| --lane "static:$PLAN_CHECK:$STATIC_RESULT" \ | |
| --lane "contracts:$RECORD_SCHEMA_FLAG:$CONTRACTS_RESULT" \ | |
| --lane "host-validation:$HOST_VALIDATION_FLAG:$HOST_VALIDATION_RESULT" \ | |
| --lane "inventory:$INVENTORY_FLAG:$INVENTORY_RESULT" \ | |
| --lane "oracle:$ORACLE_FLAG:$ORACLE_RESULT" | |
| timings: | |
| name: Publish Benchmark Timings | |
| needs: [plan, host_validation, oracle, validation] | |
| if: >- | |
| ${{ always() && | |
| needs.plan.outputs.run-benchmark-host-validation == 'true' && | |
| needs.host_validation.result == 'success' && | |
| needs.validation.result == 'success' && | |
| (needs.plan.outputs.run-benchmark-oracle != 'true' || | |
| needs.oracle.result == 'success') }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 8 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 | |
| - name: Restore previous timing history | |
| uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: .benchmark-timings.json | |
| key: benchmark-timings-unmatched | |
| restore-keys: benchmark-timings- | |
| - name: Download host timing receipts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: benchmark-host-timing-* | |
| path: timing-artifacts | |
| - name: Download benchmark shard durations | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| continue-on-error: true | |
| with: | |
| pattern: benchmark-duration-shard-* | |
| path: .ci/shards | |
| - name: Download Oracle artifacts | |
| if: needs.plan.outputs.run-benchmark-oracle == 'true' | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: benchmark-oracle-* | |
| path: timing-artifacts | |
| - name: Collect and compare shard timings | |
| env: | |
| ORACLE_MATRIX: ${{ needs.plan.outputs.benchmark-oracle-matrix }} | |
| HOST_MATRIX: ${{ needs.plan.outputs.benchmark-host-validation-matrix }} | |
| run: | | |
| uv run --locked python -m benchmarks.tooling.benchmark_timings \ | |
| --root timing-artifacts \ | |
| --output .benchmark-timings.json \ | |
| --previous .benchmark-timings.json \ | |
| --oracle-matrix-json "$ORACLE_MATRIX" \ | |
| --host-matrix-json "$HOST_MATRIX" \ | |
| --report-output benchmark-timing-report.json \ | |
| --summary-output "$GITHUB_STEP_SUMMARY" | |
| - id: benchmark-durations | |
| name: Merge complete full-suite pytest timings | |
| run: | | |
| set -eu | |
| inputs=() | |
| for shard in 1 2 3 4; do | |
| path=".ci/shards/benchmark-duration-shard-${shard}/benchmark-test-durations.json" | |
| if [ ! -f "$path" ]; then | |
| echo "complete=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| inputs+=(--input "$path") | |
| done | |
| .github/scripts/manage-test-timings merge \ | |
| --suite benchmark \ | |
| "${inputs[@]}" \ | |
| --output .ci/benchmark-test-durations.json \ | |
| --source-sha "${{ github.sha }}" \ | |
| --python-version "3.12" | |
| echo "complete=true" >> "$GITHUB_OUTPUT" | |
| - name: Upload timing evidence | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-timings-${{ github.sha }} | |
| path: | | |
| .benchmark-timings.json | |
| benchmark-timing-report.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Publish benchmark pytest timing hint | |
| if: steps.benchmark-durations.outputs.complete == 'true' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-test-durations | |
| path: .ci/benchmark-test-durations.json | |
| include-hidden-files: true | |
| retention-days: 90 | |
| - name: Save timing history for future main-branch plans | |
| if: github.ref == 'refs/heads/main' | |
| uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: .benchmark-timings.json | |
| key: benchmark-timings-${{ github.run_id }} |