disk-benchmarks-aa.yml

# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT license.

# DiskANN Daily A/A Benchmark Stability Test
#
# Runs main vs main at 9 AM UTC every day to detect environment noise.
# If any threshold is breached, a GitHub issue is created to notify @microsoft/diskann-admin.
# Can also be triggered manually for debugging.
#
# For more details see .github/docs/disk-benchmarks-aa.md

name: Disk Benchmarks (A/A)

on:
  schedule:
    # Daily at 9 AM UTC
    - cron: '0 9 * * *'
  workflow_dispatch:  # Allow manual trigger for debugging

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

env:
  RUST_BACKTRACE: 1
  PERF_INPUTS: diskann-benchmark/perf_test_inputs

defaults:
  run:
    shell: bash

permissions:
  contents: read
  issues: write  # Required for creating failure notification issues

jobs:
  # A/A benchmark: run main vs main to detect environment noise.
  aa-benchmark:
    name: A/A - ${{ matrix.dataset }}
    runs-on: [ self-hosted, 1ES.Pool=diskann-github, ubuntu-latest, "JobId=aa-benchmark-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ strategy.job-index }}" ]
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        include:
          - dataset: wikipedia-100K
            config: wikipedia-100K-disk-index.json
            archive: wikipedia-100K.tar.gz
            sha256: 5b312d186773c549e6132483b148a22b2421140f1a8d9fcd6230c4e019405027
          - dataset: openai-100K
            config: openai-100K-disk-index.json
            archive: openai-100K.tar.gz
            sha256: b6887d9a31e9ce035665e3df385db37866116a7b7effc1cbaded239956fec113

    steps:
      # Kept inline because this must run before checkout, but local action.yml
      # files are only available after checkout.
      - name: Mount high-speed NVMe SSD
        shell: bash
        run: |
          sudo mkdir -p /mnt/nvme
          sudo lsblk
          sudo mkfs.ext4 /dev/nvme0n1
          sudo mount /dev/nvme0n1 /mnt/nvme
          sudo chmod 777 /mnt/nvme
          mkdir -p /mnt/nvme/diskann_rust /mnt/nvme/baseline
          ln -s /mnt/nvme/diskann_rust diskann_rust
          ln -s /mnt/nvme/baseline baseline

      - name: Checkout main
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: main
          path: diskann_rust
          lfs: true

      - name: Setup benchmark environment
        uses: ./diskann_rust/.github/actions/setup-disk-benchmark
        with:
          dataset: ${{ matrix.dataset }}
          archive: ${{ matrix.archive }}
          sha256: ${{ matrix.sha256 }}
          extract-to: diskann_rust/target/tmp

      # A/A: build once, run twice (identical code — only detecting environment noise)
      - name: Build benchmark binary
        working-directory: diskann_rust
        run: cargo build -p diskann-benchmark --features disk-index --release

      - name: Run baseline benchmark
        working-directory: diskann_rust
        run: |
          cargo run -p diskann-benchmark --features disk-index --release -- \
            run --input-file ${{ env.PERF_INPUTS }}/${{ matrix.config }} \
            --output-file target/tmp/${{ matrix.dataset }}_baseline.json

      - name: Run target benchmark
        working-directory: diskann_rust
        run: |
          cargo run -p diskann-benchmark --features disk-index --release -- \
            run --input-file ${{ env.PERF_INPUTS }}/${{ matrix.config }} \
            --output-file target/tmp/${{ matrix.dataset }}_target.json

      - name: Validate benchmark results
        working-directory: diskann_rust
        run: |
          cargo run -p diskann-benchmark --features disk-index --release -- \
            check run \
            --tolerances ${{ env.PERF_INPUTS }}/disk-index-tolerances.json \
            --input-file ${{ env.PERF_INPUTS }}/${{ matrix.config }} \
            --before target/tmp/${{ matrix.dataset }}_baseline.json \
            --after target/tmp/${{ matrix.dataset }}_target.json

      - name: Upload benchmark results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: always()
        with:
          name: aa-results-${{ matrix.dataset }}
          path: |
            diskann_rust/target/tmp/${{ matrix.dataset }}_target.json
            diskann_rust/target/tmp/${{ matrix.dataset }}_baseline.json
          retention-days: 30

  # Notify diskann-disk-maintainers on A/A failure — but only when the failure
  # rate exceeds 5% over the last 20 runs. Our reliability promise is 95%, so
  # 1 failure in 20 runs is expected and should not trigger a notification.
  notify-on-failure:
    name: Notify on A/A Failure
    needs: [aa-benchmark]
    runs-on: ubuntu-latest
    if: failure()
    steps:
      - name: Check recent failure rate
        id: check-rate
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const { data: runs } = await github.rest.actions.listWorkflowRuns({
              owner: context.repo.owner,
              repo: context.repo.repo,
              workflow_id: 'disk-benchmarks-aa.yml',
              per_page: 20,
              status: 'completed',
            });
            const total = runs.workflow_runs.length;
            const failures = runs.workflow_runs.filter(r => r.conclusion === 'failure').length;
            const failureRate = total > 0 ? failures / total : 0;
            console.log(`Recent A/A runs: ${total}, failures: ${failures}, rate: ${(failureRate * 100).toFixed(1)}%`);
            // Only notify if failure rate exceeds our 5% reliability budget
            core.setOutput('should_notify', failureRate > 0.05 ? 'true' : 'false');
            core.setOutput('failure_rate', `${(failureRate * 100).toFixed(1)}%`);
            core.setOutput('failures', `${failures}`);
            core.setOutput('total', `${total}`);

      - name: Create GitHub issue for A/A failure
        if: steps.check-rate.outputs.should_notify == 'true'
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const date = new Date().toISOString().split('T')[0];
            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            await github.rest.issues.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title: `[Benchmark A/A] Daily stability test failed – ${date}`,
              body: [
                `## Daily A/A Benchmark Failure`,
                ``,
                `The scheduled A/A benchmark run (main vs main) **failed** on ${date}.`,
                `This indicates environment noise exceeded the configured thresholds.`,
                ``,
                `**Run:** ${runUrl}`,
                `**Recent failure rate:** ${{ steps.check-rate.outputs.failures }}/${{ steps.check-rate.outputs.total }} (${{ steps.check-rate.outputs.failure_rate }}) — exceeds 5% reliability budget`,
                ``,
                `Please review the benchmark artifacts and determine if thresholds need tuning`,
                `or if there is a runner environment issue.`,
                ``,
                `/cc @microsoft/diskann-disk-maintainers`,
              ].join('\n'),
              labels: ['benchmark', 'A/A-failure'],
            });