Non-deterministic fuzz tests #476
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Non-deterministic fuzz tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # Run nightly at 0300 | |
| - cron: "0 3 * * *" | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # How long should we run the fuzzing for, in seconds. | |
| NOIR_AST_FUZZER_BUDGET_SECS: 1800 | |
| jobs: | |
| ast-fuzz: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Setup toolchain | |
| uses: dtolnay/rust-toolchain@1.89.0 | |
| with: | |
| targets: x86_64-unknown-linux-gnu | |
| - uses: taiki-e/install-action@just | |
| - name: Run fuzzer | |
| id: fuzz | |
| shell: bash | |
| run: | | |
| set -o pipefail | |
| just fuzz-nightly 2>&1 | tee fuzz-output.log | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| RUST_MIN_STACK: 8388608 | |
| - name: Extract fuzzer seeds | |
| if: failure() | |
| id: seeds | |
| shell: bash | |
| run: | | |
| ./.github/scripts/extract-fuzz-seeds.sh fuzz-output.log . | |
| seed_count=$(wc -l < seeds.txt | tr -d ' ') | |
| seeds=$(paste -sd, - < seeds.txt) | |
| echo "seed_count=${seed_count}" >> "$GITHUB_OUTPUT" | |
| echo "seeds=${seeds}" >> "$GITHUB_OUTPUT" | |
| - name: Upload fuzz failure artifact | |
| if: failure() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: nightly-fuzz-failure | |
| path: | | |
| fuzz-output.log | |
| seeds.txt | |
| seeds-by-test.tsv | |
| retention-days: 30 | |
| - name: Get current date | |
| id: date | |
| run: echo "DAY=date::$(date +'%u')" >> $GITHUB_OUTPUT | |
| - name: Notify Slack and dispatch ClaudeBox on failure | |
| id: claudebox | |
| if: ${{ failure() && steps.date.outputs.DAY <= 5 }} | |
| uses: ./.github/actions/claudebox-dispatch | |
| with: | |
| claudebox-url: ${{ secrets.CLAUDEBOX_URL }} | |
| claudebox-api-secret: ${{ secrets.CLAUDEBOX_API_SECRET }} | |
| repo: noir-lang/noir | |
| run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| user: noir-nightly-fuzz | |
| mode: public | |
| target-ref: origin/${{ github.ref_name }} | |
| link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| slack-bot-token: ${{ secrets.CLAUDEBOX_SLACK_BOT_TOKEN }} | |
| slack-channel: ${{ secrets.FUZZER_FAILURE_SLACK_CHANNEL_ID }} | |
| slack-alert: >- | |
| :rotating_light: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Nightly compiler fuzzing> | |
| failed — ${{ steps.seeds.outputs.seed_count }} seed(s): `${{ steps.seeds.outputs.seeds }}`. | |
| ClaudeBox is reproducing and diagnosing the failure in this thread. | |
| prompt: | | |
| The nightly Noir compiler fuzzer failed on `${{ github.ref_name }}`. | |
| Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| Failing seeds (${{ steps.seeds.outputs.seed_count }}): ${{ steps.seeds.outputs.seeds }} | |
| Working one by one, reproduce the fuzzer failures using those seeds | |
| against noir-lang/noir master. For each seed perform an RCA and determine | |
| whether they are from different underlying bugs or if they are all | |
| symptoms of the same issue. Create gists which detail the issue, with a | |
| suggested fix along with minimal loadbearing unit tests. | |
| Repository tooling may help; the `bisect-ssa-pass` skill is particularly | |
| useful for SSA failures. Check for known issues which trigger the fuzzer by | |
| looking for issues/PRs with the `AST Fuzzer` label, and say in this thread | |
| whether this is a duplicate. | |
| Post the RCA and the proposed fix in this thread and wait for a human to | |
| say go before opening a PR. | |
| - name: Send the existing fallback alert in Slack | |
| if: ${{ always() && steps.claudebox.outcome == 'failure' }} | |
| uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 | |
| with: | |
| webhook: ${{ secrets.FUZZING_ALERT_URL }} | |
| webhook-type: webhook-trigger | |
| payload: | | |
| workflow-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| seed-count: ${{ steps.seeds.outputs.seed_count }} | |
| seeds: "${{ steps.seeds.outputs.seeds }}" | |
| artifact-name: nightly-fuzz-failure |