Nightly Tests #33
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Tests | |
| # Run once a day at midnight Pacific Daylight Time. | |
| # Note: GitHub Actions cron is in UTC. | |
| on: | |
| schedule: | |
| - cron: '0 7 * * *' # Daily at 07:00 UTC → 00:00 PDT | |
| workflow_dispatch: # Allow manual triggering | |
| # Only allow one pipeline to run at a time. | |
| concurrency: | |
| group: ${{ github.repository }} | |
| cancel-in-progress: false | |
| jobs: | |
| # Run bazel build before bazel tests because the logging is not chronologically | |
| # ordered if a build step fails while unrelated tests are running. | |
| # | |
| # Additionally, we rely on the fact that we disabled any pipeline concurrency and that there is | |
| # a single self-hosted runner. So each job can assume the working directory is preserved | |
| # from the previous job (bazel-build). This is fine for now since our CI throughput demands are not | |
| # very high. | |
| bazel-build: | |
| runs-on: self-hosted | |
| steps: | |
| # actions/checkout v7.0.0 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| - name: Copy ci.bazelrc | |
| working-directory: ${{ github.workspace }} | |
| run: cp /home/gh-actions/bedrock-infra/ci.bazelrc ci.bazelrc | |
| - name: Build | |
| run: bazel build --noshow_loading_progress //... | |
| bazel-test-jg: | |
| runs-on: self-hosted | |
| environment: ci-badge-publishing | |
| needs: bazel-build | |
| outputs: | |
| total_tests: ${{ steps.test.outputs.total_tests }} | |
| passing_tests: ${{ steps.test.outputs.passing_tests }} | |
| failing_timeout_tests: ${{ steps.test.outputs.failing_timeout_tests }} | |
| exit_code: ${{ steps.test.outputs.exit_code }} | |
| steps: | |
| - name: Verilog formal tests (using Cadence JasperGold) | |
| id: test | |
| # Use --test_output=summary to suppress log output, even on failure (proprietary EDA tool output may be sensitive). | |
| # | |
| # Set a timeout of 60 seconds per test and randomly sample 1000 tests to run every night, since there are tens of | |
| # thousands of tests and we have very few licenses. | |
| # | |
| # For now, exclude //fifo/fpv/... //cdc/fpv/... as they are known to be very slow. | |
| run: | | |
| bazel query 'tests(//...) intersect attr(tags, "jg", //...)' \ | |
| | grep -v -E '//fifo/fpv:|//cdc/fpv:' \ | |
| | shuf -n 1000 \ | |
| | xargs -r bazel test --noshow_loading_progress --test_output=summary --test_timeout=60 2>&1 \ | |
| | tee .bazel-test-jg.log | |
| BAZEL_EXIT_CODE=${PIPESTATUS[3]} | |
| TOTAL_TESTS=$(grep -Po '(?<=out of )\d+(?= test)' .bazel-test-jg.log) | |
| PASSING_TESTS=$(grep -Po '\d+(?= test passes?\.| tests pass)' .bazel-test-jg.log) | |
| FAILING_TIMEOUT_TESTS=$(grep -c ' TIMEOUT in' .bazel-test-jg.log) | |
| echo "total_tests=$TOTAL_TESTS" >> $GITHUB_OUTPUT | |
| echo "passing_tests=$PASSING_TESTS" >> $GITHUB_OUTPUT | |
| echo "failing_timeout_tests=$FAILING_TIMEOUT_TESTS" >> $GITHUB_OUTPUT | |
| echo "exit_code=$BAZEL_EXIT_CODE" >> $GITHUB_OUTPUT | |
| exit $BAZEL_EXIT_CODE | |
| - name: Post nightly test summary to GitHub Gist | |
| if: always() | |
| run: | | |
| # Fetch current log via Gist API | |
| EXISTING_CONTENT=$( | |
| curl -s https://api.github.com/gists/4c1321baf4a3e4b67d12480df7fd2d0c \ | |
| | jq -r --arg f "jg-nightly-log.txt" '.files[$f].content' | |
| ) | |
| echo "$EXISTING_CONTENT" | |
| # Create summary line | |
| TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") | |
| SUMMARY="[$TIMESTAMP] Passed: ${{ needs.bazel-test-jg.outputs.passing_tests }} | Timeouts: ${{ needs.bazel-test-jg.outputs.failing_timeout_tests }} | Total: ${{ needs.bazel-test-jg.outputs.total_tests }}" | |
| # Combine existing content and new summary | |
| UPDATED_CONTENT="$(printf "%s\n%s" "$EXISTING_CONTENT" "$SUMMARY")" | |
| # Escape UPDATED_CONTENT for JSON: | |
| # - Replace all literal \ with \\ | |
| # - Replace all literal " with \" | |
| # - Replace all literal newlines with \n | |
| # Thus we obtain one long string that we can embed into a JSON "content" field. | |
| escaped=$(sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -e ':a;N;$!ba;s/\n/\\n/g' <<< "$UPDATED_CONTENT") | |
| # Build JSON payload by hand | |
| PATCH_DATA=$(printf '{"files":{"jg-nightly-log.txt":{"content":"%s"}}}' "$escaped") | |
| # Append and push updated content | |
| echo "$PATCH_DATA" \ | |
| | curl -s -X PATCH \ | |
| -H "Authorization: token ${{ secrets.MGOTTSCHO_GISTS_SECRET }}" \ | |
| -H "Content-Type: application/json" \ | |
| --data @- https://api.github.com/gists/4c1321baf4a3e4b67d12480df7fd2d0c | |
| jg-badge: | |
| runs-on: ubuntu-24.04 | |
| environment: ci-badge-publishing | |
| needs: bazel-test-jg | |
| # Run when bazel-test-jg completes, even if it failed. | |
| if: ${{ always() }} | |
| steps: | |
| - name: Create jg badge | |
| # schneegans/dynamic-badges-action v1.7.0 | |
| uses: schneegans/dynamic-badges-action@e9a478b16159b4d31420099ba146cdc50f134483 | |
| with: | |
| auth: ${{ secrets.MGOTTSCHO_GISTS_SECRET }} | |
| gistID: c66dc2ddc0e513ba06ce338620977b26 | |
| filename: jg.json | |
| label: "jg (nightly)" | |
| message: ${{ needs.bazel-test-jg.outputs.passing_tests }} passed, ${{ needs.bazel-test-jg.outputs.failing_timeout_tests }} timed out, total ${{ needs.bazel-test-jg.outputs.total_tests }} | |
| color: ${{ needs.bazel-test-jg.outputs.exit_code == '0' && 'brightgreen' || 'red' }} |