sv-tests-ci #489
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: sv-tests-ci | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 1 * * *' # run daily at 01:00 am (UTC) | |
| jobs: | |
| Run: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tool: | |
| - name: icarus | |
| deps: autoconf autotools-dev bison flex libfl-dev gperf clang cabal-install | |
| - name: slang | |
| deps: cmake pkg-config clang cabal-install | |
| # - name: surelog | |
| # repo: Surelog | |
| # submodules: third_party/UHDM third_party/antlr4 third_party/googletest | |
| # deps: cmake default-jre pkg-config tclsh uuid-dev | |
| - name: verible | |
| deps: bazel=7.6.1 bison flex libfl-dev clang cabal-install | |
| runners_filter: Verible | |
| skip-ccache: 1 | |
| - name: verilator | |
| deps: autoconf autotools-dev bison flex help2man libfl-dev libelf-dev clang cabal-install | |
| make_jobs_max: 4 | |
| - name: yosys | |
| deps: bison clang tcl-dev flex libfl-dev pkg-config libreadline-dev cabal-install | |
| - name: yosys-slang | |
| deps: bison clang tcl-dev flex libfl-dev pkg-config libreadline-dev cmake pkg-config cabal-install | |
| submodules: third_party/slang third_party/fmt | |
| # - name: yosys-synlig | |
| # repo: synlig | |
| # submodules: third_party/yosys third_party/surelog | |
| # deps: cmake clang tcl-dev bison default-jre flex libfl-dev libreadline-dev pkg-config tclsh uuid-dev | |
| # runners_filter: SynligYosys | |
| env: | |
| RUNNERS_FILTER: ${{ matrix.tool.runners_filter }} | |
| CCACHE_DIR: "/root/sv-tests/sv-tests/.cache/" | |
| # those generators can use a lot of RAM/cpu and starve other tests | |
| # tests from those generators are run without "-j" flag | |
| BIG_GENERATORS: "fusesoc black-parrot" | |
| DEBIAN_FRONTEND: "noninteractive" | |
| GHA_MACHINE_TYPE: "n2-highmem-16" | |
| name: ${{ matrix.tool.name }} | |
| runs-on: [ubuntu-latest] | |
| # runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| container: ubuntu:latest | |
| steps: | |
| # - name: Cancel previous | |
| # uses: styfle/cancel-workflow-action@0.8.0 | |
| # with: | |
| # access_token: ${{ github.token }} | |
| - name: Show shell | |
| run: | | |
| echo "$SHELL" | |
| - name: Update apt repos and install git | |
| run: | | |
| apt-get update -qq | |
| apt install -y git | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Fix git safe directory | |
| run: | | |
| git config --global --add safe.directory ${{ github.workspace }} | |
| - name: Setup Node | |
| run: | | |
| apt -qq -y install curl | |
| touch "$HOME/.bash_profile" | |
| # From https://nodejs.org/en/download | |
| curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash | |
| . "$HOME/.nvm/nvm.sh" | |
| nvm install 22 | |
| - name: Add bazel repo (if needed) | |
| if: ${{ contains(matrix.tool.deps, 'bazel') }} | |
| run: | | |
| apt -qq -y install apt-transport-https curl gnupg | |
| curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor > bazel.gpg | |
| chmod a+r bazel.gpg | |
| mv bazel.gpg /etc/apt/trusted.gpg.d/ | |
| echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list | |
| apt-get update -qq | |
| apt -qq -y install bazel-7.6.0 | |
| - name: Install common dependencies | |
| run: | | |
| apt install -y python3 python3-pip git wget file | |
| update-alternatives --install /usr/bin/python python /usr/bin/python3 1 | |
| update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 | |
| pip install --break-system-packages --use-pep517 --upgrade setuptools | |
| pip install --break-system-packages --use-pep517 -r conf/requirements.txt | |
| - name: Install VeeR dependencies | |
| run: | | |
| apt install -y cpanminus | |
| cpanm Bit::Vector JSON | |
| - name: Install ccache | |
| if: ${{ !matrix.tool.skip-ccache }} | |
| run: | | |
| apt install -y ccache | |
| - name: Install tool-specific dependencies | |
| if: ${{ matrix.tool.deps }} | |
| run: | | |
| apt -qq -y install ${{ matrix.tool.deps }} | |
| - name: Setup Rust (if needed) | |
| if: ${{ matrix.tool.rust_ver }} | |
| run: | | |
| apt -y install curl | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > setup.sh | |
| sh setup.sh -y | |
| source $HOME/.cargo/env | |
| rustup install ${{ matrix.tool.rust_ver }} | |
| rustup default ${{ matrix.tool.rust_ver }} | |
| - name: Update haskell (if needed) | |
| if: ${{ contains(matrix.tool.deps, 'haskell') }} | |
| run: | | |
| stack upgrade | |
| - name: Fix git safe directory | |
| run: | | |
| git config --global --add safe.directory /__w/sv-tests/sv-tests | |
| - name: Checkout the tool submodule | |
| shell: bash | |
| run: | | |
| # Github dropped support for unauthorized git: https://github.blog/2021-09-01-improving-git-protocol-security-github/ | |
| # Make sure we always use https:// instead of git:// | |
| git config --global url.https://github.com/.insteadOf git://github.com/ | |
| # take verilator from github (some tested tools still use a submodule | |
| # from veripool) | |
| git config --global url."https://github.com/verilator/verilator".insteadOf http://git.veripool.org/git/verilator | |
| REPOSITORY_NAME=${{ matrix.tool.name }} | |
| if [[ ! -z "${{ matrix.tool.repo }}" ]]; then | |
| REPOSITORY_NAME=${{ matrix.tool.repo }} | |
| fi | |
| git submodule update --init --depth 1 third_party/tools/${REPOSITORY_NAME} | |
| if [[ ! -z "${{ matrix.tool.submodules }}" ]]; then | |
| pushd third_party/tools/${REPOSITORY_NAME} | |
| git submodule update --init --recursive --depth 1 ${{ matrix.tool.submodules }} | |
| popd | |
| fi | |
| # yosys tool contains tests, is a dependency of other tools | |
| git submodule update --init --recursive --depth 1 third_party/tools/yosys | |
| # icarus contains tests | |
| git submodule update --init --depth 1 third_party/tools/icarus | |
| - name: Create Cache Timestamp | |
| id: cache_timestamp | |
| uses: nanzm/get-time-action@v1.1 | |
| with: | |
| format: 'YYYY-MM-DD-HH-mm-ss' | |
| - name: Setup cache | |
| uses: actions/cache@v4 | |
| timeout-minutes: 3 | |
| continue-on-error: true | |
| with: | |
| path: "/root/sv-tests/sv-tests/.cache/" | |
| key: cache_${{ matrix.tool.name }}_${{ steps.cache_timestamp.outputs.time }} | |
| restore-keys: cache_${{ matrix.tool.name }}_ | |
| - name: Build | |
| shell: bash | |
| run: | | |
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | |
| [[ -f $HOME/.cargo/env ]] && source $HOME/.cargo/env | |
| . "$HOME/.bash_profile" # GitHub Actions runs bash with --noprofile | |
| make ${{ matrix.tool.name }} -j$(nproc) | |
| - name: Check | |
| run: | |
| make info | |
| - name: Checkout third party tests and cores | |
| run: | | |
| git submodule update --init --recursive --depth 1 third_party/tests | |
| git submodule update --init --recursive --depth 1 third_party/cores | |
| - name: Run | |
| run: | | |
| BIG_GENERATORS_EXPR=$(echo $BIG_GENERATORS | sed 's/ /\\|/g') | |
| export STABLE_GENERATORS=$(make list-generators | tr ' ' '\n' | grep -v "${BIG_GENERATORS_EXPR}") | |
| export UNSTABLE_GENERATORS=$(make list-generators | tr ' ' '\n' | grep "${BIG_GENERATORS_EXPR}") | |
| export JOBS=${{ matrix.tool.make_jobs_max }} | |
| if [[ "${JOBS}x" == "x" ]]; then export JOBS=$(nproc); fi | |
| for gen in ${STABLE_GENERATORS}; do echo "==GEN stable $gen -j $JOBS" ; make generate-$gen -j${JOBS}; make -j ${JOBS}; done | |
| for gen in ${UNSTABLE_GENERATORS}; do echo "==GEN unstable $gen" ; make generate-$gen; make; done | |
| - name: Prepare Report | |
| run: | |
| mv out/report/report.csv out/report/${{ matrix.tool.name }}_report.csv | |
| - name: Pack results | |
| run: | | |
| tar -cvf out_${{ matrix.tool.name }}.tar ./out/report/${{ matrix.tool.name }}_report.csv ./out/logs/ | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: report_${{ matrix.tool.name }} | |
| path: | | |
| out_${{ matrix.tool.name }}.tar | |
| **/plot_*.svg | |
| Summary: | |
| name: Summary | |
| runs-on: [ubuntu-latest] | |
| # runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| container: ubuntu:latest | |
| needs: Run | |
| env: | |
| ANALYZER: "./tools/report_analyzer.py" | |
| GRAPHER: "./tools/history-graph" | |
| OUT_REPORT_DIR: "./out/report/" | |
| COMPARE_REPORT: "./out/report/report.csv" | |
| CHANGES_SUMMARY_JSON: "./out/report/tests_summary.json" | |
| CHANGES_SUMMARY_MD: "./out/report/tests_summary.md" | |
| TESTS_SUMMARY_DIR: "tests_summary/" | |
| DEBIAN_FRONTEND: "noninteractive" | |
| steps: | |
| - name: Update apt repos and install git | |
| run: | | |
| apt-get update -qq | |
| apt install -y git | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup python | |
| run: | | |
| apt-get update -qq | |
| apt install -y python3 python3-pip wget git curl jq clang cabal-install | |
| pip install --break-system-packages --upgrade setuptools setuptools_scm | |
| pip install --break-system-packages -r conf/requirements.txt | |
| - name: Install VeeR dependencies | |
| run: | | |
| apt install -y cpanminus | |
| cpanm Bit::Vector JSON | |
| - name: Prepare output directories | |
| run: | | |
| mkdir -p out/report | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: ./out/ | |
| - name: Extract | |
| run: | | |
| for file in $(find out/ -name *.tar -print); do tar -xf $file --strip-components=2 -C $(dirname $file); done | |
| - name: Fix git safe directory | |
| run: | | |
| git config --global --add safe.directory /__w/sv-tests/sv-tests | |
| - name: Checkout third party tests and cores | |
| run: | | |
| # take verilator from github | |
| git config --global url."https://github.com/verilator/verilator".insteadOf http://git.veripool.org/git/verilator | |
| git submodule update --init --recursive --depth 1 third_party/tests | |
| git submodule update --init --recursive --depth 1 third_party/cores | |
| # yosys tool also contains tests | |
| git submodule update --init --recursive --depth 1 third_party/tools/yosys | |
| # icarus contains tests | |
| git submodule update --init --depth 1 third_party/tools/icarus | |
| - name: Summary | |
| run: | | |
| ./.github/workflows/summary.sh | |
| ./.github/workflows/report.sh | |
| - name: Update sv-tests-results repository | |
| if: github.ref == 'refs/heads/master' | |
| shell: bash | |
| run: | | |
| eval $(ssh-agent -s) | |
| ssh-add - <<< "${{ secrets.REPORT_DEPLOY_KEY }}" | |
| mkdir -p ~/.ssh | |
| chmod 700 ~/.ssh | |
| git config --global core.sshCommand 'ssh -o StrictHostKeyChecking=no' | |
| ./.github/workflows/update_report.sh | |
| - name: Prepare artifacts for PR commenter | |
| if: github.event_name == 'pull_request' | |
| shell: bash | |
| run: | | |
| mkdir $TESTS_SUMMARY_DIR | |
| echo ${{ github.event.number }} > $TESTS_SUMMARY_DIR/issue_num | |
| cp $COMPARE_REPORT $TESTS_SUMMARY_DIR/report_base.csv | |
| cp $OUT_REPORT_DIR/report.csv $TESTS_SUMMARY_DIR/report_new.csv | |
| cp $OUT_REPORT_DIR/new_*csv $TESTS_SUMMARY_DIR | |
| cp $OUT_REPORT_DIR/tests_summary.json $TESTS_SUMMARY_DIR | |
| cp $OUT_REPORT_DIR/tests_summary.md $TESTS_SUMMARY_DIR | |
| find out -name plot_*.svg -exec cp {} $TESTS_SUMMARY_DIR \; | |
| - name: Post GitHub summary | |
| run: | | |
| cat $CHANGES_SUMMARY_MD > $GITHUB_STEP_SUMMARY | |
| - name: Upload artifacts for summary | |
| uses: actions/upload-artifact@v4 | |
| if: github.event_name == 'pull_request' | |
| with: | |
| name: tests_summary | |
| path: | | |
| ./tests_summary/ | |
| - name: Find artifacts that are no longer needed | |
| id: get-artifacts-to-delete | |
| if: github.event_name == 'pull_request' | |
| shell: bash | |
| run: | | |
| artifacts=$(find ./out -type d -name 'report_*' -exec basename {} \;) | |
| echo $artifacts | |
| artifacts="${artifacts//'%'/'%25'}" | |
| artifacts="${artifacts//$'\n'/'%0A'}" | |
| artifacts="${artifacts//$'\r'/'%0D'}" | |
| echo "artifacts=$artifacts" >> GITHUB_OUTPUT | |
| echo $artifacts | |
| - name: Delete Old Artifacts | |
| if: github.event_name == 'pull_request' | |
| uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: ${{ steps.get-artifacts-to-delete.outputs.artifacts }} | |
| Automerge: | |
| name: Automerge dependabot pull requests | |
| permissions: | |
| pull-requests: write | |
| contents: write | |
| runs-on: ubuntu-latest | |
| needs: Summary | |
| if: ${{ github.event_name == 'pull_request' && github.event.pull_request.user.login == 'dependabot[bot]' }} | |
| steps: | |
| - name: Dependabot metadata | |
| id: metadata | |
| uses: dependabot/fetch-metadata@v1.1.1 | |
| with: | |
| github-token: "${{ secrets.GITHUB_TOKEN }}" | |
| - name: Enable auto-merge for Dependabot PRs | |
| run: gh pr merge --auto --merge "$PR_URL" | |
| env: | |
| PR_URL: ${{ github.event.pull_request.html_url }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |