cts.yml

name: CTS

on:
  push:
    branches-ignore: [
        # Renovate branches are always PRs, so they will be covered
        # by the pull_request event.
        "renovate/**",
        # Branches with the `gh-readonly-queue` prefix are used by the
        # merge queue, so they are already covered by the `merge_group` event.
        "gh-readonly-queue/**",
      ]
  pull_request:
  merge_group:

env:
  #
  # Keep in sync with ci.yml
  #

  REPO_MSRV: "1.93"
  CARGO_INCREMENTAL: false
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: full
  RUSTFLAGS: -D warnings
  RUSTDOCFLAGS: -D warnings

  #
  # Runtime configuration
  #

  LVP_POISON_MEMORY: true # to test memory init; see docs/testing.md
  # cts_runner uses dxc by default

# Every time a PR is pushed to, cancel any previous jobs. This
# makes us behave nicer to github and get faster turnaround times
# on PRs that are pushed to multiple times in rapid succession.
concurrency:
  group: ${{github.workflow}}-${{github.ref}}
  cancel-in-progress: ${{github.event_name == 'pull_request'}}

jobs:
  cts:
    strategy:
      fail-fast: false
      matrix:
        suite: ["API Operation", "API Validation", "Other"]
        platform: ["Windows x86_64", "Mac aarch64", "Linux x86_64"]
        include:
          # When an `include` item matches existing matrix entries, the additional items are
          # added only to the matching entries. The two lines above define the matrix, and
          # the items below attach the rest of the configuration values that apply to suites
          # and platforms.
          - suite: API Operation
            filter: "^webgpu:api,operation,"
          - suite: API Validation
            filter: "^webgpu:api,validation,"
          - suite: Other
            filter: "!^webgpu:api,operation,|^webgpu:api,validation,"

          # Windows
          - platform: Windows x86_64
            os: windows-2022
            target: x86_64-pc-windows-msvc
            backend: dx12
            shell: bash

          # Mac
          # Need to use `zsh` for `source <(cmd)` because it doesn't work in
          # ancient MacOS bash. `zsh` is a non-standard shell in GitHub
          # actions, so must be specified as a command string.
          - platform: Mac aarch64
            os: macos-14
            target: x86_64-apple-darwin
            backend: metal
            shell: zsh {0}

          # Linux
          - platform: Linux x86_64
            os: ubuntu-24.04
            target: x86_64-unknown-linux-gnu
            backend: vulkan
            shell: bash

    name: ${{ matrix.suite }} ${{ matrix.platform }}
    runs-on: ${{ matrix.os }}

    defaults:
      run:
        # Substitution of ${{ matrix.shell }} appears not to work in per-step configuration
        shell: ${{ matrix.shell }}

    steps:
      - name: checkout repo
        uses: actions/checkout@v6

      - name: Install Repo MSRV toolchain
        run: |
          rustup toolchain install ${{ env.REPO_MSRV }} --no-self-update --profile=minimal --target ${{ matrix.target }} --component llvm-tools
          rustup override set ${{ env.REPO_MSRV }}
          cargo -V

      - name: Install `cargo-llvm-cov`
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-llvm-cov

      - name: caching
        uses: Swatinem/rust-cache@v2
        with:
          # The version number can be incremented for cache busting.
          prefix-key: v3-rust-${{ hashFiles('cts_runner/revision.txt') }}
          env-vars: ImageVersion
          cache-directories: cts
          # Save the cache from the `other` job because it also runs the cts_runner
          # integration tests.
          save-if: ${{ github.ref == 'refs/heads/trunk' && matrix.suite == 'other' }}

      # We enable line numbers for panics, but that's it
      - name: disable debug
        shell: bash
        run: |
          mkdir -p .cargo
          cat <<EOF >> .cargo/config.toml
          [profile.dev]
          debug = "line-tables-only"
          EOF

      # Unlike regular `llvm-cov run`, which builds artifacts in `target/llvm-cov-target`,
      # `llvm-cov show-env` uses the regular target directory for artifacts. Because of this,
      # this job configures the `install-mesa` and `install-warp` actions with the regular
      # target directory, not the `llvm-cov-target` directory.
      #
      # The `GITHUB_ENV` file is not subject to shell expansion, so we need to un-escape the
      # `llvm-cov show-env` output.
      - name: Set llvm-cov environment
        shell: bash
        run: |
          cargo llvm-cov --no-cfg-coverage show-env | while read -r assignment; do
            key="${assignment%%=*}"
            eval "$assignment"
            echo "$key=${!key}" >> "$GITHUB_ENV"
          done

      - name: (Windows) Install DXC
        if: matrix.os == 'windows-2022'
        uses: ./.github/actions/install-dxc

      # Note: `target-dir` is intentionally different from other jobs.
      # See note above about `llvm-cov show-env`.
      - name: (Windows) Install WARP
        if: matrix.os == 'windows-2022'
        uses: ./.github/actions/install-warp
        with:
          target-dir: "target/debug"

      - name: (Linux) Install Mesa
        if: matrix.os == 'ubuntu-24.04'
        uses: ./.github/actions/install-mesa
        with:
          target-dir: "target/debug"

      # Some of these tests check stdout, so we explicitly set the backend
      # to avoid getting EGL messages in the output.
      - name: Test cts_runner
        if: matrix.suite == 'other'
        shell: bash
        run: |
          export LLVM_PROFILE_FILE=${{ github.workspace }}/target/wgpu-%p-%m.profraw
          export DENO_WEBGPU_BACKEND=${{ matrix.backend }}
          cargo --locked test -p cts_runner

      - name: Run CTS
        shell: bash
        run: cargo --locked xtask cts --llvm-cov --backend ${{ matrix.backend }} --filter '${{matrix.filter }}'

      - name: Generate coverage report
        id: coverage
        continue-on-error: true
        run: |
          set -e

          cargo --locked llvm-cov report --lcov --output-path lcov.info

      - name: Upload coverage report to Codecov
        uses: codecov/codecov-action@v7
        if: steps.coverage.outcome == 'success'
        with:
          files: lcov.info
          token: ${{ secrets.CODECOV_TOKEN }}