[DO NOT MERGE] Resolve CI: torchaudio, dynamo FutureWarning, torch 2.13 named tensors #10417
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI testing | |
| # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows | |
| on: # Trigger the workflow on push or pull request, but only for the main branch | |
| push: | |
| branches: [main] | |
| pull_request: {} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| #CI: "true" | |
| TORCH_URL_RC: "https://download.pytorch.org/whl/test/cpu/torch_test.html" | |
| TORCH_URL_NIGHTLY: "https://download.pytorch.org/whl/nightly/cpu" | |
| TORCH_URL_STABLE: "https://download.pytorch.org/whl/cpu" | |
| jobs: | |
| pytester: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ["ubuntu-22.04", "macOS-15", "windows-latest"] | |
| python-version: ["3.10"] | |
| requires: ["latest", "nightly"] # , 'oldest' | |
| suite: ["core", "ops", "grads"] | |
| include: | |
| - { os: "ubuntu-22.04", python-version: "3.10", requires: "oldest", suite: "core" } | |
| - { os: "ubuntu-22.04", python-version: "3.10", requires: "oldest", suite: "ops" } | |
| - { os: "ubuntu-22.04", python-version: "3.10", requires: "oldest", suite: "grads" } | |
| - { os: "ubuntu-24.04", python-version: "3.11", requires: "latest" } | |
| - { os: "ubuntu-24.04", python-version: "3.12", requires: "latest" } | |
| - { os: "ubuntu-24.04", python-version: "3.13", requires: "latest" } | |
| - { os: "ubuntu-24.04", python-version: "3.14", requires: "latest" } | |
| exclude: | |
| - { os: "windows-latest", suite: "ops" } | |
| - { os: "windows-latest", suite: "grads" } | |
| # Timeout: https://stackoverflow.com/a/59076067/4521646 | |
| timeout-minutes: 35 | |
| env: | |
| # torch's Windows CPU wheels carry AVX-512 instructions inside kernels compiled for the | |
| # AVX2 target, so on a runner whose CPU stops at AVX2 they fault with | |
| # STATUS_ILLEGAL_INSTRUCTION (0xc000001d) and take the xdist worker down with them. The | |
| # reported fault sits in the bfloat16 vector helpers, which is why the nanogpt and litgpt | |
| # tests are the ones that crash: they are the only ones here running a model in bfloat16. | |
| # Fall back to the unvectorized kernels on Windows until this is fixed upstream. | |
| # Ref: https://github.com/pytorch/pytorch/issues/145702 | |
| ATEN_CPU_CAPABILITY: ${{ startsWith(matrix.os, 'windows') && 'default' || '' }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "pip" | |
| - name: Setup Ubuntu | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo apt-get update -qq --fix-missing | |
| sudo apt-get install -y libblas3 liblapack3 libopenblas-dev | |
| pip install --force-reinstall numpy | |
| - name: Set min. dependencies | |
| if: matrix.requires == 'oldest' | |
| run: | | |
| for fpath in ('requirements/base.txt', 'requirements/test.txt'): | |
| req = open(fpath).read().replace('>=', '==') | |
| open(fpath, 'w').write(req) | |
| shell: python | |
| - name: set default Torch source | |
| run: echo "TORCH_URL=$TORCH_URL_STABLE" >> $GITHUB_ENV | |
| - name: switch Torch source to nightly | |
| if: matrix.requires == 'nightly' | |
| run: | | |
| echo "TORCH_URL=$TORCH_URL_NIGHTLY" >> $GITHUB_ENV | |
| echo "PIP_EXTRA_FLAG=--pre" >> $GITHUB_ENV | |
| - name: Install Mac specific dependencies | |
| if: runner.os == 'macOS' | |
| run: brew install libomp | |
| - name: Install package & dependencies | |
| run: | | |
| pip install -e . -U \ | |
| ${PIP_EXTRA_FLAG} -r requirements/test.txt \ | |
| --extra-index-url=${TORCH_URL} | |
| pip list | |
| - name: re-Install dependencies | |
| if: matrix.requires != 'nightly' | |
| run: | | |
| pip install -q wget | |
| # download script and adjust torch versions in requirements | |
| python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py | |
| for fpath in `ls requirements/*.txt`; do \ | |
| python ./adjust-torch-versions.py $fpath ${{ matrix.pytorch-version }}; \ | |
| done | |
| # re-install with adjusted torch versions | |
| pip install ${PIP_EXTRA_FLAG} -r requirements/devel.txt \ | |
| --extra-index-url=${TORCH_URL} | |
| pip list | |
| - name: Show CPU dispatch | |
| # So a green Windows run can be told apart from a run that happened to land on a CPU | |
| # the bad kernels do not fault on. Expect DEFAULT here, not AVX2. | |
| if: runner.os == 'Windows' | |
| run: python -c "import torch; print('cpu capability:', torch.backends.cpu.get_cpu_capability())" | |
| - name: Testing Local | |
| if: matrix.suite == 'core' | |
| run: | | |
| coverage run --source thunder -m \ | |
| pytest thunder/tests/ \ | |
| --ignore=thunder/tests/distributed \ | |
| --ignore=thunder/tests/test_ops.py \ | |
| --ignore=thunder/tests/test_grad.py \ | |
| -v --datefmt="%Y%m%d-%H:%M:%S.%f" \ | |
| --random-order-seed=$GITHUB_RUN_ID \ | |
| -n 4 --durations=250 | |
| - name: Testing Distributed | |
| # run all found tests in given past as standalone | |
| if: runner.os == 'Linux' && matrix.suite == 'core' | |
| run: | | |
| pytest thunder/tests/distributed/ \ | |
| -v --datefmt="%Y%m%d-%H:%M:%S.%f" \ | |
| --random-order-seed=$GITHUB_RUN_ID \ | |
| --durations=250 | |
| - name: Testing Ops | |
| if: matrix.suite == 'ops' | |
| run: | | |
| coverage run --source thunder -m \ | |
| pytest thunder/tests/test_ops.py \ | |
| -v --datefmt="%Y%m%d-%H:%M:%S.%f" \ | |
| --random-order-seed=$GITHUB_RUN_ID \ | |
| -n 4 --durations=250 | |
| - name: Testing Grads | |
| if: matrix.suite == 'grads' | |
| run: | | |
| coverage run --source thunder -m \ | |
| pytest thunder/tests/test_grad.py \ | |
| -v --datefmt="%Y%m%d-%H:%M:%S.%f" \ | |
| --random-order-seed=$GITHUB_RUN_ID \ | |
| -n 4 --durations=250 | |
| - name: Testing interpreter | |
| if: matrix.python-version != '3.10' | |
| run: | | |
| python -m pytest \ | |
| thunder/tests/test_interpreter.py \ | |
| thunder/tests/test_jit_general.py \ | |
| -v -n 4 --durations=50 --cov=thunder | |
| - name: Statistics | |
| run: | | |
| coverage report | |
| coverage xml | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./coverage.xml | |
| flags: unittests | |
| env_vars: OS,PYTHON | |
| name: codecov-umbrella | |
| fail_ci_if_error: false | |
| testing-guardian: | |
| runs-on: ubuntu-latest | |
| needs: pytester | |
| if: always() | |
| steps: | |
| - run: echo "${{ needs.pytester.result }}" | |
| - name: failing... | |
| if: needs.pytester.result == 'failure' | |
| run: exit 1 | |
| - name: cancelled or skipped... | |
| if: contains(fromJSON('["cancelled", "skipped"]'), needs.pytester.result) | |
| timeout-minutes: 1 | |
| run: sleep 90 |