Build wheels and deploy #11
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will: | |
| # - Create a new Github release | |
| # - Build wheels for supported architectures | |
| # - Deploy the wheels to the Github release | |
| # - Release the static code to PyPi | |
| # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries | |
| name: Build wheels and deploy | |
| on: | |
| create: | |
| tags: | |
| - v* | |
| jobs: | |
| setup_release: | |
| name: Create Release | |
| runs-on: ubuntu-latest | |
| outputs: | |
| release-version: ${{ steps.extract_branch.outputs.branch }} | |
| steps: | |
| - name: Get the tag version | |
| id: extract_branch | |
| run: echo "branch=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Create Release | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh release create ${{ steps.extract_branch.outputs.branch }} --repo $GITHUB_REPOSITORY --title ${{ steps.extract_branch.outputs.branch }} --generate-notes | |
| shell: bash | |
| build_wheels: | |
| name: Build Wheel | |
| needs: setup_release | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Using ubuntu-22.04 instead of 24.04 for more compatibility (glibc). Ideally we'd use the | |
| # manylinux docker image, but I haven't figured out how to install CUDA on manylinux. | |
| os: [ubuntu-22.04, ubuntu-22.04-arm] | |
| python-version: ['3.10', '3.11', '3.12', '3.13'] | |
| torch-version: ['2.6.0', '2.7.1', '2.8.0', '2.9.1', '2.10.0'] | |
| cuda-version: ['11.8.0', '12.9.1', '13.0.1'] | |
| # We need separate wheels that either uses C++11 ABI (-D_GLIBCXX_USE_CXX11_ABI) or not. | |
| # Pytorch wheels currently don't use it, but nvcr images have Pytorch compiled with C++11 ABI. | |
| # Without this we get import error (undefined symbol: _ZN3c105ErrorC2ENS_14SourceLocationESs) | |
| # when building without C++11 ABI and using it on nvcr images. | |
| cxx11_abi: ['FALSE', 'TRUE'] | |
| exclude: | |
| # CUDA 11.8 is not supported by PyTorch 2.8+ | |
| - torch-version: '2.8.0' | |
| cuda-version: '11.8.0' | |
| - torch-version: '2.9.1' | |
| cuda-version: '11.8.0' | |
| - torch-version: '2.10.0' | |
| cuda-version: '11.8.0' | |
| # CUDA 13.0 is only supported by PyTorch 2.9+ | |
| - torch-version: '2.6.0' | |
| cuda-version: '13.0.1' | |
| - torch-version: '2.7.1' | |
| cuda-version: '13.0.1' | |
| - torch-version: '2.8.0' | |
| cuda-version: '13.0.1' | |
| # No aarch64 PyTorch wheels for 2.6.0, or 2.7.1+cu118 | |
| - torch-version: '2.6.0' | |
| os: ubuntu-22.04-arm | |
| - torch-version: '2.7.1' | |
| cuda-version: '11.8.0' | |
| os: ubuntu-22.04-arm | |
| # PyTorch 2.7+ pip wheels use CXX11_ABI=1 by default, no need for FALSE | |
| - torch-version: '2.7.1' | |
| cxx11_abi: 'FALSE' | |
| - torch-version: '2.8.0' | |
| cxx11_abi: 'FALSE' | |
| - torch-version: '2.9.1' | |
| cxx11_abi: 'FALSE' | |
| - torch-version: '2.10.0' | |
| cxx11_abi: 'FALSE' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.setup_release.outputs.release-version }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set CUDA and PyTorch versions | |
| run: | | |
| echo "MATRIX_CUDA_VERSION=$(echo ${{ matrix.cuda-version }} | awk -F \. {'print $1 $2'})" >> $GITHUB_ENV | |
| echo "MATRIX_TORCH_VERSION=$(echo ${{ matrix.torch-version }} | awk -F \. {'print $1 "." $2'})" >> $GITHUB_ENV | |
| echo "WHEEL_CUDA_VERSION=$(echo ${{ matrix.cuda-version }} | awk -F \. {'print $1'})" >> $GITHUB_ENV | |
| echo "MATRIX_PYTHON_VERSION=$(echo ${{ matrix.python-version }} | awk -F \. {'print $1 $2'})" >> $GITHUB_ENV | |
| - name: Free up disk space | |
| if: ${{ runner.os == 'Linux' }} | |
| # https://github.com/easimon/maximize-build-space/blob/master/action.yml | |
| # https://github.com/easimon/maximize-build-space/tree/test-report | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| - name: Set up swap space | |
| if: runner.os == 'Linux' | |
| uses: pierotofy/set-swap-space@v1.0 | |
| with: | |
| swap-size-gb: 10 | |
| - name: Install CUDA ${{ matrix.cuda-version }} | |
| if: ${{ matrix.cuda-version != 'cpu' }} | |
| uses: Jimver/cuda-toolkit@v0.2.30 | |
| id: cuda-toolkit | |
| with: | |
| cuda: ${{ matrix.cuda-version }} | |
| linux-local-args: '["--toolkit"]' | |
| method: 'network' | |
| sub-packages: '["nvcc"]' | |
| - name: Install PyTorch ${{ matrix.torch-version }}+cu${{ matrix.cuda-version }} | |
| run: | | |
| pip install --upgrade pip | |
| pip install setuptools==68.0.0 | |
| pip install typing-extensions==4.12.2 | |
| # Pick the highest available PyTorch wheel CUDA version that doesn't exceed system CUDA | |
| export TORCH_CUDA_VERSION=$(python -c "from os import environ as env; \ | |
| available = { \ | |
| '2.6': [118, 124, 126], \ | |
| '2.7': [118, 126, 128], \ | |
| '2.8': [126, 128, 129], \ | |
| '2.9': [126, 128, 130], \ | |
| '2.10': [126, 128, 130], \ | |
| }[env['MATRIX_TORCH_VERSION']]; \ | |
| sys_cuda = int(env['MATRIX_CUDA_VERSION']); \ | |
| print(max(v for v in available if v <= sys_cuda))" \ | |
| ) | |
| pip install --no-cache-dir torch==${{ matrix.torch-version }} --index-url https://download.pytorch.org/whl/cu${TORCH_CUDA_VERSION} | |
| nvcc --version | |
| python --version | |
| python -c "import torch; print('PyTorch:', torch.__version__)" | |
| python -c "import torch; print('CUDA:', torch.version.cuda)" | |
| python -c "from torch.utils import cpp_extension; print (cpp_extension.CUDA_HOME)" | |
| shell: bash | |
| - name: Build wheel | |
| run: | | |
| pip install ninja packaging wheel | |
| export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH | |
| export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH | |
| # Limit MAX_JOBS otherwise the github runner goes OOM | |
| MAX_JOBS=2 FAST_HADAMARD_TRANSFORM_FORCE_BUILD="TRUE" FAST_HADAMARD_TRANSFORM_FORCE_CXX11_ABI=${{ matrix.cxx11_abi}} python setup.py bdist_wheel --dist-dir=dist | |
| tmpname=cu${WHEEL_CUDA_VERSION}torch${MATRIX_TORCH_VERSION}cxx11abi${{ matrix.cxx11_abi }} | |
| wheel_name=$(ls dist/*whl | xargs -n 1 basename | sed "s/-/+$tmpname-/2") | |
| ls dist/*whl |xargs -I {} mv {} dist/${wheel_name} | |
| echo "wheel_name=${wheel_name}" >> $GITHUB_ENV | |
| - name: Log Built Wheels | |
| run: | | |
| ls dist | |
| - name: Get Release with tag | |
| id: get_current_release | |
| uses: joutvhu/get-release@v1 | |
| with: | |
| tag_name: ${{ needs.setup_release.outputs.release-version }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Upload Release Asset | |
| id: upload_release_asset | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ steps.get_current_release.outputs.upload_url }} | |
| asset_path: ./dist/${{env.wheel_name}} | |
| asset_name: ${{env.wheel_name}} | |
| asset_content_type: application/* | |
| publish_package: | |
| name: Publish package | |
| needs: [build_wheels] | |
| if: always() && !cancelled() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: | | |
| pip install ninja packaging setuptools wheel twine | |
| # We don't want to download anything CUDA-related here | |
| pip install torch --index-url https://download.pytorch.org/whl/cpu | |
| - name: Build core package | |
| env: | |
| FAST_HADAMARD_TRANSFORM_SKIP_CUDA_BUILD: "TRUE" | |
| run: | | |
| python setup.py sdist --dist-dir=dist | |
| - name: Deploy | |
| env: | |
| TWINE_USERNAME: "__token__" | |
| TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} | |
| run: | | |
| python -m twine upload dist/* |