Skip to content
Linux CUDA x86

Fix build failure on Python 3.14t without --enable-shared #4399

Sign in to view logs

Fix build failure on Python 3.14t without --enable-shared

Fix build failure on Python 3.14t without --enable-shared #4399

Workflow file for this run

.github/workflows/linux_cuda_wheel.yaml at a9002c7
name: Linux CUDA x86
on:
pull_request:
push:
branches:
- nightly
- main
- release/*
tags:
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
- v[0-9]+.[0-9]+.[0-9]+
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
cancel-in-progress: true
permissions:
id-token: write
contents: write
defaults:
run:
shell: bash -l -eo pipefail {0}
jobs:
generate-matrix:
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
with:
package-type: wheel
os: linux
test-infra-repository: pytorch/test-infra
test-infra-ref: main
with-cpu: disable
with-xpu: disable
with-rocm: disable
with-cuda: enable
build-python-only: "disable"
build:
needs: generate-matrix
strategy:
fail-fast: false
name: Build and Upload wheel
uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
with:
repository: meta-pytorch/torchcodec
ref: ""
test-infra-repository: pytorch/test-infra
test-infra-ref: main
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
pre-script: packaging/pre_build_script.sh
post-script: packaging/post_build_script.sh
smoke-test-script: packaging/fake_smoke_test.py
package-name: torchcodec
trigger-event: ${{ github.event_name }}
build-platform: "python-build-package"
build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation"
install-and-test:
runs-on: linux.g5.4xlarge.nvidia.gpu
strategy:
fail-fast: false
matrix:
# 3.10 corresponds to the minimum python version for which we build
# the wheel unless the label cliflow/binaries/all is present in the
# PR.
# For the actual release we should add that label and change this to
# include more python versions.
python-version: ['3.10']
# We test against 12.6 and 13.0 to avoid having too big of a CI matrix,
# but for releases we should add 12.8.
cuda-version: ['12.6', '13.0']
# TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325
ffmpeg-version-for-tests: ['4', '6', '7', '8']
container:
image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}"
options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility"
needs: build
steps:
- name: Setup env vars
run: |
cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g')
echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g')
echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV
- name: Check out repo
uses: actions/checkout@v6
- name: Remove src/ folder
run: bash packaging/remove_src.sh
- name: Setup miniconda using test-infra
uses: pytorch/test-infra/.github/actions/setup-miniconda@main
with:
python-version: ${{ matrix.python-version }}
# We install conda packages at the start because otherwise conda may have conflicts with dependencies.
# Note: xorg-libxau was addded to fix a problem with ffmpeg 4. We should consider removing it.
default-packages: "nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }} conda-forge::xorg-libxau"
- name: Check env, set LD_LIBRARY_PATH
run: |
${CONDA_RUN} env
${CONDA_RUN} conda info
${CONDA_RUN} nvidia-smi
${CONDA_RUN} conda list
echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV
- name: Assert ffmpeg exists
run: |
${CONDA_RUN} ffmpeg -buildconf
- name: Update pip
run: ${CONDA_RUN} python -m pip install --upgrade pip
- name: Install PyTorch
run: |
${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"
${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'
- uses: actions/download-artifact@v4
with:
name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64
path: dist/
- name: Install torchcodec from the wheel
run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"
- name: Install test dependencies
run: ${CONDA_RUN} bash packaging/install_test_dependencies.sh
- name: Run Python tests
run: |
${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest --override-ini="addopts=-v" test --tb=short
- name: Run Python benchmark
run: |
${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none
install-and-test-on-cpu-only-machine:
# This job tests that CUDA wheels work fine on CPU-only machines. Note that
# we still install a CUDA-enabled version of torch, and that's by design.
# Essentially, what we want to make sure is that
# `pip install torch torchcodec` works on CPU-only machines, and this
# command should install CUDA-enabled versions of both torch and torchcodec.
# It's critical that this job runs on a CPU-only machine.
runs-on: ubuntu-latest
needs: build
env:
PYTHON_VERSION: '3.10'
CUDA_VERSION: '12.6'
FFMPEG_VERSION: '7'
steps:
- name: Setup env vars
run: |
cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g')
echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
python_version_without_periods=$(echo "${{ env.PYTHON_VERSION }}" | sed 's/\.//g')
echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV
- name: Check out repo
uses: actions/checkout@v6
- name: Remove src/ folder
run: bash packaging/remove_src.sh
- name: Setup conda env
uses: conda-incubator/setup-miniconda@v3
with:
auto-update-conda: true
miniforge-version: latest
activate-environment: test
python-version: ${{ env.PYTHON_VERSION }}
- name: Update pip
run: python -m pip install --upgrade pip
- name: Install PyTorch
run: bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"
- uses: actions/download-artifact@v4
with:
name: meta-pytorch_torchcodec__${{ env.PYTHON_VERSION }}_cu${{ env.cuda_version_without_periods }}_x86_64
path: dist/
- name: Install torchcodec from the wheel
run: bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"
- name: Install ffmpeg
run: bash packaging/install_ffmpeg.sh ${{ env.FFMPEG_VERSION }}
- name: Install test dependencies
run: bash packaging/install_test_dependencies.sh
- name: Assert CUDA is not available
run: |
python -c "import torch; assert not torch.cuda.is_available()"
- name: Run Python tests
run: |
pytest --override-ini="addopts=-v" test --tb=short
build-docs:
runs-on: linux.g5.4xlarge.nvidia.gpu
env:
PYTHON_VERSION: '3.10'
CUDA_VERSION: '12.6'
FFMPEG_VERSION: '7'
container:
image: "pytorch/manylinux2_28-builder:cuda12.6" # must be same as env!!
options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility"
needs: build
steps:
- name: Setup env vars
run: |
cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g')
echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
python_version_without_periods=$(echo "${{ env.PYTHON_VERSION }}" | sed 's/\.//g')
echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV
- name: Check out repo
uses: actions/checkout@v6
- name: Remove src/ folder
run: bash packaging/remove_src.sh
- name: Setup miniconda using test-infra
uses: pytorch/test-infra/.github/actions/setup-miniconda@main
with:
python-version: ${{ env.PYTHON_VERSION }}
# We install conda packages at the start because otherwise conda may have conflicts with dependencies.
default-packages: "nvidia::cuda-nvrtc=${{ env.CUDA_VERSION }} nvidia::cuda-toolkit=${{ env.CUDA_VERSION }} nvidia::cuda-cudart=${{ env.CUDA_VERSION }} nvidia::cuda-driver-dev=${{ env.CUDA_VERSION }} conda-forge::ffmpeg=${{ env.FFMPEG_VERSION }}"
- name: Check env, set LD_LIBRARY_PATH
run: |
${CONDA_RUN} env
${CONDA_RUN} conda info
${CONDA_RUN} nvidia-smi
${CONDA_RUN} conda list
echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV
- name: Assert ffmpeg exists
run: |
${CONDA_RUN} ffmpeg -buildconf
- name: Update pip
run: ${CONDA_RUN} python -m pip install --upgrade pip
- name: Install PyTorch
run: |
${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"
${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'
- uses: actions/download-artifact@v4
with:
name: meta-pytorch_torchcodec__${{ env.PYTHON_VERSION }}_cu${{ env.cuda_version_without_periods }}_x86_64
path: dist/
- name: Install torchcodec from the wheel
run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"
- name: Install doc dependencies
run: |
cd docs
${CONDA_RUN} python -m pip install -r requirements.txt
- name: Build docs
run: |
cd docs
${CONDA_RUN} make html
- uses: actions/upload-artifact@v4
with:
name: Built-Docs
path: docs/build/html/
doc-preview:
runs-on: [self-hosted, linux.2xlarge]
needs: build-docs
if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'pull_request'
steps:
- uses: actions/download-artifact@v4
with:
name: Built-Docs
path: docs
# Update HTML to add the no-index tag so that search engines do not index these ephemeral docs
- name: Add no-index tag
run: |
find docs -name "*.html" -print0 | xargs -0 sed -i '/<head>/a \ \ <meta name="robots" content="noindex">';
- name: Upload docs preview
uses: seemethere/upload-artifact-s3@v5
with:
retention-days: 14
s3-bucket: doc-previews
if-no-files-found: error
path: docs
s3-prefix: meta-pytorch/torchcodec/${{ github.event.pull_request.number }}
upload-docs:
# This job uploads built docs:
# - to the `main` folder of the gh-pages branch (https://meta-pytorch.org/torchcodec/main) on every commit to the `main` branch
# - to the (e.g.) `0.10` folder in the gh-pages branch whenever a corresponding tag is pushed, like `v0.10.0` (https://meta-pytorch.org/torchcodec/0.10).
needs: build-docs
if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'push' &&
((github.ref_type == 'branch' && github.ref_name == 'main') ||
(github.ref_type == 'tag' && !contains(github.ref_name, '-rc')))
runs-on: ubuntu-latest
steps:
- name: Check out gh-pages branch
uses: actions/checkout@v6
with:
ref: gh-pages
- uses: actions/download-artifact@v4
with:
name: Built-Docs
path: docs-artifact/
- name: Update docs and push
run: |
set -euo pipefail
REF_TYPE=${{ github.ref_type }}
REF_NAME=${{ github.ref_name }}
if [[ "${REF_TYPE}" == branch ]]; then
TARGET_FOLDER="${REF_NAME}"
elif [[ "${REF_TYPE}" == tag ]]; then
# Strip the leading "v" as well as the trailing patch version. For example:
# 'v0.10.2' -> '0.10'
TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/')
fi
echo "Target Folder: ${TARGET_FOLDER}"
mkdir -p "${TARGET_FOLDER}"
rm -rf "${TARGET_FOLDER}"/*
cp -r docs-artifact/* "${TARGET_FOLDER}/"
git add "${TARGET_FOLDER}"
git config user.name 'pytorchbot'
git config user.email 'soumith+bot@pytorch.org'
git commit -m "auto-generating sphinx docs for ${TARGET_FOLDER}" || echo "No changes to commit"
git push