linux_cuda_wheel.yaml

name: Linux CUDA x86

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
    tags:
        - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
  cancel-in-progress: true

permissions:
  id-token: write
  contents: write

defaults:
  run:
    shell: bash -l -eo pipefail {0}

jobs:
  generate-matrix:
    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
    with:
      package-type: wheel
      os: linux
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      with-cpu: disable
      with-xpu: disable
      with-rocm: disable
      with-cuda: enable
      build-python-only: "disable"
  build:
    needs: generate-matrix
    strategy:
      fail-fast: false
    name: Build and Upload wheel
    uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
    with:
      repository: meta-pytorch/torchcodec
      ref: ""
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
      pre-script: packaging/pre_build_script.sh
      post-script: packaging/post_build_script.sh
      smoke-test-script: packaging/fake_smoke_test.py
      package-name: torchcodec
      trigger-event: ${{ github.event_name }}
      build-platform: "python-build-package"
      build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation"

  install-and-test:
    runs-on: linux.g5.4xlarge.nvidia.gpu
    strategy:
      fail-fast: false
      matrix:
          # 3.10 corresponds to the minimum python version for which we build
          # the wheel unless the label cliflow/binaries/all is present in the
          # PR.
          # For the actual release we should add that label and change this to
          # include more python versions.
        python-version: ['3.10']
        # We test against 12.6 and 13.0 to avoid having too big of a CI matrix,
        # but for releases we should add 12.8.
        cuda-version: ['12.6', '13.0']
        # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325
        ffmpeg-version-for-tests: ['4.4.2', '6', '7', '8.0']

    container:
      image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}"
      options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility"
    needs: build
    steps:
      - name: Setup env vars
        run: |
          cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g')
          echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
          python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g')
          echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV

      - name: Check out repo
        uses: actions/checkout@v6

      - name: Remove src/ folder
        run: bash packaging/remove_src.sh

      - name: Setup miniconda using test-infra
        uses: pytorch/test-infra/.github/actions/setup-miniconda@main
        with:
          python-version: ${{ matrix.python-version }}
          # We install conda packages at the start because otherwise conda may have conflicts with dependencies.
          # Note: xorg-libxau was addded to fix a problem with ffmpeg 4. We should consider removing it.
          default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }} conda-forge::xorg-libxau"
      - name: Check env, set LD_LIBRARY_PATH
        run: |
          ${CONDA_RUN} env
          ${CONDA_RUN} conda info
          ${CONDA_RUN} nvidia-smi
          ${CONDA_RUN} conda list
          echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV
      - name: Assert ffmpeg exists
        run: |
          ${CONDA_RUN} ffmpeg -buildconf
      - name: Update pip
        run: ${CONDA_RUN} python -m pip install --upgrade pip
      - name: Install PyTorch
        run: |
          ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"
          ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'

      - uses: actions/download-artifact@v4
        with:
          name: meta-pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64
          path: dist/

      - name: Install torchcodec from the wheel
        run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"

      - name: Install test dependencies
        run: ${CONDA_RUN} bash packaging/install_test_dependencies.sh
      - name: Run Python tests
        run: |
          ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest --override-ini="addopts=-v" test --tb=short
      - name: Run Python benchmark
        run: |
          ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none

  build-docs:
    runs-on: linux.g5.4xlarge.nvidia.gpu
    env:
      PYTHON_VERSION: '3.10'
      CUDA_VERSION: '12.6'
      FFMPEG_VERSION: '7'
    container:
      image: "pytorch/manylinux2_28-builder:cuda12.6"  # must be same as env!!
      options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility"
    needs: build
    steps:
      - name: Setup env vars
        run: |
          cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g')
          echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV
          python_version_without_periods=$(echo "${{ env.PYTHON_VERSION }}" | sed 's/\.//g')
          echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV

      - name: Check out repo
        uses: actions/checkout@v6

      - name: Remove src/ folder
        run: bash packaging/remove_src.sh

      - name: Setup miniconda using test-infra
        uses: pytorch/test-infra/.github/actions/setup-miniconda@main
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          # We install conda packages at the start because otherwise conda may have conflicts with dependencies.
          default-packages: "nvidia/label/cuda-${{ env.CUDA_VERSION }}.0::libnpp nvidia::cuda-nvrtc=${{ env.CUDA_VERSION }} nvidia::cuda-toolkit=${{ env.CUDA_VERSION }} nvidia::cuda-cudart=${{ env.CUDA_VERSION }} nvidia::cuda-driver-dev=${{ env.CUDA_VERSION }} conda-forge::ffmpeg=${{ env.FFMPEG_VERSION }}"
      - name: Check env, set LD_LIBRARY_PATH
        run: |
          ${CONDA_RUN} env
          ${CONDA_RUN} conda info
          ${CONDA_RUN} nvidia-smi
          ${CONDA_RUN} conda list
          echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV
      - name: Assert ffmpeg exists
        run: |
          ${CONDA_RUN} ffmpeg -buildconf
      - name: Update pip
        run: ${CONDA_RUN} python -m pip install --upgrade pip
      - name: Install PyTorch
        run: |
          ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision"
          ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'

      - uses: actions/download-artifact@v4
        with:
          name: meta-pytorch_torchcodec__${{ env.PYTHON_VERSION }}_cu${{ env.cuda_version_without_periods }}_x86_64
          path: dist/

      - name: Install torchcodec from the wheel
        run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"

      - name: Install doc dependencies
        run: |
          cd docs
          ${CONDA_RUN} python -m pip install -r requirements.txt
      - name: Build docs
        run: |
          cd docs
          ${CONDA_RUN} make html
      - uses: actions/upload-artifact@v4
        with:
          name: Built-Docs
          path: docs/build/html/

  doc-preview:
    runs-on: [self-hosted, linux.2xlarge]
    needs: build-docs
    if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'pull_request'
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: Built-Docs
          path: docs

      # Update HTML to add the no-index tag so that search engines do not index these ephemeral docs
      - name: Add no-index tag
        run: |
          find docs -name "*.html" -print0 | xargs -0 sed -i '/<head>/a \ \ <meta name="robots" content="noindex">';

      - name: Upload docs preview
        uses: seemethere/upload-artifact-s3@v5
        with:
          retention-days: 14
          s3-bucket: doc-previews
          if-no-files-found: error
          path: docs
          s3-prefix: meta-pytorch/torchcodec/${{ github.event.pull_request.number }}

  upload-docs:
    # This job uploads built docs:
    # - to the `main` folder of the gh-pages branch (https://meta-pytorch.org/torchcodec/main) on every commit to the `main` branch
    # - to the (e.g.) `0.10` folder in the gh-pages branch whenever a corresponding tag is pushed, like `v0.10.0` (https://meta-pytorch.org/torchcodec/0.10).

    needs: build-docs
    if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'push' &&
        ((github.ref_type == 'branch' && github.ref_name == 'main') || github.ref_type == 'tag')
    runs-on: ubuntu-latest
    steps:
      - name: Check out gh-pages branch
        uses: actions/checkout@v6
        with:
          ref: gh-pages

      - uses: actions/download-artifact@v4
        with:
          name: Built-Docs
          path: docs-artifact/

      - name: Update docs and push
        run: |
          set -euo pipefail

          REF_TYPE=${{ github.ref_type }}
          REF_NAME=${{ github.ref_name }}

          if [[ "${REF_TYPE}" == branch ]]; then
            TARGET_FOLDER="${REF_NAME}"
          elif [[ "${REF_TYPE}" == tag ]]; then
            case "${REF_NAME}" in
              *-rc*)
                echo "Aborting upload since this is an RC tag: ${REF_NAME}"
                exit 0
                ;;
              *)
                # Strip the leading "v" as well as the trailing patch version. For example:
                # 'v0.10.2' -> '0.10'
                TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/')
                ;;
            esac
          fi
          echo "Target Folder: ${TARGET_FOLDER}"

          mkdir -p "${TARGET_FOLDER}"
          rm -rf "${TARGET_FOLDER}"/*
          cp -r docs-artifact/* "${TARGET_FOLDER}/"
          git add "${TARGET_FOLDER}"

          git config user.name 'pytorchbot'
          git config user.email 'soumith+bot@pytorch.org'
          git commit -m "auto-generating sphinx docs for ${TARGET_FOLDER}" || echo "No changes to commit"
          git push