Add initial configurations for the fbgemm-xpu package #194

Workflow file for this run

	name: ci

	# This workflow builds manylinux torchlib wheels as they will appear
	# in the production environment. The challenging part is to create
	# environment to properly build torchcodec which must be able to work
	# in environments with different versions of FFmpeg. We handle that by
	# pre-building few torchcodec op DLLs against few versions of FFmpeg.
	#
	# We could use quay.io/pypa/manylinux_2_28_x86_64 container instead
	# of the pytorch container in all the jobs. However, with PyPa container
	# we would need to setup Intel driver stack ourselves while pytorch
	# container handles this for us though it weighs more.

	on:
	push:
	branches:
	- main
	- 'release/**'
	pull_request:
	branches:
	- main
	- 'release/**'

	permissions: read-all

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	env:
	PR_OR_SHA: ${{ github.event.pull_request.number \|\| github.sha }}
	UNIQUE_ID: ${{ github.run_id }}

	jobs:
	# This job builds few bare minimal versions of FFmpeg with all
	# features disabled. We use the produced artifacts later building
	# torchcodec. At runtime we use other FFmpeg versions from Ubuntu
	# or Conda.
	ffmpeg:
	strategy:
	matrix:
	ffmpeg-version: [ 'n6.1.4', 'n7.1.3', 'n8.0.1' ]
	runs-on: ubuntu-24.04
	container:
	image: 'quay.io/pypa/manylinux_2_28_x86_64'
	env:
	PREFIX: ${{ github.workspace }}/_install/ffmpeg-${{ matrix.ffmpeg-version }}
	steps:
	- name: 'Checkout'
	uses: actions/checkout@v6
	with:
	repository: FFmpeg/FFmpeg
	ref: ${{ matrix.ffmpeg-version }}
	- name: 'Install prerequisites'
	run: \|
	dnf config-manager --set-enabled powertools # for nasm
	dnf install -y nasm
	- name: 'Configure FFmpeg'
	run: \|
	./configure \
	--prefix="$PREFIX" \
	--libdir="$PREFIX/lib" \
	--disable-everything \
	--disable-static \
	--disable-doc \
	--enable-shared
	- name: 'Build and install FFmpeg'
	run: \|
	make -j$(nproc) && make install
	- name: 'Verify FFmpeg'
	working-directory: ${{ env.PREFIX }}
	env:
	LD_LIBRARY_PATH: ${{ env.PREFIX }}/lib
	run: \|
	./bin/ffmpeg -version
	- name: 'Upload FFmpeg'
	uses: actions/upload-artifact@v7
	with:
	name: ffmpeg-${{ matrix.ffmpeg-version }}
	path: _install

	build:
	needs: ffmpeg
	strategy:
	matrix:
	python-version: [ '3.10', '3.11', '3.12', '3.13', '3.14' ]
	runs-on: ubuntu-24.04
	env:
	WHEELS: torchlib-${{ matrix.python-version }}-wheels
	container:
	image: 'pytorch/manylinux2_28-builder:xpu-v2.12.0-rc1'
	steps:
	- name: 'Checkout'
	uses: actions/checkout@v6
	- name: 'Download FFmpeg'
	uses: actions/download-artifact@v8
	with:
	path: ${{ github.workspace }}/_install
	# The goal is to get the following layout:
	# _install/
	# ffmpeg-x
	# ffmpeg-y
	merge-multiple: true
	- name: Install uv
	uses: astral-sh/setup-uv@v7
	with:
	python-version: ${{ matrix.python-version }}
	- name: 'Install prerequisites'
	run: \|
	rpm --import https://repositories.intel.com/gpu/intel-graphics.key
	dnf install -y libva-devel
	- name: 'Build the project'
	env:
	TORCHCODEC_FFMPEG6_INSTALL_PREFIX: ${{ github.workspace }}/_install/ffmpeg-n6.1.4
	TORCHCODEC_FFMPEG7_INSTALL_PREFIX: ${{ github.workspace }}/_install/ffmpeg-n7.1.3
	TORCHCODEC_FFMPEG8_INSTALL_PREFIX: ${{ github.workspace }}/_install/ffmpeg-n8.0.1
	run: \|
	source /opt/intel/oneapi/setvars.sh
	uv build --all --index https://download.pytorch.org/whl/xpu -vv
	- name: 'Repair wheels with auditwheel'
	run: \|
	uvx auditwheel repair dist/*.whl \
	--wheel-dir _wheels/${{ env.WHEELS }}-${{ env.PR_OR_SHA }}-${{ env.UNIQUE_ID }} \
	--exclude "*" --allow-pure-python-wheel
	- name: 'Test installing wheels'
	run: \|
	uv venv
	uv pip install \
	_wheels/${{ env.WHEELS }}-${{ env.PR_OR_SHA }}-${{ env.UNIQUE_ID }}/*.whl \
	--index https://download.pytorch.org/whl/xpu -vv
	- name: 'Test importing torchcodec-xpu with ffmpeg n6'
	env:
	LD_LIBRARY_PATH: '${{ github.workspace }}/_install/ffmpeg-n6.1.4/lib'
	run: \|
	uv run --no-project python -c "import torchcodec_xpu; print(torchcodec_xpu.__version__)"
	- name: 'Test importing torchcodec-xpu with ffmpeg n7'
	env:
	LD_LIBRARY_PATH: '${{ github.workspace }}/_install/ffmpeg-n7.1.3/lib'
	run: \|
	uv run --no-project python -c "import torchcodec_xpu; print(torchcodec_xpu.__version__)"
	- name: 'Test importing torchcodec-xpu with ffmpeg n8'
	env:
	LD_LIBRARY_PATH: '${{ github.workspace }}/_install/ffmpeg-n8.0.1/lib'
	run: \|
	uv run --no-project python -c "import torchcodec_xpu; print(torchcodec_xpu.__version__)"
	- name: 'Upload wheels'
	uses: actions/upload-artifact@v7
	with:
	name: ${{ env.WHEELS }}-${{ env.PR_OR_SHA }}-${{ env.UNIQUE_ID }}
	path: _wheels
	compression-level: 0

	# This job just merges all wheels into single archive
	compose:
	needs: build
	runs-on: ubuntu-24.04
	env:
	WHEELS: torchlib-all-wheels
	steps:
	- name: 'Download wheels'
	uses: actions/download-artifact@v8
	with:
	pattern: torchlib-*
	path: _all_wheels
	# The goal is to get the following layout:
	# _all_wheels/
	# torchlib-3.14-wheels-xxx-yyy
	# torchlib-3.13-wheels-xxx-yyy
	merge-multiple: true
	- name: 'Install prerequisites'
	run: \|
	sudo apt-get update
	sudo apt-get install -y tree
	- name: 'List downloads'
	run: \|
	tree _all_wheels/
	- name: 'Form uploads'
	run: \|
	TO=${{ env.WHEELS }}-${{ env.PR_OR_SHA }}-${{ env.UNIQUE_ID }}
	mkdir _wheels && mkdir _wheels/$TO
	find _all_wheels/ -name "-cp[0-9]-cp[0-9]-.whl" \
	-exec cp {} _wheels/$TO \;
	# Take "*-none-any.whl" from the oldest python we've built for
	find "$(ls -d _all_wheels// \| sort \| head -n 1)" -name "-none-any.whl" \
	-exec cp {} _wheels/$TO \;
	- name: 'List uploads'
	run: \|
	tree _wheels/
	- name: 'Upload wheels'
	uses: actions/upload-artifact@v7
	with:
	name: ${{ env.WHEELS }}-${{ env.PR_OR_SHA }}-${{ env.UNIQUE_ID }}
	path: _wheels/
	compression-level: 0

	test-torchcodec:
	needs: build
	strategy:
	matrix:
	ffmpeg-version: [ '6', '7', '8' ]
	# Testing in minimum and maximum supported versions of python.
	python-version: [ '3.10', '3.14' ]
	runner: [ 'bmg', 'dg2' ]
	# Default fallback value for standard matrix combinations
	cpu_fallback: [ false ]

	include:
	- ffmpeg-version: '8'
	python-version: '3.14'
	runner: 'bmg'
	cpu_fallback: true
	runs-on: ${{matrix.runner}}
	container:
	image: 'ubuntu:24.04'
	options: --device /dev/dri:/dev/dri -v /dev/dri/by-path:/dev/dri/by-path
	env:
	FORCE_CPU_FALLBACK: ${{ matrix.cpu_fallback == true }}
	WHEELS: torchlib-${{ matrix.python-version }}-wheels
	defaults:
	run:
	shell: bash -l {0} # for conda environment activation
	steps:
	- name: 'Install prerequisites'
	run: \|
	apt-get update
	apt-get install -y \
	ca-certificates \
	curl \
	git \
	gpg \
	software-properties-common \
	wget
	git config --global user.email "nobody@intel.com"
	git config --global user.name "Nobody"
	- name: 'Download wheels'
	uses: actions/download-artifact@v8
	with:
	name: ${{ env.WHEELS }}-${{ env.PR_OR_SHA }}-${{ env.UNIQUE_ID }}
	path: _wheels/
	- name: 'Checkout'
	uses: actions/checkout@v6
	with:
	path: torchlib-xpu
	- name: 'Checkout'
	uses: actions/checkout@v6
	with:
	repository: meta-pytorch/torchcodec
	ref: v0.14.0
	path: torchcodec
	- name: 'Setup Conda'
	uses: conda-incubator/setup-miniconda@v4
	with:
	# conda configuration
	auto-update-conda: false
	channels: conda-forge
	channel-priority: strict
	conda-remove-defaults: true
	miniforge-version: latest
	use-mamba: true
	# environment configuration
	activate-environment: test-env
	python-version: ${{ matrix.python-version }}
	- name: 'Install Intel drivers'
	run: \|
	add-apt-repository -y ppa:kobuk-team/intel-graphics
	apt-get install -y \
	libva2 \
	libze1 \
	libze-intel-gpu1 \
	intel-media-va-driver-non-free \
	intel-opencl-icd
	- name: 'Install FFmpeg ${{ matrix.ffmpeg-version }}'
	run: \|
	conda install ffmpeg=${{ matrix.ffmpeg-version }} pip
	- name: 'Verify FFmpeg'
	run: \|
	ffmpeg -version
	- name: 'Install torchcodec'
	run: \|
	find _wheels/ -name "*.whl"
	pip install \
	"$(ls ./_wheels/${{ env.WHEELS }}-${{ env.PR_OR_SHA }}-${{ env.UNIQUE_ID }}/torchcodec*.whl)[test]" \
	--extra-index-url https://download.pytorch.org/whl/xpu
	- name: 'Test gpu'
	run: \|
	python -c "import torch; print(torch.xpu.is_available())"
	- name: 'Patch torchcodec'
	working-directory: torchcodec
	run: \|
	git am ../torchlib-xpu/packages/torchcodec-xpu/patches/0001-Add-XPU-support-to-tests.patch
	- name: 'Test'
	working-directory: torchcodec
	run: \|
	if [ "$FORCE_CPU_FALLBACK" == "true" ] \|\| [ "${{matrix.runner}}" == "dg2" ]; then
	# On dg2 see: https://github.com/intel/media-driver/issues/1954
	known_failures="not test_get_frame_at_av1[xpu]"
	fi
	pytest -rsf -k "${known_failures}" test/

	# TODO: change this placeholder to the actual job publishing artifacts to PyPI
	publish:
	needs: [ compose, test-torchcodec ]
	runs-on: ubuntu-24.04
	env:
	WHEELS: torchlib-all-wheels
	steps:
	- name: 'Download wheels'
	uses: actions/download-artifact@v8
	with:
	name: ${{ env.WHEELS }}-${{ env.PR_OR_SHA }}-${{ env.UNIQUE_ID }}
	path: _wheels
	- name: 'List wheels'
	run: \|
	ls _wheels

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add initial configurations for the fbgemm-xpu package #194

Workflow file

Add initial configurations for the fbgemm-xpu package #194

Uh oh!

Workflow file for this run