refactor: move pg rewrite tests to a non-published crate (#2055) #1406

Workflow file for this run

.github/workflows/python-wheels.yml at 2286c7a

	# This file is loosely based on the one generated by maturin v1.9.2 using
	#
	# maturin generate-ci github
	#
	# The script has been heavily modified from there.
	# In particular, we run `maturin build` manually instead of using the `maturin-action`,
	# so we can setup the workspace dependencies (rust, llvm, conan, tket-c-api) correctly.
	#
	name: Python wheels 🐍

	on:
	push:
	branches:
	- main
	tags:
	- "tket-py-v*"
	workflow_dispatch:

	permissions:
	contents: read

	env:
	MODULE_DIR: tket/tket-py
	# Pinned version for the uv package manager
	UV_VERSION: "0.11.15"
	UV_FROZEN: 1
	# Python version used to build the wheels
	# We use abi3-compatible building, so this still supports newer Python versions
	PYTHON_VERSION: "3.10"
	# sccache config
	SCCACHE_GHA_ENABLED: "true"
	RUSTC_WRAPPER: "sccache"

	# Python version used to build the WebAssembly / Emscripten wheels.
	# This must match the pyodide ABI targeted by the emscripten version below
	# (`pyemscripten_2025_0` -> Python 3.13 / Emscripten 4.0.9).
	WASM_PYTHON_VERSION: "3.13"
	# Emscripten SDK version used for the WebAssembly wheels.
	# Must match the version used to build the tket-c-api wasm target in
	# hugrverse-env, and the pyodide ABI targeted above.
	EMSDK_VERSION: "4.0.9"
	# hugrverse-env target used to fetch the pre-built tket-c-api WebAssembly library.
	HUGRVERSE_WASM_TARGET: "emscripten_4_0_9_wasm32"
	# Rust nightly version used to build the WebAssembly wheels.
	RUST_NIGHTLY_VERSION: "nightly-2026-08-20"

	jobs:
	# Check if the tag matches the package name,
	# or if the workflow is running on a non-tag event.
	check-tag:
	name: Check tag
	runs-on: ubuntu-latest
	outputs:
	run: ${{ steps.check-tag.outputs.run }}
	steps:
	- name: Check tag
	id: check-tag
	run: \|
	echo "run=$SHOULD_RUN" >> $GITHUB_OUTPUT
	env:
	SHOULD_RUN: ${{ github.ref_type != 'tag' \|\| startsWith(github.ref, 'refs/tags/tket-py-v') }}

	build:
	name: ${{ matrix.platform.target }}
	needs: check-tag
	if: ${{ needs.check-tag.outputs.run == 'true' }}
	runs-on: ${{ matrix.platform.runner }}
	strategy:
	fail-fast: false
	matrix:
	# List of platforms to build the wheels for.
	#
	# Each entry defines:
	# - runner: The github runner to use for the build.
	# - target: The target platform used when building the rust code.
	# - manylinux: (optional) If doing manylinux-compatible wheels,
	# the manylinux version to use. Setting this option causes the wheels
	# to be built with zig.
	# - test-wheels: Whether to test installing the wheels.
	# This is only done when the target platform matches the runner platform.
	# - macos-compatibility: the minimum supported macosx version to build macos
	# wheels for. Violations (e.g. library dependencies on newer system libraries)
	# will result in CI failure.
	#
	# Note that platform support is limited by the pytket wheels' availability.
	# Since we have it as a dependency, wheels built for other platforms will be unusable.
	platform:
	# ---------------- Linux
	- runner: ubuntu-24.04-arm
	target: aarch64-unknown-linux-gnu
	manylinux: manylinux_2_28
	test-wheels: true
	- runner: ubuntu-24.04
	target: x86_64-unknown-linux-gnu
	manylinux: manylinux_2_28
	test-wheels: true
	# ---------------- Mac
	- runner: macos-14
	target: aarch64-apple-darwin
	macos-compatibility: "11.0"
	test-wheels: true
	- runner: macos-15-intel
	target: x86_64-apple-darwin
	macos-compatibility: "11.0"
	test-wheels: true
	# ---------------- Windows
	- runner: windows-2025
	target: x86_64-pc-windows-msvc
	test-wheels: true
	steps:
	- uses: actions/checkout@v7
	- uses: mozilla-actions/sccache-action@v0.0.11
	- name: Install stable toolchain
	uses: dtolnay/rust-toolchain@stable
	with:
	targets: ${{ matrix.platform.target }}
	- name: Set up uv
	uses: astral-sh/setup-uv@v7
	with:
	version: ${{ env.UV_VERSION }}
	enable-cache: true
	- uses: quantinuum/hugrverse-env/install-hugrenv-action@main
	- uses: mlugg/setup-zig@v2
	if: ${{ matrix.platform.manylinux }}
	- name: Set MACOSX_DEPLOYMENT_TARGET on macos
	if: matrix.platform.macos-compatibility
	run: echo "MACOSX_DEPLOYMENT_TARGET=${{ matrix.platform.macos-compatibility }}" >> $GITHUB_ENV

	- name: Set compatibility flag
	id: compatibility-flag
	shell: bash
	run: \|
	if [ -n "${{ matrix.platform.manylinux }}" ]; then
	WITH_ZIG_FLAG="--with ziglang --with maturin[zig]"
	COMPATIBILITY_FLAG="--compatibility ${{ matrix.platform.manylinux }} --zig"
	else
	WITH_ZIG_FLAG=""
	COMPATIBILITY_FLAG=""
	fi
	echo "COMPATIBILITY_FLAG: $COMPATIBILITY_FLAG"
	echo "WITH_ZIG_FLAG: $WITH_ZIG_FLAG"
	echo "COMPATIBILITY_FLAG=$COMPATIBILITY_FLAG" >> "$GITHUB_OUTPUT"
	echo "WITH_ZIG_FLAG=$WITH_ZIG_FLAG" >> "$GITHUB_OUTPUT"

	- name: Copy DLL to package directory (Windows)
	if: runner.os == 'Windows'
	shell: bash
	run: \|
	# Copy the DLL to the package directory so maturin can include it in the wheel
	if [ -f "$TKET_C_API_PATH/bin/tket-c-api.dll" ]; then
	cp "$TKET_C_API_PATH/bin/tket-c-api.dll" "$MODULE_DIR/tket/tket-c-api.dll"
	echo "Copied DLL to $MODULE_DIR/tket/tket-c-api.dll"
	ls -lh "$MODULE_DIR/tket/tket-c-api.dll"
	else
	echo "ERROR: DLL not found at $TKET_C_API_PATH/bin/tket-c-api.dll"
	exit 1
	fi

	- name: Build wheels
	shell: bash
	run: \|
	cd ${{ env.MODULE_DIR }}
	uv run --python ${{ env.PYTHON_VERSION }} \
	${{ steps.compatibility-flag.outputs.WITH_ZIG_FLAG }} \
	maturin build ${{ steps.compatibility-flag.outputs.COMPATIBILITY_FLAG }} \
	--release \
	--out ../../dist \
	--target ${{ matrix.platform.target }} \
	--strip

	- name: Build free-threaded wheels
	# TODO: Waiting for hugr 0.16.1 / 0.17.0 to release python 3.14 free-threaded wheels.
	# <https://github.com/Quantinuum/hugr/pull/3021>
	if: false
	shell: bash
	run: \|
	cd ${{ env.MODULE_DIR }}
	uv run --python ${{ env.PYTHON_VERSION }}+freethreaded \
	${{ steps.compatibility-flag.outputs.WITH_ZIG_FLAG }} \
	maturin build \
	${{ steps.compatibility-flag.outputs.COMPATIBILITY_FLAG }} \
	--release \
	--out ../../dist \
	--target ${{ matrix.platform.target }} \
	--strip

	- name: Repair macOS wheels with delocate
	if: runner.os == 'macOS'
	shell: bash
	run: \|
	echo "Repairing macOS wheels with delocate..."
	export DYLD_LIBRARY_PATH="$HUGRENV_PATH/lib:$HUGRENV_PATH/lib64:$DYLD_LIBRARY_PATH"
	ls -lh dist \|\| true
	MAC_WHEELS=$(ls dist/macosx.whl 2>/dev/null \|\| true)
	if [ -z "$MAC_WHEELS" ]; then
	echo "No macOS wheels found to repair; skipping delocate."
	exit 0
	fi
	uvx --from delocate delocate-listdeps $MAC_WHEELS \|\| true
	uvx --from delocate delocate-wheel -v -w dist/repaired $MAC_WHEELS
	rm -v -f $MAC_WHEELS
	mv -v dist/repaired/*.whl dist/
	rmdir -v dist/repaired \|\| true
	echo "Repaired wheels:"
	ls -lh dist/macosx.whl

	- name: Upload wheels
	uses: actions/upload-artifact@v7
	with:
	name: wheels-${{ matrix.platform.target }}
	path: dist
	if-no-files-found: error

	- name: Test installing the built wheels
	shell: bash
	run: \|
	echo "Testing the newly built ${{ matrix.platform.target }} wheels..."
	if [ "$TEST_INSTALL" = "true" ]; then
	# Remove original libraries from the build environment to ensure that
	# the test installation is actually using the built wheels, and not picking up
	# the libraries from the hugrverse environment.
	rm -rf $HUGRENV_PATH;
	unset HUGRENV_PATH;
	unset TKET_C_API_PATH;
	unset LLVM_SYS_211_PREFIX;

	# Import tket from within a fresh environment/directory rather than the source
	# directory to avoid source conflicts (the repaired wheel has modified library paths)
	tmpdir=$(mktemp -d);
	cd $tmpdir;

	# To avoid interference from the existing environment, use a python venv.
	# Note that this approach is chosen because some unwanted behaviour was seen
	# on some platforms when using uv directly.
	uv run python -m venv venv

	if [ -d "venv/Scripts" ]; then
	venv_python="venv/Scripts/python"
	venv_pip="venv/Scripts/pip"
	else
	venv_python="venv/bin/python"
	venv_pip="venv/bin/pip"
	fi

	# First install the pure python packages from this commit; this avoids fetching them
	# from pypi and providing false positives or false negatives in the presence of a conflict.
	${venv_pip} install ${GITHUB_WORKSPACE}/tket/tket-eccs
	${venv_pip} install ${GITHUB_WORKSPACE}/tket/tket-exts

	# Install the built wheel
	wheel_path=$(ls "$GITHUB_WORKSPACE/dist/"*.whl \| head -n 1)
	${venv_pip} install ${wheel_path}

	# Try importing tket to verify that the installation works.
	# Although not a comprehensive test (TODO), this at least verifies
	# that the dynamic libraries are correctly linked and can be loaded.
	${venv_python} -c "import tket; print(tket.__file__);"

	# Now return to the original workspace for further steps.
	cd $GITHUB_WORKSPACE;
	fi
	uvx twine check --strict dist/*
	env:
	WORKSPACE: ${{ github.workspace }}
	TEST_INSTALL: ${{ matrix.platform.test-wheels }}

	wasm:
	name: wasm32-unknown-emscripten
	needs: check-tag
	if: ${{ needs.check-tag.outputs.run == 'true' }}
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: mozilla-actions/sccache-action@v0.0.11
	- uses: actions/setup-python@v7
	with:
	python-version: ${{ env.WASM_PYTHON_VERSION }}
	- name: Install nightly Rust toolchain
	uses: dtolnay/rust-toolchain@stable
	with:
	toolchain: ${{ env.RUST_NIGHTLY_VERSION }}
	- name: Add required rust targets and components
	run: \|
	rustup override set ${{ env.RUST_NIGHTLY_VERSION }}
	rustup target add wasm32-unknown-emscripten
	rustup component add rust-src --toolchain ${{ env.RUST_NIGHTLY_VERSION }}
	- name: Install Node.js for emsdk
	uses: actions/setup-node@v7
	with:
	node-version: "18"
	- name: Setup emsdk manually
	# The emscripten version must match both the version used to build the
	# tket-c-api wasm target in hugrverse-env and the pyodide ABI.
	# See <https://doc.rust-lang.org/beta/rustc/platform-support/wasm32-unknown-emscripten.html>
	# and <https://github.com/pyodide/pyodide/blob/main/docs/development/abi.md>
	run: \|
	git clone --depth 1 --revision=e5bd3d0874e302a18f13c5b41f5bacf9a40c8e59 https://github.com/emscripten-core/emsdk.git
	cd emsdk
	./emsdk install ${{ env.EMSDK_VERSION }}
	./emsdk activate ${{ env.EMSDK_VERSION }}
	- name: Set up uv
	uses: astral-sh/setup-uv@v7
	with:
	version: ${{ env.UV_VERSION }}
	enable-cache: true
	# Fetch the pre-built tket-c-api WebAssembly library from hugrverse-env.
	# `llvm` has no wasm build, so we only request the `tket` package.
	- uses: quantinuum/hugrverse-env/install-hugrenv-action@main
	with:
	packages: tket
	target: ${{ env.HUGRVERSE_WASM_TARGET }}

	# Emscripten's ABI is unstable, so we pass `-Zbuild-std` to rebuild Rust
	# std with the specific emscripten version we are using, as indicated in
	# <https://doc.rust-lang.org/beta/rustc/platform-support/wasm32-unknown-emscripten.html#emscripten-abi-compatibility>
	- name: Build wasm wheels
	run: \|
	source ../../emsdk/emsdk_env.sh
	uvx maturin build \
	-Z build-std \
	--release \
	-i python${{ env.WASM_PYTHON_VERSION }} \
	--target wasm32-unknown-emscripten \
	-o ../../dist \
	-v
	working-directory: ${{ env.MODULE_DIR }}
	env:
	CARGO_BUILD_TARGET: wasm32-unknown-emscripten
	TARGET: wasm32-unknown-emscripten
	CFLAGS: -fPIC
	CXXFLAGS: -fPIC

	- name: Upload wheels
	uses: actions/upload-artifact@v7
	with:
	name: wheels-wasm32-unknown-emscripten
	path: dist
	if-no-files-found: error

	sdist:
	needs: check-tag
	if: ${{ needs.check-tag.outputs.run == 'true' }}
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: mozilla-actions/sccache-action@v0.0.11
	- name: Set up uv
	uses: astral-sh/setup-uv@v7
	with:
	version: ${{ env.UV_VERSION }}
	enable-cache: true
	- name: Build sdist
	run: \|
	cd ${{ env.MODULE_DIR }}
	uv run --no-project \
	--python ${{ env.PYTHON_VERSION }} \
	--with maturin \
	maturin sdist \
	--out ../../dist \
	--manifest-path Cargo.toml
	- name: Upload sdist
	uses: actions/upload-artifact@v7
	with:
	name: wheels-sdist
	path: dist
	if-no-files-found: error

	release:
	name: Release
	runs-on: ubuntu-latest
	if: ${{ github.ref_type == 'tag' && startsWith(github.ref, 'refs/tags/tket-py-v') }}
	needs: [build, sdist]
	environment: release
	permissions:
	# Use to sign the release artifacts
	id-token: write
	# Used to upload release artifacts
	contents: write
	# Used to generate artifact attestation
	attestations: write
	steps:
	- uses: actions/checkout@v7
	- name: Check if the tag has an associated release
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	if ! gh release view ${{ github.ref_name }}; then
	echo "No release found for tag ${{ github.ref_name }}"
	exit 1
	fi
	- uses: actions/download-artifact@v8
	with:
	path: dist
	merge-multiple: true
	- name: Generate artifact attestation
	uses: actions/attest@v4.2.2
	with:
	subject-path: "dist/*"
	- name: Prepare distributions for PyPI
	run: \|
	# PyPI doesn't support wasm wheels, so pick all non-emscripten wheels
	# and put them in a separate directory for publishing.
	mkdir pypi-dist
	find dist -maxdepth 1 -type f \
	$ -name '.tar.gz' -o \( -name '.whl' ! -name 'emscripten.whl' $ \) \
	-exec cp -- {} pypi-dist/ \;
	- name: Report
	run: \|
	echo "Publishing to PyPI..."
	echo "Based on the following workflow variables, this is a new version tag push:"
	echo " - event_name: ${{ github.event_name }}"
	echo " - ref_type: ${{ github.ref_type }}"
	echo " - ref: ${{ github.ref }}"
	- name: Publish to PyPI
	uses: pypa/gh-action-pypi-publish@release/v1
	with:
	packages-dir: pypi-dist
	verbose: true
	skip-existing: true

	- name: Upload wheels to the release and un-draft it
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	# Upload wheels to the release assets
	gh release upload ${{ github.ref_name }} dist/*
	# Un-draft the release on GitHub now that the package has been published to PyPI
	gh release edit ${{ github.ref_name }} --draft=false

Navigation Menu

refactor: move pg rewrite tests to a non-published crate (#2055) #1406

Workflow file

refactor: move pg rewrite tests to a non-published crate (#2055) #1406

Uh oh!

Workflow file for this run