[CI] adapt .github/workflows/*.yml for MetaxGPU new #51

Workflow file for this run

.github/workflows/gpu-smoke.yml at b5768b1

	name: GPU Smoke

	permissions:
	contents: read
	checks: read
	pull-requests: read

	on:
	push:
	branches: [dev, main, testbed]
	tags: ["v*"]
	pull_request:
	branches: [dev, main, testbed]
	types: [opened, synchronize, reopened, ready_for_review]
	workflow_dispatch:

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	jobs:
	ci-prereq:
	runs-on: ubuntu-latest
	outputs:
	skip: ${{ steps.check.outputs.skip }}
	steps:
	- name: Wait for CI prerequisites
	id: check
	env:
	GH_TOKEN: ${{ github.token }}
	EVENT_NAME: ${{ github.event_name }}
	EVENT_ACTION: ${{ github.event.action \|\| '' }}
	IS_DRAFT: ${{ github.event.pull_request.draft \|\| false }}
	HEAD_SHA: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	run: \|
	set -euo pipefail

	if [[ "$EVENT_NAME" != "pull_request" ]]; then
	echo "skip=false" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	if [[ "$IS_DRAFT" == "true" ]]; then
	echo "Draft PR; skipping GPU smoke."
	echo "skip=true" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	required_checks=(pre-commit gitleaks actionlint)
	deadline=$((SECONDS + 900))

	while true; do
	checks_json=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs")
	latest_checks=$(echo "$checks_json" \| jq '
	.check_runs
	\| group_by(.name)
	\| map(max_by(.id))
	')

	pending="false"
	failed=""

	for check_name in "${required_checks[@]}"; do
	status=$(echo "$latest_checks" \| jq -r --arg name "$check_name" '
	map(select(.name == $name)) \| first \| .status // ""
	')
	conclusion=$(echo "$latest_checks" \| jq -r --arg name "$check_name" '
	map(select(.name == $name)) \| first \| .conclusion // ""
	')

	if [[ -z "$status" \|\| "$status" != "completed" ]]; then
	pending="true"
	continue
	fi

	if [[ "$conclusion" == "skipped" ]]; then
	pending="true"
	continue
	fi

	if [[ "$conclusion" != "success" ]]; then
	failed="$check_name"
	break
	fi
	done

	title_status=$(echo "$latest_checks" \| jq -r '
	map(select(.name == "validate-pr-title")) \| first \| .status // ""
	')
	title_conclusion=$(echo "$latest_checks" \| jq -r '
	map(select(.name == "validate-pr-title")) \| first \| .conclusion // ""
	')
	if [[ -n "$title_status" && "$title_status" == "completed" && "$title_conclusion" != "success" && "$title_conclusion" != "skipped" ]]; then
	failed="validate-pr-title"
	elif [[ -n "$title_status" && "$title_status" != "completed" ]]; then
	pending="true"
	fi

	if [[ -n "$failed" ]]; then
	echo "CI prerequisite failed: $failed"
	echo "skip=true" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	if [[ "$pending" == "false" ]]; then
	break
	fi

	if (( SECONDS >= deadline )); then
	echo "Timed out waiting for CI prerequisites."
	echo "skip=true" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	sleep 10
	done

	if [[ "$EVENT_ACTION" == "ready_for_review" ]]; then
	CHECKS=$(echo "$latest_checks" \| jq '
	[
	.[]
	\| select(.name == "gpu-smoke")
	\| .conclusion
	]
	')
	ALL_SUCCESS=$(echo "$CHECKS" \| jq 'all(. == "success")')
	COUNT=$(echo "$CHECKS" \| jq 'length')
	if [[ "$ALL_SUCCESS" == "true" && "$COUNT" -ge 1 ]]; then
	echo "GPU smoke already passed for SHA $HEAD_SHA; skipping"
	echo "skip=true" >> "$GITHUB_OUTPUT"
	exit 0
	fi
	fi

	echo "skip=false" >> "$GITHUB_OUTPUT"

	security-policy:
	needs: ci-prereq
	if: ${{ github.repository == 'MetaX-MACA/TileOPs-Metax' && needs.ci-prereq.outputs.skip != 'true' }}
	runs-on: ubuntu-latest
	outputs:
	is_fork: ${{ steps.policy.outputs.is_fork }}
	allow_fast_path: ${{ steps.policy.outputs.allow_fast_path }}
	skip_gpu_smoke: ${{ steps.policy.outputs.skip_gpu_smoke }}
	scope: ${{ steps.policy.outputs.scope }}
	pytest_targets: ${{ steps.policy.outputs.pytest_targets }}
	reason: ${{ steps.policy.outputs.reason }}
	steps:
	- name: Checkout test files
	uses: actions/checkout@v4
	with:
	repository: ${{ github.event.pull_request.head.repo.full_name \|\| github.repository }}
	ref: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	persist-credentials: false
	fetch-depth: 1
	sparse-checkout: \|
	tests/ops

	- name: Determine GPU policy and pytest scope
	id: policy
	env:
	GH_TOKEN: ${{ github.token }}
	EVENT_NAME: ${{ github.event_name }}
	PR_NUMBER: ${{ github.event.pull_request.number \|\| '' }}
	HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name \|\| github.repository }}
	BASE_REPO: ${{ github.repository }}
	AUTHOR_ASSOC: ${{ github.event.pull_request.author_association \|\| '' }}
	run: \|
	set -euo pipefail

	add_target() {
	local target="$1"
	[[ -e "$target" ]] \|\| return 0
	if [[ -z "${seen_targets[$target]:-}" ]]; then
	selected_targets+=("$target")
	seen_targets["$target"]=1
	fi
	}

	is_fork="false"
	allow_fast_path="true"
	skip_gpu_smoke="false"
	scope="full-smoke"
	pytest_targets="tests"
	reason="GPU smoke will run."

	# `is_fork` gates the downstream runtime isolation: when true, the
	# gpu-smoke job refuses to write persistent state under
	# /home/ci-runner and instead runs from a disposable per-run dir.
	# A repo member opening a PR from their personal fork technically
	# satisfies head != base, but they already have push access to the
	# base repo — treating their PR as untrusted forces them to
	# round-trip through a fresh venv rebuild every run, for no real
	# security gain. Keep external contributors on the isolated path;
	# promote OWNER/MEMBER/COLLABORATOR authors back onto the trusted
	# runtime layout.
	if [[ "$EVENT_NAME" == "pull_request" && "$HEAD_REPO" != "$BASE_REPO" ]]; then
	case "$AUTHOR_ASSOC" in
	OWNER\|MEMBER\|COLLABORATOR)
	is_fork="false"
	echo "Member-fork PR (author_association=${AUTHOR_ASSOC}); treating as trusted."
	;;
	*)
	is_fork="true"
	;;
	esac
	fi

	if [[ "$EVENT_NAME" == "pull_request" ]]; then
	mapfile -t changed_files < <(
	gh api "repos/${BASE_REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename'
	)

	echo "Changed files (${#changed_files[@]}):"
	printf ' - %s\n' "${changed_files[@]}"

	# Fast path: if no Python files changed, GPU smoke is unnecessary.
	# For fork PRs, also check for high-risk files (workflow, build
	# config, etc.) that must still trigger GPU smoke even without
	# Python changes to avoid security bypass.
	has_python="false"
	has_high_risk="false"
	for path in "${changed_files[@]}"; do
	if [[ "$path" == *.py ]]; then
	has_python="true"
	fi
	# if [[ "$is_fork" == "true" ]]; then
	# case "$path" in
	# .github/workflows/\|pyproject.toml\|requirements.txt\|requirements.in\|setup.py\|setup.cfg\|Dockerfile\|scripts/bootstrap\|scripts/install)
	# has_high_risk="true"
	# ;;
	# esac
	# fi
	# TODO: Temporarily allow workflow changes to trigger GPU smoke for testing.
	case "$path" in
	.github/workflows/\|pyproject.toml\|requirements.txt\|requirements.in\|setup.py\|setup.cfg\|Dockerfile\|scripts/bootstrap\|scripts/install)
	has_high_risk="true"
	;;
	esac
	done
	if [[ "$has_python" == "false" && "$has_high_risk" == "false" ]]; then
	skip_gpu_smoke="true"
	reason="No Python files changed; GPU smoke skipped."
	{
	echo "skip_gpu_smoke=$skip_gpu_smoke"
	echo "is_fork=$is_fork"
	echo "allow_fast_path=$allow_fast_path"
	echo "scope=skip"
	echo "pytest_targets="
	echo "reason=$reason"
	} >> "$GITHUB_OUTPUT"
	exit 0
	fi
	if [[ "$has_python" == "false" && "$has_high_risk" == "true" ]]; then
	# Fork PR changed high-risk build/infra files but no Python.
	# Run full GPU smoke to validate the environment is not broken.
	skip_gpu_smoke="false"
	allow_fast_path="false"
	scope="full-smoke"
	pytest_targets="tests"
	reason="Fork PR changed high-risk file(s) without Python changes; running full GPU smoke."
	{
	echo "skip_gpu_smoke=$skip_gpu_smoke"
	echo "is_fork=$is_fork"
	echo "allow_fast_path=$allow_fast_path"
	echo "scope=$scope"
	echo "pytest_targets=$pytest_targets"
	echo "reason=$reason"
	} >> "$GITHUB_OUTPUT"
	exit 0
	fi

	scope="skip"
	declare -A seen_targets=()
	selected_targets=()

	for path in "${changed_files[@]}"; do
	if [[ "$is_fork" == "true" ]]; then
	case "$path" in
	.github/ISSUE_TEMPLATE/\|.github/pull_request_template.md\|tileops/ops/.py\|tests/ops/test_.py\|.md)
	;;
	.github/workflows/\|pyproject.toml\|requirements.txt\|requirements.in\|setup.py\|setup.cfg\|Dockerfile\|scripts/bootstrap\|scripts/install)
	allow_fast_path="false"
	reason="Fork PR changed a high-risk file: $path"
	;;
	*)
	allow_fast_path="false"
	reason="Fork PR changed a file outside the future fast-path policy: $path"
	;;
	esac
	fi

	case "$path" in
	.github/ISSUE_TEMPLATE/\|.github/pull_request_template.md\|.github/workflows/auto-label.yml\|.github/workflows/stale-issues.yml\|.md)
	;;
	tests/ops/test_*.py)
	add_target "$path"
	if [[ "$scope" == "skip" ]]; then
	scope="targeted"
	fi
	;;
	*)
	scope="full-smoke"
	reason="Shared or unsupported path $path; falling back to full smoke."
	break
	;;
	esac
	done

	if [[ "$scope" == "skip" ]]; then
	if [[ "$allow_fast_path" == "true" ]]; then
	# All changed files matched docs/metadata patterns; skip GPU smoke
	# for both trusted and fork PRs.
	skip_gpu_smoke="true"
	pytest_targets=""
	reason="Docs-only or metadata-only change; GPU smoke skipped."
	elif [[ "$is_fork" == "true" ]]; then
	scope="full-smoke"
	pytest_targets="tests"
	reason="${reason}; current policy still runs GPU smoke."
	else
	skip_gpu_smoke="true"
	pytest_targets=""
	reason="Docs-only or metadata-only change; GPU smoke skipped."
	fi
	elif [[ "$scope" == "targeted" ]]; then
	if [[ "${#selected_targets[@]}" -eq 0 ]]; then
	scope="full-smoke"
	pytest_targets="tests"
	reason="No mapped tests were collected; falling back to full smoke."
	else
	pytest_targets="${selected_targets[*]}"
	if [[ "$is_fork" == "true" && "$allow_fast_path" != "true" ]]; then
	reason="${reason}; current policy still runs targeted GPU smoke."
	else
	reason="Running targeted GPU smoke tests."
	fi
	fi
	fi
	fi

	{
	echo "is_fork=$is_fork"
	echo "allow_fast_path=$allow_fast_path"
	echo "skip_gpu_smoke=$skip_gpu_smoke"
	echo "scope=$scope"
	echo "pytest_targets=$pytest_targets"
	echo "reason=$reason"
	} >> "$GITHUB_OUTPUT"

	gpu-smoke:
	needs: [ci-prereq, security-policy]
	if: ${{ github.repository == 'MetaX-MACA/TileOPs-Metax' && needs.ci-prereq.outputs.skip != 'true' && needs.security-policy.outputs.skip_gpu_smoke != 'true' }}
	runs-on: tileops-metax-runner
	steps:
	- name: Checkout code for fork PR
	if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
	uses: actions/checkout@v4
	with:
	repository: ${{ github.event.pull_request.head.repo.full_name }}
	ref: ${{ github.event.pull_request.head.sha }}
	persist-credentials: false
	fetch-depth: 1

	- name: Checkout code for trusted branch
	if: ${{ github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository }}
	uses: actions/checkout@v4
	with:
	repository: ${{ github.repository }}
	ref: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	persist-credentials: false
	fetch-depth: 1

	- name: Install system dependencies
	run: \|
	sudo apt-get update && sudo apt-get install -y git
	# Trusted actions checkout: must run AFTER the workspace checkouts so
	# their `git clean -ffdx` pass does not wipe `.trusted/`. Anything
	# loaded via `uses: ./...` from here on MUST resolve through
	# `.trusted/` so that fork-controlled PR code cannot substitute the
	# composite action.
	#
	# Ref selection by trust level. `is_fork` alone is too coarse: a repo
	# member may open a PR from their personal fork, which still counts as
	# a fork PR but the author already has push access to the base repo.
	# Combine repo equality with `author_association` so repo-owned CI
	# changes can self-validate on the PR that introduces them, while
	# external contributors are kept on the trusted base ref.
	#
	# - external fork PR (author not OWNER/MEMBER/COLLABORATOR)
	# → `base.sha` (authoritative; external author can't alter)
	# - same-repo PR, member-fork PR, or push
	# → head sha (author already has write access to base repo)
	- name: Checkout trusted actions (base repo)
	uses: actions/checkout@v4
	with:
	repository: ${{ github.repository }}
	ref: ${{ (needs.security-policy.outputs.is_fork == 'true' && !contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association)) && github.event.pull_request.base.sha \|\| github.event.pull_request.head.sha \|\| github.sha }}
	path: .trusted
	persist-credentials: false
	fetch-depth: 1
	sparse-checkout: \|
	.github/actions
	scripts/ci_venv_hash.py
	pyproject.toml

	- name: Reclaim runner disk
	uses: ./.trusted/.github/actions/reclaim-runner-disk
	with:
	reclaim-below-gib: "10"
	cache-trim-cooldown-minutes: "120"
	cache-dirs: \|
	${{ runner.temp }}/ci-cache/triton
	${{ runner.temp }}/ci-cache/pip
	${{ runner.temp }}/ci-cache/wheels
	${{ runner.temp }}/ci-cache/tilelang/tmp
	atomic-cache-dirs: \|
	${{ runner.temp }}/ci-cache/tilelang/autotuner
	df-paths: "${{ runner.temp }} ${{ runner.temp }}/ci-cache"
	# Daily runner-maintenance.yml owns the destructive autotuner
	# age-trim; per-PR runs only need sentinel-repair (still
	# unconditional) so they don't evict tuning artefacts the next
	# PR would otherwise reuse.
	skip-atomic-age-trim: "true"

	- name: Resolve runtime state
	env:
	IS_FORK: ${{ needs.security-policy.outputs.is_fork }}
	run: \|
	set -euo pipefail

	# Add conda to PATH
	export PATH="/opt/conda/bin:${PATH}"

	# Set MACA environment variables
	export USE_MACA=ON
	# Narrow the venv cache key to sections that actually drive the
	# installed package set. Hashing the whole pyproject.toml means a
	# tweak to [tool.pytest] or [tool.ruff] invalidates every warm
	# fork-PR venv, defeating #996's fast path for no packaging
	# reason. See scripts/ci_venv_hash.py for the parsed sections and
	# the full-file fallback contract.
	#
	# Trust note: always run the hash script from `.trusted/` (the
	# sparse-checked-out base/head ref selected above). This prevents
	# external fork PRs from executing attacker-controlled Python on
	# the self-hosted runner before the IS_FORK runtime isolation
	# branches. We also hash the `.trusted/` pyproject.toml so the
	# computed key identifies the trusted venv that was actually
	# installed from that same pyproject, not the fork's edits.
	PYPROJECT_HASH=$(python .trusted/scripts/ci_venv_hash.py .trusted/pyproject.toml)
	HASH_INPUT="$(
	{
	echo "python=3.11"
	echo "install=PIP_NO_BUILD_ISOLATION=1 pip install -e '.[dev]'"
	echo "pyproject=${PYPROJECT_HASH}"
	} \| sha256sum \| cut -c1-16
	)"
	VENV_PREFIX="tileops_ci_venv"
	VENV_DIR="${VENV_PREFIX}_${HASH_INPUT}"
	TRUSTED_VENV_PATH="${{ runner.tool_cache }}/${VENV_DIR}"
	TRUSTED_CACHE_ROOT="${RUNNER_TEMP}/ci-cache"
	FORK_CAN_COPY_TRUSTED_VENV="false"

	if [[ "$IS_FORK" == "true" ]]; then
	RUNTIME_ROOT="${RUNNER_TEMP}/gpu-smoke-${{ github.run_id }}-${{ github.run_attempt }}"
	# Per-run isolated cache: rsync from trusted cache into temp dir.
	# Fork PRs never write to any persistent shared state.
	TILELANG_CACHE_DIR="${RUNTIME_ROOT}/tilelang/cache"
	TILELANG_TMP_DIR="${TILELANG_CACHE_DIR}/tmp"
	TRITON_CACHE_DIR="${RUNTIME_ROOT}/triton/cache"
	PIP_CACHE_DIR="${RUNTIME_ROOT}/pip/cache"
	WHEEL_DIR="${RUNTIME_ROOT}/wheels"
	VENV_PATH="${RUNTIME_ROOT}/venv"
	# Divergence guard: TRUSTED_VENV_PATH was derived from the trusted
	# base-repo pyproject.toml hash. A fork PR may have edited its own
	# workspace pyproject.toml (added/removed dependencies) without
	# that touching the base-repo file. If we copy the trusted venv
	# for such a fork, the installed package set silently disagrees
	# with `.[dev]` in the fork's workspace. Re-run the (trusted) hash
	# script against the fork's workspace pyproject.toml and only
	# fast-path when both hashes match; otherwise fall through to a
	# fresh isolated pip install below.
	WORKSPACE_PYPROJECT_HASH=$(python .trusted/scripts/ci_venv_hash.py ./pyproject.toml)
	echo "WORKSPACE_PYPROJECT_HASH=${WORKSPACE_PYPROJECT_HASH}"
	echo "TRUSTED_PYPROJECT_HASH=${PYPROJECT_HASH}"
	if [[ -x "${TRUSTED_VENV_PATH}/bin/python" && "${WORKSPACE_PYPROJECT_HASH}" == "${PYPROJECT_HASH}" ]]; then
	FORK_CAN_COPY_TRUSTED_VENV="true"
	VENV_REUSED="true"
	else
	if [[ -x "${TRUSTED_VENV_PATH}/bin/python" ]]; then
	echo "Fork pyproject diverges from trusted base; forcing fresh isolated venv install"
	fi
	VENV_REUSED="false"
	fi
	mkdir -p "${TILELANG_CACHE_DIR}" "${TILELANG_TMP_DIR}" "${TRITON_CACHE_DIR}" "${PIP_CACHE_DIR}" "${WHEEL_DIR}"

	# Copy trusted compile caches into the isolated per-run dir.
	# rsync is fast for warm caches (<5s); fork state is fully disposable.
	TRUSTED_WHEEL_DIR="${TRUSTED_CACHE_ROOT}/wheels"
	copy_start=$SECONDS
	if command -v rsync >/dev/null 2>&1; then
	rsync -a "${TRUSTED_CACHE_ROOT}/tilelang/" "${TILELANG_CACHE_DIR}/" 2>/dev/null \|\| true
	rsync -a "${TRUSTED_CACHE_ROOT}/triton/" "${TRITON_CACHE_DIR}/" 2>/dev/null \|\| true
	rsync -a "${TRUSTED_CACHE_ROOT}/pip/" "${PIP_CACHE_DIR}/" 2>/dev/null \|\| true
	if [[ -d "${TRUSTED_WHEEL_DIR}" ]]; then
	rsync -a "${TRUSTED_WHEEL_DIR}/" "${WHEEL_DIR}/" 2>/dev/null \|\| true
	fi
	else
	cp -a "${TRUSTED_CACHE_ROOT}/tilelang/." "${TILELANG_CACHE_DIR}/" 2>/dev/null \|\| true
	cp -a "${TRUSTED_CACHE_ROOT}/triton/." "${TRITON_CACHE_DIR}/" 2>/dev/null \|\| true
	cp -a "${TRUSTED_CACHE_ROOT}/pip/." "${PIP_CACHE_DIR}/" 2>/dev/null \|\| true
	if [[ -d "${TRUSTED_WHEEL_DIR}" ]]; then
	cp -a "${TRUSTED_WHEEL_DIR}/." "${WHEEL_DIR}/" 2>/dev/null \|\| true
	fi
	fi
	copy_elapsed=$(( SECONDS - copy_start ))
	echo "Cache copy completed in ${copy_elapsed}s"
	else
	RUNTIME_ROOT="/home/ci-runner"
	TILELANG_CACHE_DIR="${TRUSTED_CACHE_ROOT}/tilelang"
	TILELANG_TMP_DIR="${TILELANG_CACHE_DIR}/tmp"
	TRITON_CACHE_DIR="${TRUSTED_CACHE_ROOT}/triton"
	PIP_CACHE_DIR="${TRUSTED_CACHE_ROOT}/pip"
	WHEEL_DIR="${TRUSTED_CACHE_ROOT}/wheels"

	mkdir -p "${TILELANG_CACHE_DIR}" "${TILELANG_TMP_DIR}" "${TRITON_CACHE_DIR}" "${PIP_CACHE_DIR}" "${WHEEL_DIR}"

	VENV_PATH="${TRUSTED_VENV_PATH}"
	if [[ -x "${VENV_PATH}/bin/python" ]]; then
	VENV_REUSED="true"
	else
	VENV_REUSED="false"
	fi
	fi

	{
	echo "PATH=${PATH}"
	echo "USE_MACA=ON"
	echo "RUNTIME_ROOT=${RUNTIME_ROOT}"
	echo "TILELANG_CACHE_DIR=${TILELANG_CACHE_DIR}"
	echo "TILELANG_TMP_DIR=${TILELANG_TMP_DIR}"
	echo "TRITON_CACHE_DIR=${TRITON_CACHE_DIR}"
	echo "PIP_CACHE_DIR=${PIP_CACHE_DIR}"
	echo "WHEEL_DIR=${WHEEL_DIR}"
	echo "VENV_PATH=${VENV_PATH}"
	echo "TRUSTED_VENV_PATH=${TRUSTED_VENV_PATH}"
	echo "TRUSTED_CACHE_ROOT=${TRUSTED_CACHE_ROOT}"
	echo "FORK_CAN_COPY_TRUSTED_VENV=${FORK_CAN_COPY_TRUSTED_VENV}"
	echo "VENV_REUSED=${VENV_REUSED}"
	} >> "$GITHUB_ENV"

	- name: Validate GPU frequency
	run: \|
	set -euo pipefail
	TARGET_CLOCK_MHZ=1830
	RETRIES=5
	SLEEP_SECONDS=2

	if ! command -v mx-smi >/dev/null 2>&1; then
	echo "::error::mx-smi is not available on this runner."
	exit 1
	fi

	echo "Detected GPUs:"
	mx-smi -L

	attempt=1
	while [ "$attempt" -le "$RETRIES" ]; do
	echo "Validation attempt ${attempt}/${RETRIES}"
	mismatch=0
	while IFS=',' read -r gpu_index gpu_name gpu_clock; do
	gpu_index="$(echo "$gpu_index" \| xargs)"
	gpu_name="$(echo "$gpu_name" \| xargs)"
	gpu_clock="$(echo "$gpu_clock" \| xargs)"
	echo "GPU ${gpu_index} (${gpu_name}) clock: ${gpu_clock} MHz"
	if [[ "$gpu_clock" != "$TARGET_CLOCK_MHZ" ]]; then
	mismatch=1
	fi
	done < <(mx-smi --query-gpu=index,name,clocks.current.graphics --format=csv,noheader,nounits)

	if [[ "$mismatch" -eq 0 ]]; then
	echo "All GPUs locked at ${TARGET_CLOCK_MHZ} MHz."
	break
	fi

	if [[ "$attempt" -eq "$RETRIES" ]]; then
	echo "::error::GPU frequency validation failed. Expected ${TARGET_CLOCK_MHZ} MHz on all GPUs."
	exit 1
	fi

	sleep "$SLEEP_SECONDS"
	attempt=$((attempt + 1))
	done

	- name: Ensure venv and install dependencies
	env:
	IS_FORK: ${{ needs.security-policy.outputs.is_fork }}
	run: \|
	set -euo pipefail

	if [[ "$IS_FORK" == "true" && "$FORK_CAN_COPY_TRUSTED_VENV" == "true" ]]; then
	echo "Copying trusted venv from ${TRUSTED_VENV_PATH} to ${VENV_PATH}"
	rm -rf "${VENV_PATH}"
	if command -v rsync >/dev/null 2>&1; then
	rsync -a --delete "${TRUSTED_VENV_PATH}/" "${VENV_PATH}/"
	else
	mkdir -p "${VENV_PATH}"
	cp -a "${TRUSTED_VENV_PATH}/." "${VENV_PATH}/"
	fi
	# Validate the copied venv before continuing. A byte-for-byte copy can
	# still be sensitive to absolute paths embedded by venv tooling.
	# shellcheck source=/dev/null
	source "${VENV_PATH}/bin/activate"
	python --version
	python -m pip --version
	python -c "import tileops, torch; print('Copied fork venv import check ok')"
	exit 0
	fi

	if [[ "$VENV_REUSED" == "true" ]]; then
	echo "Using cached venv at ${VENV_PATH}"
	exit 0
	fi

	echo "Creating fresh venv at ${VENV_PATH}"
	python -m venv "${VENV_PATH}"
	# shellcheck source=/dev/null
	source "${VENV_PATH}/bin/activate"

	# Copy pre-installed MACA torch packages from conda to venv
	mkdir -p "${VENV_PATH}/lib/python3.12/site-packages/"
	cp -rf /opt/conda/lib/python3.12/site-packages/* "${VENV_PATH}/lib/python3.12/site-packages/"

	python -m pip install --upgrade pip setuptools wheel --no-user

	# Install test dependencies
	pip install --no-cache-dir \
	einops numpy "tqdm>=4.62.3" "typing_extensions>=4.10.0" \
	cloudpickle ml_dtypes psutil Cython \
	setuptools wheel ninja
	pip install --no-cache-dir \
	"pytest==9.0.2" \
	"pytest-xdist>=3.0" \
	"pyyaml>=6.0"

	# Install TileLang (MACA version)
	pip install --no-cache-dir git+https://github.com/tile-ai/tilelang-metax.git@1efc3ba8a7c48bc97047e562223538d2dcb2222d

	if [[ "$IS_FORK" == "true" ]]; then
	FIND_LINKS_FLAG=""
	if [[ -d "${WHEEL_DIR}" ]] && ls "${WHEEL_DIR}"/*.whl >/dev/null 2>&1; then
	FIND_LINKS_FLAG="--find-links ${WHEEL_DIR}"
	fi
	# shellcheck disable=SC2086
	PIP_NO_BUILD_ISOLATION=1 PIP_CACHE_DIR="${PIP_CACHE_DIR}" pip install -e '.[dev]' ${FIND_LINKS_FLAG}

	# Sync libtilelang.so mtime with trusted venv so compile cache keys match.
	# TileLang cache keys include the library's mtime (kernel_cache.py); a fresh
	# pip install produces a new mtime, causing all rsync'd cache entries to miss.
	# See: https://github.com/tile-ai/TileOPs/issues/594
	#
	# Use the hash-matched TRUSTED_VENV_PATH directly. A `find`-then-`sort`
	# sweep over `runner.tool_cache` can select a stale `tileops_ci_venv_*`
	# with a different libtilelang.so mtime, which would poison the rsync'd
	# cache entries keyed off the correct trusted venv's mtime.
	if [[ -d "${TRUSTED_VENV_PATH}" ]]; then
	TRUSTED_LIB=$(find "${TRUSTED_VENV_PATH}" -name "libtilelang.so" -type f 2>/dev/null \| head -1 \|\| true)
	FORK_LIB=$(find "${VENV_PATH}" -name "libtilelang.so" -type f 2>/dev/null \| head -1 \|\| true)
	if [[ -n "$TRUSTED_LIB" && -n "$FORK_LIB" ]]; then
	touch -r "$TRUSTED_LIB" "$FORK_LIB"
	echo "Synced libtilelang.so mtime for cache key compatibility"
	else
	echo "Warning: could not locate libtilelang.so for mtime sync"
	fi
	else
	echo "Warning: hash-matched trusted venv not found at ${TRUSTED_VENV_PATH}; skipping mtime sync"
	fi
	else
	PIP_NO_BUILD_ISOLATION=1 PIP_NO_CACHE_DIR=1 pip install -e '.[dev]'
	# Export wheels (including dependencies) so fork PRs can reuse them
	pip wheel -e '.[dev]' -w "${WHEEL_DIR}" 2>/dev/null \|\| true
	fi

	- name: Run tests
	env:
	EVENT_NAME: ${{ github.event_name }}
	TEST_SCOPE: ${{ needs.security-policy.outputs.scope }}
	PYTEST_TARGETS: ${{ needs.security-policy.outputs.pytest_targets }}
	run: \|
	set -euo pipefail
	# shellcheck source=/dev/null
	source "${VENV_PATH}/bin/activate"
	current_dir="$(pwd)"
	export PYTHONPATH="${current_dir}${PYTHONPATH:+:$PYTHONPATH}"
	echo "PYTHONPATH=$PYTHONPATH"
	python - <<'PY'
	import os

	print("Runtime cache env:")
	for key in ("TILELANG_CACHE_DIR", "TILELANG_TMP_DIR", "TRITON_CACHE_DIR"):
	print(f"{key}={os.environ.get(key)}")
	PY

	TEST_MARK_EXPR="smoke or full"
	if [[ "$EVENT_NAME" == "pull_request" ]]; then
	TEST_MARK_EXPR="smoke"
	fi

	echo "Resolved test scope: ${TEST_SCOPE}"
	echo "Resolved pytest targets: ${PYTEST_TARGETS}"
	read -r -a TARGETS <<< "${PYTEST_TARGETS}"
	echo "Running pytest -m \"${TEST_MARK_EXPR}\" on ${#TARGETS[@]} target(s)"
	python -m pytest -q "${TARGETS[@]}" -m "${TEST_MARK_EXPR}" --junit-xml=gpu_smoke_results.xml \| tee gpu_smoke.log

	- name: Generate gpu-smoke report
	if: ${{ always() && hashFiles('gpu_smoke_results.xml') != '' }}
	env:
	EVENT_NAME: ${{ github.event_name }}
	run: \|
	set -euo pipefail
	# shellcheck source=/dev/null
	source "${VENV_PATH}/bin/activate"

	REPORT_TARGET="smoke, full"
	if [[ "$EVENT_NAME" == "pull_request" ]]; then
	REPORT_TARGET="smoke"
	fi

	python scripts/gpu_smoke_report.py \
	--test-xml gpu_smoke_results.xml \
	--target "${REPORT_TARGET}" \
	--output gpu_smoke_report.md \
	\|\| echo "::warning::gpu_smoke_report.py failed"

	- name: Post gpu-smoke report to step summary
	if: ${{ always() && hashFiles('gpu_smoke_report.md') != '' }}
	run: cat gpu_smoke_report.md >> "$GITHUB_STEP_SUMMARY"

	- name: Cache stats
	if: ${{ always() }}
	env:
	IS_FORK: ${{ needs.security-policy.outputs.is_fork }}
	run: \|
	# Cache stats is informational only. Do NOT use `set -e`: a transient
	# failure in find/du/wc (e.g. a cache file vanishing mid-scan due to a
	# concurrent cleanup) must not fail the gpu-smoke job, and must not
	# skip the CACHE_SIGNAL logging below.
	set -u
	echo "=== Cache stats (is_fork=${IS_FORK}) ==="
	echo "VENV_REUSED=${VENV_REUSED:-unknown}"
	echo "FORK_CAN_COPY_TRUSTED_VENV=${FORK_CAN_COPY_TRUSTED_VENV:-unknown}"
	echo "TRUSTED_VENV_PATH=${TRUSTED_VENV_PATH:-unknown}"
	for cache_dir in "${TILELANG_CACHE_DIR:-}" "${TRITON_CACHE_DIR:-}" "${WHEEL_DIR:-}"; do
	if [[ -z "${cache_dir}" ]]; then
	continue
	fi
	if [[ -d "${cache_dir}" ]]; then
	# Tolerate errors from find/du/wc; emit "unknown" when a scan
	# fails rather than aborting the step. Enable pipefail inside
	# the subshell so a find/du failure upstream of wc/cut
	# propagates to the \|\| fallback instead of being masked.
	file_count=$(set -o pipefail; find "${cache_dir}" -type f 2>/dev/null \| wc -l 2>/dev/null) \|\| file_count="unknown"
	cache_size=$(set -o pipefail; du -sh "${cache_dir}" 2>/dev/null \| cut -f1 2>/dev/null) \|\| cache_size="unknown"
	if [[ -z "${file_count}" ]]; then file_count="unknown"; fi
	if [[ -z "${cache_size}" ]]; then cache_size="unknown"; fi
	echo "${cache_dir}: ${file_count} files, ${cache_size}"
	else
	echo "${cache_dir}: does not exist"
	fi
	done

	# Emit a concise cache hit/miss signal so future regressions surface
	# in CI logs without needing to diff per-run durations manually.
	# For fork PRs, "hit" means the trusted venv was copied (no cold pip
	# install); for trusted runs, "hit" means the hash-matched venv was
	# reused in place.
	if [[ "${IS_FORK}" == "true" ]]; then
	if [[ "${FORK_CAN_COPY_TRUSTED_VENV:-false}" == "true" ]]; then
	echo "CACHE_SIGNAL: fork venv=hit"
	else
	echo "CACHE_SIGNAL: fork venv=miss (cold pip install path)"
	fi
	else
	if [[ "${VENV_REUSED:-false}" == "true" ]]; then
	echo "CACHE_SIGNAL: trusted venv=hit"
	else
	echo "CACHE_SIGNAL: trusted venv=miss (fresh venv created)"
	fi
	fi

	- name: Upload artifacts
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: gpu_smoke.log
	path: \|
	gpu_smoke.log
	gpu_smoke_results.xml
	gpu_smoke_report.md
	retention-days: 7

	- name: Cleanup isolated fork state
	if: ${{ always() && needs.security-policy.outputs.is_fork == 'true' }}
	run: \|
	if [[ -n "${RUNTIME_ROOT:-}" ]]; then
	rm -rf "${RUNTIME_ROOT}"
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[CI] adapt .github/workflows/*.yml for MetaxGPU new #51

Workflow file

[CI] adapt .github/workflows/*.yml for MetaxGPU new #51

Uh oh!

Workflow file for this run