[CI] adapt .github/workflows/*.yml for MetaxGPU new #78
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GPU Smoke | |
| permissions: | |
| contents: read | |
| checks: read | |
| pull-requests: read | |
| on: | |
| push: | |
| branches: [dev, main, testbed] | |
| tags: ["v*"] | |
| pull_request: | |
| branches: [dev, main, testbed] | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| # Cache directories for nightly test jobs | |
| env: | |
| TILELANG_CACHE_DIR: /home/runner/_temp/ci-cache/tilelang | |
| TILELANG_TMP_DIR: /home/runner/_temp/ci-cache/tilelang/tmp | |
| TRITON_CACHE_DIR: /home/runner/_temp/ci-cache/triton | |
| PIP_CACHE_DIR: /home/runner/_temp/ci-cache/pip | |
| WHEEL_DIR: /home/runner/_temp/ci-cache/wheels | |
| MAX_JOBS: "64" | |
| jobs: | |
| ci-prereq: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| skip: ${{ steps.check.outputs.skip }} | |
| steps: | |
| - name: Wait for CI prerequisites | |
| id: check | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| EVENT_ACTION: ${{ github.event.action || '' }} | |
| IS_DRAFT: ${{ github.event.pull_request.draft || false }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| run: | | |
| set -euo pipefail | |
| if [[ "$EVENT_NAME" != "pull_request" ]]; then | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if [[ "$IS_DRAFT" == "true" ]]; then | |
| echo "Draft PR; skipping GPU smoke." | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| required_checks=(pre-commit gitleaks actionlint) | |
| deadline=$((SECONDS + 900)) | |
| while true; do | |
| checks_json=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs") | |
| latest_checks=$(echo "$checks_json" | jq ' | |
| .check_runs | |
| | group_by(.name) | |
| | map(max_by(.id)) | |
| ') | |
| pending="false" | |
| failed="" | |
| for check_name in "${required_checks[@]}"; do | |
| status=$(echo "$latest_checks" | jq -r --arg name "$check_name" ' | |
| map(select(.name == $name)) | first | .status // "" | |
| ') | |
| conclusion=$(echo "$latest_checks" | jq -r --arg name "$check_name" ' | |
| map(select(.name == $name)) | first | .conclusion // "" | |
| ') | |
| if [[ -z "$status" || "$status" != "completed" ]]; then | |
| pending="true" | |
| continue | |
| fi | |
| if [[ "$conclusion" == "skipped" ]]; then | |
| pending="true" | |
| continue | |
| fi | |
| if [[ "$conclusion" != "success" ]]; then | |
| failed="$check_name" | |
| break | |
| fi | |
| done | |
| title_status=$(echo "$latest_checks" | jq -r ' | |
| map(select(.name == "validate-pr-title")) | first | .status // "" | |
| ') | |
| title_conclusion=$(echo "$latest_checks" | jq -r ' | |
| map(select(.name == "validate-pr-title")) | first | .conclusion // "" | |
| ') | |
| if [[ -n "$title_status" && "$title_status" == "completed" && "$title_conclusion" != "success" && "$title_conclusion" != "skipped" ]]; then | |
| failed="validate-pr-title" | |
| elif [[ -n "$title_status" && "$title_status" != "completed" ]]; then | |
| pending="true" | |
| fi | |
| if [[ -n "$failed" ]]; then | |
| echo "CI prerequisite failed: $failed" | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if [[ "$pending" == "false" ]]; then | |
| break | |
| fi | |
| if (( SECONDS >= deadline )); then | |
| echo "Timed out waiting for CI prerequisites." | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| sleep 10 | |
| done | |
| if [[ "$EVENT_ACTION" == "ready_for_review" ]]; then | |
| CHECKS=$(echo "$latest_checks" | jq ' | |
| [ | |
| .[] | |
| | select(.name == "gpu-smoke") | |
| | .conclusion | |
| ] | |
| ') | |
| ALL_SUCCESS=$(echo "$CHECKS" | jq 'all(. == "success")') | |
| COUNT=$(echo "$CHECKS" | jq 'length') | |
| if [[ "$ALL_SUCCESS" == "true" && "$COUNT" -ge 1 ]]; then | |
| echo "GPU smoke already passed for SHA $HEAD_SHA; skipping" | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| fi | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| security-policy: | |
| needs: ci-prereq | |
| if: ${{ github.repository == 'MetaX-MACA/TileOPs-Metax' && needs.ci-prereq.outputs.skip != 'true' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| is_fork: ${{ steps.policy.outputs.is_fork }} | |
| allow_fast_path: ${{ steps.policy.outputs.allow_fast_path }} | |
| skip_gpu_smoke: ${{ steps.policy.outputs.skip_gpu_smoke }} | |
| scope: ${{ steps.policy.outputs.scope }} | |
| pytest_targets: ${{ steps.policy.outputs.pytest_targets }} | |
| reason: ${{ steps.policy.outputs.reason }} | |
| steps: | |
| - name: Checkout test files | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| sparse-checkout: | | |
| tests/ops | |
| - name: Determine GPU policy and pytest scope | |
| id: policy | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| PR_NUMBER: ${{ github.event.pull_request.number || '' }} | |
| HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| BASE_REPO: ${{ github.repository }} | |
| AUTHOR_ASSOC: ${{ github.event.pull_request.author_association || '' }} | |
| run: | | |
| set -euo pipefail | |
| add_target() { | |
| local target="$1" | |
| [[ -e "$target" ]] || return 0 | |
| if [[ -z "${seen_targets[$target]:-}" ]]; then | |
| selected_targets+=("$target") | |
| seen_targets["$target"]=1 | |
| fi | |
| } | |
| is_fork="false" | |
| allow_fast_path="true" | |
| skip_gpu_smoke="false" | |
| scope="full-smoke" | |
| pytest_targets="tests" | |
| reason="GPU smoke will run." | |
| # `is_fork` gates the downstream runtime isolation: when true, the | |
| # gpu-smoke job refuses to write persistent state under | |
| # /home/ci-runner and instead runs from a disposable per-run dir. | |
| # A repo member opening a PR from their personal fork technically | |
| # satisfies head != base, but they already have push access to the | |
| # base repo — treating their PR as untrusted forces them to | |
| # round-trip through a fresh venv rebuild every run, for no real | |
| # security gain. Keep external contributors on the isolated path; | |
| # promote OWNER/MEMBER/COLLABORATOR authors back onto the trusted | |
| # runtime layout. | |
| if [[ "$EVENT_NAME" == "pull_request" && "$HEAD_REPO" != "$BASE_REPO" ]]; then | |
| case "$AUTHOR_ASSOC" in | |
| OWNER|MEMBER|COLLABORATOR) | |
| is_fork="false" | |
| echo "Member-fork PR (author_association=${AUTHOR_ASSOC}); treating as trusted." | |
| ;; | |
| *) | |
| is_fork="true" | |
| ;; | |
| esac | |
| fi | |
| if [[ "$EVENT_NAME" == "pull_request" ]]; then | |
| mapfile -t changed_files < <( | |
| gh api "repos/${BASE_REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename' | |
| ) | |
| echo "Changed files (${#changed_files[@]}):" | |
| printf ' - %s\n' "${changed_files[@]}" | |
| # Fast path: if no Python files changed, GPU smoke is unnecessary. | |
| # For fork PRs, also check for high-risk files (workflow, build | |
| # config, etc.) that must still trigger GPU smoke even without | |
| # Python changes to avoid security bypass. | |
| has_python="false" | |
| has_high_risk="false" | |
| for path in "${changed_files[@]}"; do | |
| if [[ "$path" == *.py ]]; then | |
| has_python="true" | |
| fi | |
| if [[ "$is_fork" == "true" ]]; then | |
| case "$path" in | |
| .github/workflows/*|pyproject.toml|requirements*.txt|requirements*.in|setup.py|setup.cfg|Dockerfile*|scripts/bootstrap*|scripts/install*) | |
| has_high_risk="true" | |
| ;; | |
| esac | |
| fi | |
| # TODO: Temporarily allow workflow changes to trigger GPU smoke for testing. | |
| # case "$path" in | |
| # .github/workflows/*|pyproject.toml|requirements*.txt|requirements*.in|setup.py|setup.cfg|Dockerfile*|scripts/bootstrap*|scripts/install*) | |
| # has_high_risk="true" | |
| # ;; | |
| # esac | |
| done | |
| if [[ "$has_python" == "false" && "$has_high_risk" == "false" ]]; then | |
| skip_gpu_smoke="true" | |
| reason="No Python files changed; GPU smoke skipped." | |
| { | |
| echo "skip_gpu_smoke=$skip_gpu_smoke" | |
| echo "is_fork=$is_fork" | |
| echo "allow_fast_path=$allow_fast_path" | |
| echo "scope=skip" | |
| echo "pytest_targets=" | |
| echo "reason=$reason" | |
| } >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if [[ "$has_python" == "false" && "$has_high_risk" == "true" ]]; then | |
| # Fork PR changed high-risk build/infra files but no Python. | |
| # Run full GPU smoke to validate the environment is not broken. | |
| skip_gpu_smoke="false" | |
| allow_fast_path="false" | |
| scope="full-smoke" | |
| pytest_targets="tests" | |
| reason="Fork PR changed high-risk file(s) without Python changes; running full GPU smoke." | |
| { | |
| echo "skip_gpu_smoke=$skip_gpu_smoke" | |
| echo "is_fork=$is_fork" | |
| echo "allow_fast_path=$allow_fast_path" | |
| echo "scope=$scope" | |
| echo "pytest_targets=$pytest_targets" | |
| echo "reason=$reason" | |
| } >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| scope="skip" | |
| declare -A seen_targets=() | |
| selected_targets=() | |
| for path in "${changed_files[@]}"; do | |
| if [[ "$is_fork" == "true" ]]; then | |
| case "$path" in | |
| .github/ISSUE_TEMPLATE/*|.github/pull_request_template.md|tileops/ops/*.py|tests/ops/test_*.py|*.md) | |
| ;; | |
| .github/workflows/*|pyproject.toml|requirements*.txt|requirements*.in|setup.py|setup.cfg|Dockerfile*|scripts/bootstrap*|scripts/install*) | |
| allow_fast_path="false" | |
| reason="Fork PR changed a high-risk file: $path" | |
| ;; | |
| *) | |
| allow_fast_path="false" | |
| reason="Fork PR changed a file outside the future fast-path policy: $path" | |
| ;; | |
| esac | |
| fi | |
| case "$path" in | |
| .github/ISSUE_TEMPLATE/*|.github/pull_request_template.md|.github/workflows/auto-label.yml|.github/workflows/stale-issues.yml|*.md) | |
| ;; | |
| tests/ops/test_*.py) | |
| add_target "$path" | |
| if [[ "$scope" == "skip" ]]; then | |
| scope="targeted" | |
| fi | |
| ;; | |
| *) | |
| scope="full-smoke" | |
| reason="Shared or unsupported path $path; falling back to full smoke." | |
| break | |
| ;; | |
| esac | |
| done | |
| if [[ "$scope" == "skip" ]]; then | |
| if [[ "$allow_fast_path" == "true" ]]; then | |
| # All changed files matched docs/metadata patterns; skip GPU smoke | |
| # for both trusted and fork PRs. | |
| skip_gpu_smoke="true" | |
| pytest_targets="" | |
| reason="Docs-only or metadata-only change; GPU smoke skipped." | |
| elif [[ "$is_fork" == "true" ]]; then | |
| scope="full-smoke" | |
| pytest_targets="tests" | |
| reason="${reason}; current policy still runs GPU smoke." | |
| else | |
| skip_gpu_smoke="true" | |
| pytest_targets="" | |
| reason="Docs-only or metadata-only change; GPU smoke skipped." | |
| fi | |
| elif [[ "$scope" == "targeted" ]]; then | |
| if [[ "${#selected_targets[@]}" -eq 0 ]]; then | |
| scope="full-smoke" | |
| pytest_targets="tests" | |
| reason="No mapped tests were collected; falling back to full smoke." | |
| else | |
| pytest_targets="${selected_targets[*]}" | |
| if [[ "$is_fork" == "true" && "$allow_fast_path" != "true" ]]; then | |
| reason="${reason}; current policy still runs targeted GPU smoke." | |
| else | |
| reason="Running targeted GPU smoke tests." | |
| fi | |
| fi | |
| fi | |
| fi | |
| { | |
| echo "is_fork=$is_fork" | |
| echo "allow_fast_path=$allow_fast_path" | |
| echo "skip_gpu_smoke=$skip_gpu_smoke" | |
| echo "scope=$scope" | |
| echo "pytest_targets=$pytest_targets" | |
| echo "reason=$reason" | |
| } >> "$GITHUB_OUTPUT" | |
| gpu-smoke: | |
| needs: [ci-prereq, security-policy] | |
| if: ${{ github.repository == 'MetaX-MACA/TileOPs-Metax' && needs.ci-prereq.outputs.skip != 'true' && needs.security-policy.outputs.skip_gpu_smoke != 'true' }} | |
| runs-on: tileops-metax-runner | |
| steps: | |
| - name: Checkout code for fork PR | |
| if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| - name: Checkout code for trusted branch | |
| if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository }} | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y git | |
| # Trusted actions checkout: must run AFTER the workspace checkouts so | |
| # their `git clean -ffdx` pass does not wipe `.trusted/`. Anything | |
| # loaded via `uses: ./...` from here on MUST resolve through | |
| # `.trusted/` so that fork-controlled PR code cannot substitute the | |
| # composite action. | |
| # | |
| # Ref selection by trust level. `is_fork` alone is too coarse: a repo | |
| # member may open a PR from their personal fork, which still counts as | |
| # a fork PR but the author already has push access to the base repo. | |
| # Combine repo equality with `author_association` so repo-owned CI | |
| # changes can self-validate on the PR that introduces them, while | |
| # external contributors are kept on the trusted base ref. | |
| # | |
| # - external fork PR (author not OWNER/MEMBER/COLLABORATOR) | |
| # → `base.sha` (authoritative; external author can't alter) | |
| # - same-repo PR, member-fork PR, or push | |
| # → head sha (author already has write access to base repo) | |
| - name: Checkout trusted actions (base repo) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository }} | |
| ref: ${{ (needs.security-policy.outputs.is_fork == 'true' && !contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association)) && github.event.pull_request.base.sha || github.event.pull_request.head.sha || github.sha }} | |
| path: .trusted | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| sparse-checkout: | | |
| .github/actions | |
| scripts/ci_venv_hash.py | |
| pyproject.toml | |
| - name: Reclaim runner disk | |
| uses: ./.trusted/.github/actions/reclaim-runner-disk | |
| with: | |
| reclaim-below-gib: "10" | |
| cache-trim-cooldown-minutes: "120" | |
| cache-dirs: | | |
| ${{ runner.temp }}/ci-cache/triton | |
| ${{ runner.temp }}/ci-cache/pip | |
| ${{ runner.temp }}/ci-cache/wheels | |
| ${{ runner.temp }}/ci-cache/tilelang/tmp | |
| atomic-cache-dirs: | | |
| ${{ runner.temp }}/ci-cache/tilelang/autotuner | |
| df-paths: "${{ runner.temp }} ${{ runner.temp }}/ci-cache" | |
| # Daily runner-maintenance.yml owns the destructive autotuner | |
| # age-trim; per-PR runs only need sentinel-repair (still | |
| # unconditional) so they don't evict tuning artefacts the next | |
| # PR would otherwise reuse. | |
| skip-atomic-age-trim: "true" | |
| - name: Resolve runtime state | |
| env: | |
| IS_FORK: ${{ needs.security-policy.outputs.is_fork }} | |
| run: | | |
| set -euo pipefail | |
| # Add conda to PATH | |
| export PATH="/opt/conda/bin:${PATH}" | |
| # Set MACA environment variables | |
| export USE_MACA=ON | |
| # Narrow the venv cache key to sections that actually drive the | |
| # installed package set. Hashing the whole pyproject.toml means a | |
| # tweak to [tool.pytest] or [tool.ruff] invalidates every warm | |
| # fork-PR venv, defeating #996's fast path for no packaging | |
| # reason. See scripts/ci_venv_hash.py for the parsed sections and | |
| # the full-file fallback contract. | |
| # | |
| # Trust note: always run the hash script from `.trusted/` (the | |
| # sparse-checked-out base/head ref selected above). This prevents | |
| # external fork PRs from executing attacker-controlled Python on | |
| # the self-hosted runner before the IS_FORK runtime isolation | |
| # branches. We also hash the `.trusted/` pyproject.toml so the | |
| # computed key identifies the trusted venv that was actually | |
| # installed from that same pyproject, not the fork's edits. | |
| PYPROJECT_HASH=$(python .trusted/scripts/ci_venv_hash.py .trusted/pyproject.toml) | |
| HASH_INPUT="$( | |
| { | |
| echo "python=3.12" | |
| echo "install=PIP_NO_BUILD_ISOLATION=1 pip install -e '.[dev]'" | |
| echo "pyproject=${PYPROJECT_HASH}" | |
| } | sha256sum | cut -c1-16 | |
| )" | |
| VENV_PREFIX="tileops_ci_venv" | |
| VENV_DIR="${VENV_PREFIX}_${HASH_INPUT}" | |
| TRUSTED_VENV_PATH="${{ runner.tool_cache }}/${VENV_DIR}" | |
| TRUSTED_CACHE_ROOT="${RUNNER_TEMP}/ci-cache" | |
| FORK_CAN_COPY_TRUSTED_VENV="false" | |
| if [[ "$IS_FORK" == "true" ]]; then | |
| RUNTIME_ROOT="${RUNNER_TEMP}/gpu-smoke-${{ github.run_id }}-${{ github.run_attempt }}" | |
| # Per-run isolated cache: rsync from trusted cache into temp dir. | |
| # Fork PRs never write to any persistent shared state. | |
| TILELANG_CACHE_DIR="${RUNTIME_ROOT}/tilelang/cache" | |
| TILELANG_TMP_DIR="${TILELANG_CACHE_DIR}/tmp" | |
| TRITON_CACHE_DIR="${RUNTIME_ROOT}/triton/cache" | |
| PIP_CACHE_DIR="${RUNTIME_ROOT}/pip/cache" | |
| WHEEL_DIR="${RUNTIME_ROOT}/wheels" | |
| VENV_PATH="${RUNTIME_ROOT}/venv" | |
| # Divergence guard: TRUSTED_VENV_PATH was derived from the trusted | |
| # base-repo pyproject.toml hash. A fork PR may have edited its own | |
| # workspace pyproject.toml (added/removed dependencies) without | |
| # that touching the base-repo file. If we copy the trusted venv | |
| # for such a fork, the installed package set silently disagrees | |
| # with `.[dev]` in the fork's workspace. Re-run the (trusted) hash | |
| # script against the fork's workspace pyproject.toml and only | |
| # fast-path when both hashes match; otherwise fall through to a | |
| # fresh isolated pip install below. | |
| WORKSPACE_PYPROJECT_HASH=$(python .trusted/scripts/ci_venv_hash.py ./pyproject.toml) | |
| echo "WORKSPACE_PYPROJECT_HASH=${WORKSPACE_PYPROJECT_HASH}" | |
| echo "TRUSTED_PYPROJECT_HASH=${PYPROJECT_HASH}" | |
| if [[ -x "${TRUSTED_VENV_PATH}/bin/python" && "${WORKSPACE_PYPROJECT_HASH}" == "${PYPROJECT_HASH}" ]]; then | |
| FORK_CAN_COPY_TRUSTED_VENV="true" | |
| VENV_REUSED="true" | |
| else | |
| if [[ -x "${TRUSTED_VENV_PATH}/bin/python" ]]; then | |
| echo "Fork pyproject diverges from trusted base; forcing fresh isolated venv install" | |
| fi | |
| VENV_REUSED="false" | |
| fi | |
| mkdir -p "${TILELANG_CACHE_DIR}" "${TILELANG_TMP_DIR}" "${TRITON_CACHE_DIR}" "${PIP_CACHE_DIR}" "${WHEEL_DIR}" | |
| # Copy trusted compile caches into the isolated per-run dir. | |
| # rsync is fast for warm caches (<5s); fork state is fully disposable. | |
| TRUSTED_WHEEL_DIR="${TRUSTED_CACHE_ROOT}/wheels" | |
| copy_start=$SECONDS | |
| if command -v rsync >/dev/null 2>&1; then | |
| rsync -a "${TRUSTED_CACHE_ROOT}/tilelang/" "${TILELANG_CACHE_DIR}/" 2>/dev/null || true | |
| rsync -a "${TRUSTED_CACHE_ROOT}/triton/" "${TRITON_CACHE_DIR}/" 2>/dev/null || true | |
| rsync -a "${TRUSTED_CACHE_ROOT}/pip/" "${PIP_CACHE_DIR}/" 2>/dev/null || true | |
| if [[ -d "${TRUSTED_WHEEL_DIR}" ]]; then | |
| rsync -a "${TRUSTED_WHEEL_DIR}/" "${WHEEL_DIR}/" 2>/dev/null || true | |
| fi | |
| else | |
| cp -a "${TRUSTED_CACHE_ROOT}/tilelang/." "${TILELANG_CACHE_DIR}/" 2>/dev/null || true | |
| cp -a "${TRUSTED_CACHE_ROOT}/triton/." "${TRITON_CACHE_DIR}/" 2>/dev/null || true | |
| cp -a "${TRUSTED_CACHE_ROOT}/pip/." "${PIP_CACHE_DIR}/" 2>/dev/null || true | |
| if [[ -d "${TRUSTED_WHEEL_DIR}" ]]; then | |
| cp -a "${TRUSTED_WHEEL_DIR}/." "${WHEEL_DIR}/" 2>/dev/null || true | |
| fi | |
| fi | |
| copy_elapsed=$(( SECONDS - copy_start )) | |
| echo "Cache copy completed in ${copy_elapsed}s" | |
| else | |
| RUNTIME_ROOT="/home/ci-runner" | |
| TILELANG_CACHE_DIR="${TRUSTED_CACHE_ROOT}/tilelang" | |
| TILELANG_TMP_DIR="${TILELANG_CACHE_DIR}/tmp" | |
| TRITON_CACHE_DIR="${TRUSTED_CACHE_ROOT}/triton" | |
| PIP_CACHE_DIR="${TRUSTED_CACHE_ROOT}/pip" | |
| WHEEL_DIR="${TRUSTED_CACHE_ROOT}/wheels" | |
| mkdir -p "${TILELANG_CACHE_DIR}" "${TILELANG_TMP_DIR}" "${TRITON_CACHE_DIR}" "${PIP_CACHE_DIR}" "${WHEEL_DIR}" | |
| VENV_PATH="${TRUSTED_VENV_PATH}" | |
| if [[ -x "${VENV_PATH}/bin/python" ]]; then | |
| VENV_REUSED="true" | |
| else | |
| VENV_REUSED="false" | |
| fi | |
| fi | |
| { | |
| echo "PATH=${PATH}" | |
| echo "USE_MACA=ON" | |
| echo "RUNTIME_ROOT=${RUNTIME_ROOT}" | |
| echo "TILELANG_CACHE_DIR=${TILELANG_CACHE_DIR}" | |
| echo "TILELANG_TMP_DIR=${TILELANG_TMP_DIR}" | |
| echo "TRITON_CACHE_DIR=${TRITON_CACHE_DIR}" | |
| echo "PIP_CACHE_DIR=${PIP_CACHE_DIR}" | |
| echo "WHEEL_DIR=${WHEEL_DIR}" | |
| echo "VENV_PATH=${VENV_PATH}" | |
| echo "TRUSTED_VENV_PATH=${TRUSTED_VENV_PATH}" | |
| echo "TRUSTED_CACHE_ROOT=${TRUSTED_CACHE_ROOT}" | |
| echo "FORK_CAN_COPY_TRUSTED_VENV=${FORK_CAN_COPY_TRUSTED_VENV}" | |
| echo "VENV_REUSED=${VENV_REUSED}" | |
| } >> "$GITHUB_ENV" | |
| - name: Validate GPU frequency | |
| run: | | |
| set -euo pipefail | |
| TARGET_CLOCK_MHZ=1830 | |
| RETRIES=5 | |
| SLEEP_SECONDS=2 | |
| if ! command -v mx-smi >/dev/null 2>&1; then | |
| echo "::error::mx-smi is not available on this runner." | |
| exit 1 | |
| fi | |
| echo "Detected GPUs:" | |
| mx-smi -L | |
| attempt=1 | |
| while [ "$attempt" -le "$RETRIES" ]; do | |
| echo "Validation attempt ${attempt}/${RETRIES}" | |
| mismatch=0 | |
| # mx-smi --show-clock outputs clock info in a table format | |
| # Parse the graphics clock (gr) from the output | |
| clock_output=$(mx-smi --show-clock gr 2>/dev/null || mx-smi --show-clock all 2>/dev/null || mx-smi --show-clock) | |
| echo "Clock output:" | |
| echo "$clock_output" | |
| # Extract graphics clock values - format varies, try multiple patterns | |
| # Typical format: lines with GPU index and clock value in MHz | |
| gpu_clocks=$(echo "$clock_output" | grep -Eo '[0-9]+ MHz' | grep -Eo '[0-9]+' | head -1 || echo "") | |
| if [[ -z "$gpu_clocks" ]]; then | |
| # Fallback: try parsing from table format | |
| gpu_clocks=$(echo "$clock_output" | awk '/gr|graphics|clock/ {for(i=1;i<=NF;i++) if($i ~ /^[0-9]+$/) print $i}' | head -1 || echo "") | |
| fi | |
| if [[ -n "$gpu_clocks" ]]; then | |
| echo "GPU graphics clock: ${gpu_clocks} MHz" | |
| if [[ "$gpu_clocks" != "$TARGET_CLOCK_MHZ" ]]; then | |
| mismatch=1 | |
| fi | |
| else | |
| echo "::warning::Could not parse GPU clock from mx-smi output, skipping validation" | |
| mismatch=0 | |
| fi | |
| if [[ "$mismatch" -eq 0 ]]; then | |
| echo "GPU frequency validation passed." | |
| break | |
| fi | |
| if [[ "$attempt" -eq "$RETRIES" ]]; then | |
| echo "::error::GPU frequency validation failed. Expected ${TARGET_CLOCK_MHZ} MHz." | |
| exit 1 | |
| fi | |
| sleep "$SLEEP_SECONDS" | |
| attempt=$((attempt + 1)) | |
| done | |
| - name: Ensure venv and install dependencies | |
| env: | |
| IS_FORK: ${{ needs.security-policy.outputs.is_fork }} | |
| run: | | |
| set -euo pipefail | |
| if [[ "$IS_FORK" == "true" && "$FORK_CAN_COPY_TRUSTED_VENV" == "true" ]]; then | |
| echo "Copying trusted venv from ${TRUSTED_VENV_PATH} to ${VENV_PATH}" | |
| rm -rf "${VENV_PATH}" | |
| if command -v rsync >/dev/null 2>&1; then | |
| rsync -a --delete "${TRUSTED_VENV_PATH}/" "${VENV_PATH}/" | |
| else | |
| mkdir -p "${VENV_PATH}" | |
| cp -a "${TRUSTED_VENV_PATH}/." "${VENV_PATH}/" | |
| fi | |
| # Validate the copied venv before continuing. A byte-for-byte copy can | |
| # still be sensitive to absolute paths embedded by venv tooling. | |
| # shellcheck source=/dev/null | |
| source "${VENV_PATH}/bin/activate" | |
| python --version | |
| python -m pip --version | |
| python -c "import tileops, torch; print('Copied fork venv import check ok')" | |
| exit 0 | |
| fi | |
| if [[ "$VENV_REUSED" == "true" ]]; then | |
| echo "Using cached venv at ${VENV_PATH}" | |
| exit 0 | |
| fi | |
| echo "Creating fresh venv at ${VENV_PATH}" | |
| python -m venv "${VENV_PATH}" | |
| # shellcheck source=/dev/null | |
| source "${VENV_PATH}/bin/activate" | |
| # Copy pre-installed MACA torch packages from conda to venv | |
| mkdir -p "${VENV_PATH}/lib/python3.12/site-packages/" | |
| cp -rf /opt/conda/lib/python3.12/site-packages/* "${VENV_PATH}/lib/python3.12/site-packages/" | |
| python -m pip install --upgrade pip setuptools wheel --no-user | |
| # Install TileLang (MACA version) | |
| pip install --no-cache-dir git+https://github.com/tile-ai/tilelang-metax.git@1efc3ba8a7c48bc97047e562223538d2dcb2222d | |
| if [[ "$IS_FORK" == "true" ]]; then | |
| FIND_LINKS_FLAG="" | |
| if [[ -d "${WHEEL_DIR}" ]] && ls "${WHEEL_DIR}"/*.whl >/dev/null 2>&1; then | |
| FIND_LINKS_FLAG="--find-links ${WHEEL_DIR}" | |
| fi | |
| # shellcheck disable=SC2086 | |
| PIP_NO_BUILD_ISOLATION=1 PIP_CACHE_DIR="${PIP_CACHE_DIR}" pip install --no-deps -e '.[dev]' ${FIND_LINKS_FLAG} | |
| # Sync libtilelang.so mtime with trusted venv so compile cache keys match. | |
| # TileLang cache keys include the library's mtime (kernel_cache.py); a fresh | |
| # pip install produces a new mtime, causing all rsync'd cache entries to miss. | |
| # See: https://github.com/tile-ai/TileOPs/issues/594 | |
| # | |
| # Use the hash-matched TRUSTED_VENV_PATH directly. A `find`-then-`sort` | |
| # sweep over `runner.tool_cache` can select a stale `tileops_ci_venv_*` | |
| # with a different libtilelang.so mtime, which would poison the rsync'd | |
| # cache entries keyed off the correct trusted venv's mtime. | |
| if [[ -d "${TRUSTED_VENV_PATH}" ]]; then | |
| TRUSTED_LIB=$(find "${TRUSTED_VENV_PATH}" -name "libtilelang.so" -type f 2>/dev/null | head -1 || true) | |
| FORK_LIB=$(find "${VENV_PATH}" -name "libtilelang.so" -type f 2>/dev/null | head -1 || true) | |
| if [[ -n "$TRUSTED_LIB" && -n "$FORK_LIB" ]]; then | |
| touch -r "$TRUSTED_LIB" "$FORK_LIB" | |
| echo "Synced libtilelang.so mtime for cache key compatibility" | |
| else | |
| echo "Warning: could not locate libtilelang.so for mtime sync" | |
| fi | |
| else | |
| echo "Warning: hash-matched trusted venv not found at ${TRUSTED_VENV_PATH}; skipping mtime sync" | |
| fi | |
| else | |
| PIP_NO_BUILD_ISOLATION=1 PIP_NO_CACHE_DIR=1 pip install --no-deps -e '.[dev]' | |
| # Export wheels (including dependencies) so fork PRs can reuse them | |
| pip wheel --no-deps -e '.[dev]' -w "${WHEEL_DIR}" 2>/dev/null || true | |
| fi | |
| - name: Run tests | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| TEST_SCOPE: ${{ needs.security-policy.outputs.scope }} | |
| PYTEST_TARGETS: ${{ needs.security-policy.outputs.pytest_targets }} | |
| run: | | |
| set -euo pipefail | |
| # shellcheck source=/dev/null | |
| source "${VENV_PATH}/bin/activate" | |
| current_dir="$(pwd)" | |
| export PYTHONPATH="${current_dir}${PYTHONPATH:+:$PYTHONPATH}" | |
| echo "PYTHONPATH=$PYTHONPATH" | |
| python - <<'PY' | |
| import os | |
| print("Runtime cache env:") | |
| for key in ("TILELANG_CACHE_DIR", "TILELANG_TMP_DIR", "TRITON_CACHE_DIR"): | |
| print(f"{key}={os.environ.get(key)}") | |
| PY | |
| TEST_MARK_EXPR="smoke or full" | |
| if [[ "$EVENT_NAME" == "pull_request" ]]; then | |
| TEST_MARK_EXPR="smoke" | |
| fi | |
| echo "Resolved test scope: ${TEST_SCOPE}" | |
| echo "Resolved pytest targets: ${PYTEST_TARGETS}" | |
| read -r -a TARGETS <<< "${PYTEST_TARGETS}" | |
| echo "Running pytest -m \"${TEST_MARK_EXPR}\" on ${#TARGETS[@]} target(s)" | |
| python -m pytest -q "${TARGETS[@]}" -m "${TEST_MARK_EXPR}" --junit-xml=gpu_smoke_results.xml | tee gpu_smoke.log | |
| - name: Generate gpu-smoke report | |
| if: ${{ always() && hashFiles('gpu_smoke_results.xml') != '' }} | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| run: | | |
| set -euo pipefail | |
| # shellcheck source=/dev/null | |
| source "${VENV_PATH}/bin/activate" | |
| REPORT_TARGET="smoke, full" | |
| if [[ "$EVENT_NAME" == "pull_request" ]]; then | |
| REPORT_TARGET="smoke" | |
| fi | |
| python scripts/gpu_smoke_report.py \ | |
| --test-xml gpu_smoke_results.xml \ | |
| --target "${REPORT_TARGET}" \ | |
| --output gpu_smoke_report.md \ | |
| || echo "::warning::gpu_smoke_report.py failed" | |
| - name: Post gpu-smoke report to step summary | |
| if: ${{ always() && hashFiles('gpu_smoke_report.md') != '' }} | |
| run: cat gpu_smoke_report.md >> "$GITHUB_STEP_SUMMARY" | |
| - name: Cache stats | |
| if: ${{ always() }} | |
| env: | |
| IS_FORK: ${{ needs.security-policy.outputs.is_fork }} | |
| run: | | |
| # Cache stats is informational only. Do NOT use `set -e`: a transient | |
| # failure in find/du/wc (e.g. a cache file vanishing mid-scan due to a | |
| # concurrent cleanup) must not fail the gpu-smoke job, and must not | |
| # skip the CACHE_SIGNAL logging below. | |
| set -u | |
| echo "=== Cache stats (is_fork=${IS_FORK}) ===" | |
| echo "VENV_REUSED=${VENV_REUSED:-unknown}" | |
| echo "FORK_CAN_COPY_TRUSTED_VENV=${FORK_CAN_COPY_TRUSTED_VENV:-unknown}" | |
| echo "TRUSTED_VENV_PATH=${TRUSTED_VENV_PATH:-unknown}" | |
| for cache_dir in "${TILELANG_CACHE_DIR:-}" "${TRITON_CACHE_DIR:-}" "${WHEEL_DIR:-}"; do | |
| if [[ -z "${cache_dir}" ]]; then | |
| continue | |
| fi | |
| if [[ -d "${cache_dir}" ]]; then | |
| # Tolerate errors from find/du/wc; emit "unknown" when a scan | |
| # fails rather than aborting the step. Enable pipefail inside | |
| # the subshell so a find/du failure upstream of wc/cut | |
| # propagates to the || fallback instead of being masked. | |
| file_count=$(set -o pipefail; find "${cache_dir}" -type f 2>/dev/null | wc -l 2>/dev/null) || file_count="unknown" | |
| cache_size=$(set -o pipefail; du -sh "${cache_dir}" 2>/dev/null | cut -f1 2>/dev/null) || cache_size="unknown" | |
| if [[ -z "${file_count}" ]]; then file_count="unknown"; fi | |
| if [[ -z "${cache_size}" ]]; then cache_size="unknown"; fi | |
| echo "${cache_dir}: ${file_count} files, ${cache_size}" | |
| else | |
| echo "${cache_dir}: does not exist" | |
| fi | |
| done | |
| # Emit a concise cache hit/miss signal so future regressions surface | |
| # in CI logs without needing to diff per-run durations manually. | |
| # For fork PRs, "hit" means the trusted venv was copied (no cold pip | |
| # install); for trusted runs, "hit" means the hash-matched venv was | |
| # reused in place. | |
| if [[ "${IS_FORK}" == "true" ]]; then | |
| if [[ "${FORK_CAN_COPY_TRUSTED_VENV:-false}" == "true" ]]; then | |
| echo "CACHE_SIGNAL: fork venv=hit" | |
| else | |
| echo "CACHE_SIGNAL: fork venv=miss (cold pip install path)" | |
| fi | |
| else | |
| if [[ "${VENV_REUSED:-false}" == "true" ]]; then | |
| echo "CACHE_SIGNAL: trusted venv=hit" | |
| else | |
| echo "CACHE_SIGNAL: trusted venv=miss (fresh venv created)" | |
| fi | |
| fi | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: gpu_smoke.log | |
| path: | | |
| gpu_smoke.log | |
| gpu_smoke_results.xml | |
| gpu_smoke_report.md | |
| retention-days: 7 | |
| - name: Cleanup isolated fork state | |
| if: ${{ always() && needs.security-policy.outputs.is_fork == 'true' }} | |
| run: | | |
| if [[ -n "${RUNTIME_ROOT:-}" ]]; then | |
| rm -rf "${RUNTIME_ROOT}" | |
| fi | |
| # ========================================================================= | |
| # Nightly test jobs (for PR testing only) | |
| # ========================================================================= | |
| nightly-benchmark: | |
| if: ${{ always() && github.repository == 'MetaX-MACA/TileOPs-Metax' }} | |
| runs-on: tileops-metax-runner | |
| # timeout-minutes: 120 | |
| timeout-minutes: 300 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up nightly venv | |
| run: scripts/ci/setup_nightly_venv.sh | |
| shell: bash | |
| - name: Run benchmark ops | |
| id: benchmark_tests | |
| continue-on-error: true | |
| run: | | |
| set -euo pipefail | |
| # shellcheck source=/dev/null | |
| source "${VENV_PATH}/bin/activate" | |
| # Validate GPU frequency | |
| TARGET_CLOCK_MHZ=1830 | |
| RETRIES=5 | |
| for i in $(seq 1 "${RETRIES}"); do | |
| gpu_clock=$(mx-smi --show-clock | awk '/XCORE/ {print $(NF-1); exit}') | |
| echo "GPU clock: ${gpu_clock} MHz (attempt ${i}/${RETRIES})" | |
| if [ "${gpu_clock}" = "${TARGET_CLOCK_MHZ}" ]; then break; fi | |
| if [ "${i}" -eq "${RETRIES}" ]; then | |
| echo "::error::GPU frequency validation failed. Expected ${TARGET_CLOCK_MHZ} MHz." | |
| exit 1 | |
| fi | |
| sleep 2 | |
| done | |
| PIP_NO_BUILD_ISOLATION=1 pip install --no-deps -e ".[dev]" | |
| export PYTHONPATH="${GITHUB_WORKSPACE}${PYTHONPATH:+:$PYTHONPATH}" | |
| set -o pipefail | |
| # TODO: Run full benchmark suite after verifying fix | |
| python -m pytest -q benchmarks/ops --junit-xml=bench_results.xml | tee tileops_benchmarks.log | |
| # python -m pytest -q benchmarks/ops/bench_rms_norm.py -v --tb=short --junit-xml=bench_results.xml | tee tileops_benchmarks.log | |
| if [ -f profile_run.log ]; then | |
| { echo; echo "===== profile_run.log summary ====="; cat profile_run.log; } >> tileops_benchmarks.log | |
| else | |
| echo "::warning::profile_run.log not found; benchmark may have failed partially" >> tileops_benchmarks.log | |
| fi | |
| shell: bash | |
| - name: Upload benchmark artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| name: tileops_benchmark_${{ github.run_id }} | |
| path: | | |
| tileops_benchmarks.log | |
| bench_results.xml | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| - name: Fail benchmark job if pytest failed | |
| if: ${{ always() && steps.benchmark_tests.outcome == 'failure' }} | |
| run: exit 1 | |
| shell: bash | |
| - name: Cleanup run-local venv | |
| if: ${{ always() }} | |
| run: | | |
| set -euo pipefail | |
| if [[ -n "${RUNTIME_ROOT:-}" && "${RUNTIME_ROOT}" == "${RUNNER_TEMP}/nightly-"* ]]; then | |
| rm -rf "${RUNTIME_ROOT}" | |
| fi | |
| shell: bash |