[CI] adapt .github/workflows/*.yml for MetaxGPU #26
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GPU Smoke | |
| permissions: | |
| contents: read | |
| checks: read | |
| pull-requests: read | |
| on: | |
| push: | |
| branches: [dev] | |
| tags: ["v*"] | |
| pull_request: | |
| branches: [dev] | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| ci-prereq: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| skip: ${{ steps.check.outputs.skip }} | |
| steps: | |
| - name: Wait for CI prerequisites | |
| id: check | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| EVENT_ACTION: ${{ github.event.action || '' }} | |
| IS_DRAFT: ${{ github.event.pull_request.draft || false }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| run: | | |
| set -euo pipefail | |
| if [[ "$EVENT_NAME" != "pull_request" ]]; then | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if [[ "$IS_DRAFT" == "true" ]]; then | |
| echo "Draft PR; skipping GPU smoke." | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| required_checks=(validate-commits pre-commit gitleaks actionlint) | |
| deadline=$((SECONDS + 900)) | |
| while true; do | |
| checks_json=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs") | |
| latest_checks=$(echo "$checks_json" | jq ' | |
| .check_runs | |
| | group_by(.name) | |
| | map(max_by(.id)) | |
| ') | |
| pending="false" | |
| failed="" | |
| for check_name in "${required_checks[@]}"; do | |
| status=$(echo "$latest_checks" | jq -r --arg name "$check_name" ' | |
| map(select(.name == $name)) | first | .status // "" | |
| ') | |
| conclusion=$(echo "$latest_checks" | jq -r --arg name "$check_name" ' | |
| map(select(.name == $name)) | first | .conclusion // "" | |
| ') | |
| if [[ -z "$status" || "$status" != "completed" ]]; then | |
| pending="true" | |
| continue | |
| fi | |
| if [[ "$conclusion" == "skipped" ]]; then | |
| pending="true" | |
| continue | |
| fi | |
| if [[ "$conclusion" != "success" ]]; then | |
| failed="$check_name" | |
| break | |
| fi | |
| done | |
| title_status=$(echo "$latest_checks" | jq -r ' | |
| map(select(.name == "validate-pr-title")) | first | .status // "" | |
| ') | |
| title_conclusion=$(echo "$latest_checks" | jq -r ' | |
| map(select(.name == "validate-pr-title")) | first | .conclusion // "" | |
| ') | |
| if [[ -n "$title_status" && "$title_status" == "completed" && "$title_conclusion" != "success" && "$title_conclusion" != "skipped" ]]; then | |
| failed="validate-pr-title" | |
| elif [[ -n "$title_status" && "$title_status" != "completed" ]]; then | |
| pending="true" | |
| fi | |
| if [[ -n "$failed" ]]; then | |
| echo "CI prerequisite failed: $failed" | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if [[ "$pending" == "false" ]]; then | |
| break | |
| fi | |
| if (( SECONDS >= deadline )); then | |
| echo "Timed out waiting for CI prerequisites." | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| sleep 10 | |
| done | |
| if [[ "$EVENT_ACTION" == "ready_for_review" ]]; then | |
| CHECKS=$(echo "$latest_checks" | jq ' | |
| [ | |
| .[] | |
| | select(.name == "gpu-smoke") | |
| | .conclusion | |
| ] | |
| ') | |
| ALL_SUCCESS=$(echo "$CHECKS" | jq 'all(. == "success")') | |
| COUNT=$(echo "$CHECKS" | jq 'length') | |
| if [[ "$ALL_SUCCESS" == "true" && "$COUNT" -ge 1 ]]; then | |
| echo "GPU smoke already passed for SHA $HEAD_SHA; skipping" | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| fi | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| security-policy: | |
| needs: ci-prereq | |
| if: ${{ needs.ci-prereq.outputs.skip != 'true' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| is_fork: ${{ steps.policy.outputs.is_fork }} | |
| allow_fast_path: ${{ steps.policy.outputs.allow_fast_path }} | |
| skip_gpu_smoke: ${{ steps.policy.outputs.skip_gpu_smoke }} | |
| scope: ${{ steps.policy.outputs.scope }} | |
| pytest_targets: ${{ steps.policy.outputs.pytest_targets }} | |
| reason: ${{ steps.policy.outputs.reason }} | |
| steps: | |
| - name: Checkout test files | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| sparse-checkout: | | |
| tests/ops | |
| - name: Determine GPU policy and pytest scope | |
| id: policy | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| PR_NUMBER: ${{ github.event.pull_request.number || '' }} | |
| HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| BASE_REPO: ${{ github.repository }} | |
| run: | | |
| set -euo pipefail | |
| add_target() { | |
| local target="$1" | |
| if [[ -z "${seen_targets[$target]:-}" ]]; then | |
| selected_targets+=("$target") | |
| seen_targets["$target"]=1 | |
| fi | |
| } | |
| is_fork="false" | |
| allow_fast_path="true" | |
| skip_gpu_smoke="false" | |
| scope="full-smoke" | |
| pytest_targets="tests" | |
| reason="GPU smoke will run." | |
| if [[ "$EVENT_NAME" == "pull_request" && "$HEAD_REPO" != "$BASE_REPO" ]]; then | |
| is_fork="true" | |
| fi | |
| if [[ "$EVENT_NAME" == "pull_request" ]]; then | |
| mapfile -t changed_files < <( | |
| gh api "repos/${BASE_REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename' | |
| ) | |
| echo "Changed files (${#changed_files[@]}):" | |
| printf ' - %s\n' "${changed_files[@]}" | |
| # Fast path: if no Python files changed, GPU smoke is unnecessary. | |
| # For fork PRs, also check for high-risk files (workflow, build | |
| # config, etc.) that must still trigger GPU smoke even without | |
| # Python changes to avoid security bypass. | |
| has_python="false" | |
| has_high_risk="false" | |
| for path in "${changed_files[@]}"; do | |
| if [[ "$path" == *.py ]]; then | |
| has_python="true" | |
| fi | |
| if [[ "$is_fork" == "true" ]]; then | |
| case "$path" in | |
| .github/workflows/*|pyproject.toml|requirements*.txt|requirements*.in|setup.py|setup.cfg|Dockerfile*|scripts/bootstrap*|scripts/install*) | |
| has_high_risk="true" | |
| ;; | |
| esac | |
| fi | |
| done | |
| if [[ "$has_python" == "false" && "$has_high_risk" == "false" ]]; then | |
| skip_gpu_smoke="true" | |
| reason="No Python files changed; GPU smoke skipped." | |
| { | |
| echo "skip_gpu_smoke=$skip_gpu_smoke" | |
| echo "is_fork=$is_fork" | |
| echo "allow_fast_path=$allow_fast_path" | |
| echo "scope=skip" | |
| echo "pytest_targets=" | |
| echo "reason=$reason" | |
| } >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if [[ "$has_python" == "false" && "$has_high_risk" == "true" ]]; then | |
| # Fork PR changed high-risk build/infra files but no Python. | |
| # Run full GPU smoke to validate the environment is not broken. | |
| skip_gpu_smoke="false" | |
| allow_fast_path="false" | |
| scope="full-smoke" | |
| pytest_targets="tests" | |
| reason="Fork PR changed high-risk file(s) without Python changes; running full GPU smoke." | |
| { | |
| echo "skip_gpu_smoke=$skip_gpu_smoke" | |
| echo "is_fork=$is_fork" | |
| echo "allow_fast_path=$allow_fast_path" | |
| echo "scope=$scope" | |
| echo "pytest_targets=$pytest_targets" | |
| echo "reason=$reason" | |
| } >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| scope="skip" | |
| declare -A seen_targets=() | |
| selected_targets=() | |
| for path in "${changed_files[@]}"; do | |
| if [[ "$is_fork" == "true" ]]; then | |
| case "$path" in | |
| .github/ISSUE_TEMPLATE/*|.github/pull_request_template.md|tileops/ops/*.py|tests/ops/test_*.py|*.md) | |
| ;; | |
| .github/workflows/*|pyproject.toml|requirements*.txt|requirements*.in|setup.py|setup.cfg|Dockerfile*|scripts/bootstrap*|scripts/install*) | |
| allow_fast_path="false" | |
| reason="Fork PR changed a high-risk file: $path" | |
| ;; | |
| *) | |
| allow_fast_path="false" | |
| reason="Fork PR changed a file outside the future fast-path policy: $path" | |
| ;; | |
| esac | |
| fi | |
| case "$path" in | |
| .github/ISSUE_TEMPLATE/*|.github/pull_request_template.md|.github/workflows/auto-label.yml|.github/workflows/stale-issues.yml|*.md) | |
| ;; | |
| tileops/ops/*.py) | |
| test_path="tests/ops/test_$(basename "${path%.py}").py" | |
| if [[ -f "$test_path" ]]; then | |
| add_target "$test_path" | |
| if [[ "$scope" == "skip" ]]; then | |
| scope="targeted" | |
| fi | |
| else | |
| scope="full-smoke" | |
| reason="No direct smoke test mapping for $path; falling back to full smoke." | |
| break | |
| fi | |
| ;; | |
| tests/ops/test_*.py) | |
| add_target "$path" | |
| if [[ "$scope" == "skip" ]]; then | |
| scope="targeted" | |
| fi | |
| ;; | |
| *) | |
| scope="full-smoke" | |
| reason="Shared or unsupported path $path; falling back to full smoke." | |
| break | |
| ;; | |
| esac | |
| done | |
| if [[ "$scope" == "skip" ]]; then | |
| if [[ "$allow_fast_path" == "true" ]]; then | |
| # All changed files matched docs/metadata patterns; skip GPU smoke | |
| # for both trusted and fork PRs. | |
| skip_gpu_smoke="true" | |
| pytest_targets="" | |
| reason="Docs-only or metadata-only change; GPU smoke skipped." | |
| elif [[ "$is_fork" == "true" ]]; then | |
| scope="full-smoke" | |
| pytest_targets="tests" | |
| reason="${reason}; current policy still runs GPU smoke." | |
| else | |
| skip_gpu_smoke="true" | |
| pytest_targets="" | |
| reason="Docs-only or metadata-only change; GPU smoke skipped." | |
| fi | |
| elif [[ "$scope" == "targeted" ]]; then | |
| if [[ "${#selected_targets[@]}" -eq 0 ]]; then | |
| scope="full-smoke" | |
| pytest_targets="tests" | |
| reason="No mapped tests were collected; falling back to full smoke." | |
| else | |
| pytest_targets="${selected_targets[*]}" | |
| if [[ "$is_fork" == "true" && "$allow_fast_path" != "true" ]]; then | |
| reason="${reason}; current policy still runs targeted GPU smoke." | |
| else | |
| reason="Running targeted GPU smoke tests." | |
| fi | |
| fi | |
| fi | |
| fi | |
| { | |
| echo "is_fork=$is_fork" | |
| echo "allow_fast_path=$allow_fast_path" | |
| echo "skip_gpu_smoke=$skip_gpu_smoke" | |
| echo "scope=$scope" | |
| echo "pytest_targets=$pytest_targets" | |
| echo "reason=$reason" | |
| } >> "$GITHUB_OUTPUT" | |
| gpu-smoke: | |
| needs: [ci-prereq, security-policy] | |
| if: ${{ needs.ci-prereq.outputs.skip != 'true' && needs.security-policy.outputs.skip_gpu_smoke != 'true' }} | |
| runs-on: tileops-metax-runner | |
| steps: | |
| - name: Checkout code for fork PR | |
| if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| - name: Checkout code for trusted branch | |
| if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository }} | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y git | |
| - name: Set up MACA environment | |
| run: | | |
| set -euo pipefail | |
| # Add conda to PATH | |
| export PATH="/opt/conda/bin:${PATH}" | |
| # Set MACA environment variables | |
| export USE_MACA=ON | |
| export UV_INDEX="http://mirrors.aliyun.com/pypi/simple" | |
| # Create virtual environment | |
| python -m venv .venv | |
| source .venv/bin/activate | |
| # Copy pre-installed MACA torch packages from conda to venv | |
| cp -rf /opt/conda/lib/python3.12/site-packages/* .venv/lib/python3.12/site-packages/ | |
| # Verify torch installation | |
| python -c "import sys; print('Python version:', sys.version)" | |
| python -c "import torch; print('Torch version:', torch.__version__)" | |
| # Set up cache directories | |
| RUNTIME_ROOT="${{ github.workspace }}/.cache" | |
| TILELANG_CACHE_DIR="${RUNTIME_ROOT}/tilelang/cache" | |
| TRITON_CACHE_DIR="${RUNTIME_ROOT}/triton/cache" | |
| WHEEL_DIR="${RUNTIME_ROOT}/wheels" | |
| mkdir -p "${TILELANG_CACHE_DIR}" "${TRITON_CACHE_DIR}" "${WHEEL_DIR}" | |
| # Save environment for subsequent steps | |
| { | |
| echo "PATH=${PATH}" | |
| echo "USE_MACA=ON" | |
| echo "UV_INDEX=http://mirrors.aliyun.com/pypi/simple" | |
| echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" | |
| echo "RUNTIME_ROOT=${RUNTIME_ROOT}" | |
| echo "TILELANG_CACHE_DIR=${TILELANG_CACHE_DIR}" | |
| echo "TRITON_CACHE_DIR=${TRITON_CACHE_DIR}" | |
| echo "WHEEL_DIR=${WHEEL_DIR}" | |
| } >> "$GITHUB_ENV" | |
| - name: Install dependencies | |
| run: | | |
| set -euo pipefail | |
| source .venv/bin/activate | |
| pip install --no-cache-dir \ | |
| einops numpy "tqdm>=4.62.3" "typing_extensions>=4.10.0" \ | |
| cloudpickle ml_dtypes psutil Cython \ | |
| setuptools wheel ninja | |
| pip install --no-cache-dir \ | |
| "pytest==9.0.2" \ | |
| "pytest-xdist>=3.0" \ | |
| "pyyaml>=6.0" | |
| - name: Validate MACA GPU frequency | |
| run: | | |
| set -euo pipefail | |
| source .venv/bin/activate | |
| TARGET_CLOCK_MHZ=1830 | |
| RETRIES=5 | |
| SLEEP_SECONDS=2 | |
| if command -v mx-smi >/dev/null 2>&1; then | |
| echo "Detected MACA GPUs:" | |
| mx-smi -L | |
| attempt=1 | |
| while [ "$attempt" -le "$RETRIES" ]; do | |
| echo "Validation attempt ${attempt}/${RETRIES}" | |
| mismatch=0 | |
| while IFS=',' read -r gpu_index gpu_name gpu_clock; do | |
| gpu_index="$(echo "$gpu_index" | xargs)" | |
| gpu_name="$(echo "$gpu_name" | xargs)" | |
| gpu_clock="$(echo "$gpu_clock" | xargs)" | |
| echo "GPU ${gpu_index} (${gpu_name}) clock: ${gpu_clock} MHz" | |
| if [[ "$gpu_clock" != "$TARGET_CLOCK_MHZ" ]]; then | |
| mismatch=1 | |
| fi | |
| done < <(mx-smi --query-gpu=index,name,clocks.current.graphics --format=csv,noheader,nounits) | |
| if [[ "$mismatch" -eq 0 ]]; then | |
| echo "All GPUs locked at ${TARGET_CLOCK_MHZ} MHz." | |
| break | |
| fi | |
| if [[ "$attempt" -eq "$RETRIES" ]]; then | |
| echo "::warning::GPU frequency validation failed. Expected ${TARGET_CLOCK_MHZ} MHz." | |
| fi | |
| sleep "$SLEEP_SECONDS" | |
| attempt=$((attempt + 1)) | |
| done | |
| else | |
| echo "::warning::mx-smi not available, skipping GPU frequency validation." | |
| fi | |
| - name: Install TileOPs and TileLang | |
| uses: ./.github/actions/install-tileops | |
| - name: Run tests | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| TEST_SCOPE: ${{ needs.security-policy.outputs.scope }} | |
| PYTEST_TARGETS: ${{ needs.security-policy.outputs.pytest_targets }} | |
| run: | | |
| set -euo pipefail | |
| source .venv/bin/activate | |
| current_dir="$(pwd)" | |
| export PYTHONPATH="${current_dir}${PYTHONPATH:+:$PYTHONPATH}" | |
| echo "PYTHONPATH=$PYTHONPATH" | |
| TEST_MARK_EXPR="smoke or full" | |
| if [[ "$EVENT_NAME" == "pull_request" ]]; then | |
| TEST_MARK_EXPR="smoke" | |
| fi | |
| echo "Resolved test scope: ${TEST_SCOPE}" | |
| echo "Resolved pytest targets: ${PYTEST_TARGETS}" | |
| read -r -a TARGETS <<< "${PYTEST_TARGETS}" | |
| echo "Running pytest -m \"${TEST_MARK_EXPR}\" on ${#TARGETS[@]} target(s)" | |
| python -m pytest -q "${TARGETS[@]}" -m "${TEST_MARK_EXPR}" \ | |
| --junit-xml=gpu_smoke_results.xml \ | |
| | tee gpu_smoke.log | |
| - name: Generate gpu-smoke report | |
| if: ${{ always() && hashFiles('gpu_smoke_results.xml') != '' }} | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| run: | | |
| set -euo pipefail | |
| source .venv/bin/activate | |
| REPORT_TARGET="smoke, full" | |
| if [[ "$EVENT_NAME" == "pull_request" ]]; then | |
| REPORT_TARGET="smoke" | |
| fi | |
| if [[ -f scripts/gpu_smoke_report.py ]]; then | |
| python scripts/gpu_smoke_report.py \ | |
| --test-xml gpu_smoke_results.xml \ | |
| --target "${REPORT_TARGET}" \ | |
| --output gpu_smoke_report.md \ | |
| || echo "::warning::gpu_smoke_report.py failed" | |
| else | |
| echo "::warning::scripts/gpu_smoke_report.py not found, skipping report generation" | |
| fi | |
| - name: Post gpu-smoke report to step summary | |
| if: ${{ always() && hashFiles('gpu_smoke_report.md') != '' }} | |
| run: cat gpu_smoke_report.md >> "$GITHUB_STEP_SUMMARY" | |
| - name: Cache stats | |
| if: ${{ always() }} | |
| env: | |
| IS_FORK: ${{ needs.security-policy.outputs.is_fork }} | |
| run: | | |
| set -u | |
| echo "=== Cache stats (is_fork=${IS_FORK}) ===" | |
| echo "VENV_REUSED=${VENV_REUSED:-unknown}" | |
| echo "FORK_CAN_COPY_TRUSTED_VENV=${FORK_CAN_COPY_TRUSTED_VENV:-unknown}" | |
| echo "TRUSTED_VENV_PATH=${TRUSTED_VENV_PATH:-unknown}" | |
| for cache_dir in "${TILELANG_CACHE_DIR:-}" "${TRITON_CACHE_DIR:-}" "${WHEEL_DIR:-}"; do | |
| if [[ -z "${cache_dir}" ]]; then | |
| continue | |
| fi | |
| if [[ -d "${cache_dir}" ]]; then | |
| file_count=$(set -o pipefail; find "${cache_dir}" -type f 2>/dev/null | wc -l 2>/dev/null) || file_count="unknown" | |
| cache_size=$(set -o pipefail; du -sh "${cache_dir}" 2>/dev/null | cut -f1 2>/dev/null) || cache_size="unknown" | |
| if [[ -z "${file_count}" ]]; then file_count="unknown"; fi | |
| if [[ -z "${cache_size}" ]]; then cache_size="unknown"; fi | |
| echo "${cache_dir}: ${file_count} files, ${cache_size}" | |
| else | |
| echo "${cache_dir}: does not exist" | |
| fi | |
| done | |
| if [[ "${IS_FORK}" == "true" ]]; then | |
| if [[ "${FORK_CAN_COPY_TRUSTED_VENV:-false}" == "true" ]]; then | |
| echo "CACHE_SIGNAL: fork venv=hit" | |
| else | |
| echo "CACHE_SIGNAL: fork venv=miss (cold pip install path)" | |
| fi | |
| else | |
| if [[ "${VENV_REUSED:-false}" == "true" ]]; then | |
| echo "CACHE_SIGNAL: trusted venv=hit" | |
| else | |
| echo "CACHE_SIGNAL: trusted venv=miss (fresh venv created)" | |
| fi | |
| fi | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: gpu_smoke_artifacts | |
| path: | | |
| gpu_smoke.log | |
| gpu_smoke_results.xml | |
| gpu_smoke_report.md | |
| retention-days: 7 | |
| - name: Cleanup isolated fork state | |
| if: ${{ always() && needs.security-policy.outputs.is_fork == 'true' }} | |
| run: | | |
| if [[ -n "${RUNTIME_ROOT:-}" ]]; then | |
| rm -rf "${RUNTIME_ROOT}" | |
| fi |