[SPEC] Spec mthreads root python code #403
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rpu3.6-Build-And-Test | |
| on: | |
| push: | |
| branches: [ "main", "triton_v3.6.x" ] | |
| pull_request: | |
| branches: [ "main", "triton_v3.6.x" ] | |
| concurrency: | |
| # Serialize ALL runs on the single shared RPU board with one fixed, board-wide | |
| # group (NOT per-PR): only one job may drive the board at a time. Different PRs | |
| # -- and manual re-runs -- otherwise race, sharing ~/FlagTree-ci (rsync | |
| # --delete), the build dir, and /dev/rpu, which caused flaky failures. Queue | |
| # (cancel-in-progress: false) so an in-flight board test is never interrupted | |
| # mid-run (a cancelled smoke can orphan a launch_kernel_runner that wedges | |
| # /dev/rpu for the next job). | |
| group: rpu3.6-board | |
| cancel-in-progress: false | |
| # The RPU CI runner is a small x86 host that orchestrates the build/test on a | |
| # separate aarch64 RPU board over SSH. The board cannot be reached directly | |
| # from a generic runner, so every wiring detail (board address, login user, | |
| # SSH key) lives only in the runner's local ~/.ssh/config under the opaque | |
| # alias `rpuboard` -- nothing host-specific is committed here. The runner | |
| # checks the source out locally, mirrors it to the board, then drives a clean | |
| # build and the test suite on the board. | |
| jobs: | |
| rpu36x-unit-test: | |
| runs-on: rpu3.6 | |
| if: ${{ github.repository == 'FlagTree/flagtree' || github.repository == 'flagos-ai/flagtree' }} | |
| steps: | |
| - name: Setup environment | |
| shell: bash | |
| run: | | |
| # The orchestrating runner has no ~/env.sh (the toolchain lives on the | |
| # board); tolerate its absence and only forward any proxy settings. | |
| source ~/env.sh 2>/dev/null || true | |
| env | grep -E '^(http_proxy|https_proxy|all_proxy|no_proxy)=' >> $GITHUB_ENV || true | |
| - name: Smart Checkout | |
| uses: flagos-ai/FlagTree/.github/actions/smart-checkout@main | |
| with: | |
| checkout_version: 'v6' | |
| - name: Check if backend-relevant files changed | |
| id: check_backend | |
| uses: flagos-ai/FlagTree/.github/actions/check-backend-changed@main | |
| with: | |
| backend: rpu | |
| - name: Sync source to RPU board | |
| if: steps.check_backend.outputs.should_skip != 'true' | |
| shell: bash | |
| run: | | |
| set -x | |
| # Exact mirror to the board (--delete removes files deleted upstream; | |
| # .git is not needed for the build). Retry: the board's sshd can drop | |
| # the connection during key exchange under load | |
| # (kex_exchange_identification: Connection closed), which is transient. | |
| n=0 | |
| until rsync -az --delete --exclude '.git' \ | |
| -e 'ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=20' \ | |
| ./ rpuboard:~/FlagTree-ci/; do | |
| n=$((n+1)) | |
| if [ "$n" -ge 6 ]; then echo "::error::rsync to board failed after $n attempts"; exit 1; fi | |
| echo "sync attempt $n failed (transient SSH drop?); retrying in $((n*15))s" | |
| sleep $((n*15)) | |
| done | |
| - name: Build FlagTree | |
| if: steps.check_backend.outputs.should_skip != 'true' | |
| shell: bash | |
| run: | | |
| set -x | |
| # Clean build on the board: drop the previous build tree so nothing is | |
| # stale, then build/install. The build dir is ./build at the repo root | |
| # (setup.py lives here, not under python/), so clean that -- otherwise | |
| # it persists across runs and fills the board disk. ccache (configured | |
| # in the board's env.sh) keeps the rebuild fast. | |
| # Retry only on SSH connection errors (exit 255 = kex drop under | |
| # load); a real build failure returns a different code and is NOT | |
| # retried. The build is idempotent (rm -rf build), so a reconnect is | |
| # safe. | |
| n=0 | |
| while :; do | |
| rc=0 | |
| ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=20 rpuboard 'set -e | |
| cd ~/FlagTree-ci | |
| source ~/env.sh | |
| export FLAGTREE_BACKEND=rpu | |
| rm -rf build | |
| MAX_JOBS=8 pip3 install --user --force-reinstall . --no-build-isolation' || rc=$? | |
| if [ "$rc" -ne 255 ]; then exit "$rc"; fi | |
| n=$((n+1)) | |
| if [ "$n" -ge 6 ]; then echo "::error::ssh to board failed (connection) after $n attempts"; exit 255; fi | |
| echo "ssh connection dropped (rc=255); retrying $n in $((n*15))s" | |
| sleep $((n*15)) | |
| done | |
| - name: Unit Test | |
| if: steps.check_backend.outputs.should_skip != 'true' | |
| shell: bash | |
| run: | | |
| set -x | |
| # Unit suite, then the on-board launch_kernel dispatch smoke. The smoke | |
| # uses --require-board so a misconfigured board (missing toolchain / | |
| # runtime / /dev/rpu) fails the job instead of silently skipping. | |
| # Retry only on SSH connection errors (255 = kex drop under load); a | |
| # real test failure returns a different code and is NOT retried. Unit + | |
| # smoke are idempotent (the smoke reaps first), so a reconnect re-runs | |
| # safely. | |
| n=0 | |
| while :; do | |
| rc=0 | |
| ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=20 rpuboard 'set -e | |
| cd ~/FlagTree-ci | |
| source ~/env.sh | |
| # Reap any leaked launch_kernel_runner before the smoke: an orphan | |
| # from a prior run holding /dev/rpu leaves run_work stuck in the RPU | |
| # queue and makes every dispatch here time out. Runs are serialized | |
| # board-wide (see workflow `concurrency: rpu3.6-board`), so no | |
| # concurrent job owns a runner -- reap ALL of them, not just old | |
| # ones. (The previous 130s age guard left a gap: a runner orphaned | |
| # <130s before the next run survived and wedged it.) Match on the | |
| # real executable via /proc/PID/exe so this never kills its own | |
| # reap shell, whose cmdline also contains "launch_kernel_runner". | |
| for pid in $(ls /proc 2>/dev/null | grep -E "^[0-9]+$"); do | |
| case "$(readlink /proc/$pid/exe 2>/dev/null)" in | |
| */launch_kernel_runner) sudo -n kill -9 "$pid" 2>/dev/null || true ;; | |
| esac | |
| done | |
| pytest -q third_party/rpu/python/test/unit | |
| python3 third_party/rpu/python/test/board/lk_board_smoke.py --require-board -v' || rc=$? | |
| if [ "$rc" -ne 255 ]; then exit "$rc"; fi | |
| n=$((n+1)) | |
| if [ "$n" -ge 6 ]; then echo "::error::ssh to board failed (connection) after $n attempts"; exit 255; fi | |
| echo "ssh connection dropped (rc=255); retrying $n in $((n*15))s" | |
| sleep $((n*15)) | |
| done |