rpu3.6-build-and-test.yml

# Copyright 2025-     FlagOS Contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

name: Rpu3.6-Build-And-Test

on:
  push:
    branches: [ "main", "triton_v3.6.x" ]
  pull_request:
    branches: [ "main", "triton_v3.6.x" ]
    types: [opened, synchronize, reopened, ready_for_review]

permissions:
  contents: read
  pull-requests: read

concurrency:
  # Serialize ALL runs on the single shared RPU board with one fixed, board-wide
  # group (NOT per-PR): only one job may drive the board at a time. Different PRs
  # -- and manual re-runs -- otherwise race, sharing ~/FlagTree-ci (rsync
  # --delete), the build dir, and /dev/rpu, which caused flaky failures. Queue
  # (cancel-in-progress: false) so an in-flight board test is never interrupted
  # mid-run (a cancelled smoke can orphan a launch_kernel_runner that wedges
  # /dev/rpu for the next job).
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: false

# The RPU CI runner is a small x86 host that orchestrates the build/test on a
# separate aarch64 RPU board over SSH. The board cannot be reached directly
# from a generic runner, so every wiring detail (board address, login user,
# SSH key) lives only in the runner's local ~/.ssh/config under the opaque
# alias `rpuboard` -- nothing host-specific is committed here. The runner
# checks the source out locally, mirrors it to the board, then drives a clean
# build and the test suite on the board.
jobs:
  rpu36x-unit-test:
    runs-on: rpu3.6
    if: ${{ github.repository == 'flagos-ai/flagtree' && github.event.pull_request.draft == false }}
    steps:
      - name: Setup environment
        shell: bash
        run: |
          # The orchestrating runner has no ~/env.sh (the toolchain lives on the
          # board); tolerate its absence and only forward any proxy settings.
          source ~/env.sh 2>/dev/null || true
          env | grep -E '^(http_proxy|https_proxy|all_proxy|no_proxy)=' >> $GITHUB_ENV || true

      - name: Smart Checkout
        uses: flagos-ai/FlagTree/.github/actions/smart-checkout@main
        with:
          checkout_version: 'v6'

      - name: Check if backend-relevant files changed
        id: check_backend
        uses: flagos-ai/FlagTree/.github/actions/check-backend-changed@main
        with:
          backend: rpu

      - name: Sync source to RPU board
        if: steps.check_backend.outputs.should_skip != 'true'
        shell: bash
        run: |
          set -x
          # Exact mirror to the board (--delete removes files deleted upstream;
          # .git is not needed for the build). Retry: the board's sshd can drop
          # the connection during key exchange under load
          # (kex_exchange_identification: Connection closed), which is transient.
          n=0
          until rsync -az --delete --exclude '.git' \
              -e 'ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=20' \
              ./ rpuboard:~/FlagTree-ci/; do
            n=$((n+1))
            if [ "$n" -ge 6 ]; then echo "::error::rsync to board failed after $n attempts"; exit 1; fi
            echo "sync attempt $n failed (transient SSH drop?); retrying in $((n*15))s"
            sleep $((n*15))
          done

      - name: Build FlagTree
        if: steps.check_backend.outputs.should_skip != 'true'
        shell: bash
        run: |
          set -x
          # Clean build on the board: drop the previous build tree so nothing is
          # stale, then build/install. The build dir is ./build at the repo root
          # (setup.py lives here, not under python/), so clean that -- otherwise
          # it persists across runs and fills the board disk. ccache (configured
          # in the board's env.sh) keeps the rebuild fast.
          # Retry only on SSH connection errors (exit 255 = kex drop under
          # load); a real build failure returns a different code and is NOT
          # retried. The build is idempotent (rm -rf build), so a reconnect is
          # safe.
          n=0
          while :; do
            rc=0
            ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=20 rpuboard 'set -e
              cd ~/FlagTree-ci
              source ~/env.sh
              export FLAGTREE_BACKEND=rpu
              rm -rf build
              MAX_JOBS=8 pip3 install --user --force-reinstall . --no-build-isolation' || rc=$?
            if [ "$rc" -ne 255 ]; then exit "$rc"; fi
            n=$((n+1))
            if [ "$n" -ge 6 ]; then echo "::error::ssh to board failed (connection) after $n attempts"; exit 255; fi
            echo "ssh connection dropped (rc=255); retrying $n in $((n*15))s"
            sleep $((n*15))
          done

      - name: Clear cache if ClearCache label present
        if: steps.check_backend.outputs.should_skip != 'true'
        id: clear_cache
        uses: flagos-ai/FlagTree/.github/actions/clear-cache-if-labeled@main
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          backend: rpu

      - name: Unit Test
        if: steps.check_backend.outputs.should_skip != 'true'
        shell: bash
        run: |
          set -x
          # Unit suite, then the on-board launch_kernel dispatch smoke. The smoke
          # uses --require-board so a misconfigured board (missing toolchain /
          # runtime / /dev/rpu) fails the job instead of silently skipping.
          # Retry only on SSH connection errors (255 = kex drop under load); a
          # real test failure returns a different code and is NOT retried. Unit +
          # smoke are idempotent (the smoke reaps first), so a reconnect re-runs
          # safely.
          n=0
          while :; do
            rc=0
            ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=20 rpuboard 'set -e
            cd ~/FlagTree-ci
            source ~/env.sh
            # Reap any leaked launch_kernel_runner before the smoke: an orphan
            # from a prior run holding /dev/rpu leaves run_work stuck in the RPU
            # queue and makes every dispatch here time out. Runs are serialized
            # board-wide (see workflow `concurrency: rpu3.6-board`), so no
            # concurrent job owns a runner -- reap ALL of them, not just old
            # ones. (The previous 130s age guard left a gap: a runner orphaned
            # <130s before the next run survived and wedged it.) Match on the
            # real executable via /proc/PID/exe so this never kills its own
            # reap shell, whose cmdline also contains "launch_kernel_runner".
            for pid in $(ls /proc 2>/dev/null | grep -E "^[0-9]+$"); do
              case "$(readlink /proc/$pid/exe 2>/dev/null)" in
                */launch_kernel_runner) sudo -n kill -9 "$pid" 2>/dev/null || true ;;
              esac
            done
            pytest -q third_party/rpu/python/test/unit
            python3 third_party/rpu/python/test/board/lk_board_smoke.py --require-board -v' || rc=$?
            if [ "$rc" -ne 255 ]; then exit "$rc"; fi
            n=$((n+1))
            if [ "$n" -ge 6 ]; then echo "::error::ssh to board failed (connection) after $n attempts"; exit 255; fi
            echo "ssh connection dropped (rc=255); retrying $n in $((n*15))s"
            sleep $((n*15))
          done