test.yml

name: Perf Tests in Container

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

env:
  MPI_HOME: /usr/local/mpi
  PERF_BIN: /__w/FlagCX/FlagCX/test/perf/host_api/build/bin
  DEVICE_API_BIN: /__w/FlagCX/FlagCX/test/perf/device_api/build/bin

jobs:
  perf-test:
    runs-on: [self-hosted, cx-build]
    container:
      image: localhost:5000/flagscale:cuda12.8.1-cudnn9.7.1-python3.12-torch2.7.0-time2507111538
      options: --gpus all --privileged --ipc=host --ulimit memlock=-1 --ulimit stack=67108864

    steps:
      - name: Checkout Code
        uses: actions/checkout@v6
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
          ssh-strict: true
          ssh-user: git
          persist-credentials: true
          clean: true
          sparse-checkout-cone-mode: true
          fetch-tags: false
          show-progress: true
          lfs: false
          submodules: true
          set-safe-directory: true

      # FORCE_DEFAULT_PATH only affects kernel object compilation (comm traits);
      # host-side library and vendor API dispatch are unchanged.
      - name: Build FlagCX
        run: |
          cd /__w/FlagCX/FlagCX
          make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1

      - name: Build perf tests
        run: |
          cd /__w/FlagCX/FlagCX/test/perf
          make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1

      - name: Wait for GPU
        shell: bash
        run: |
          cd /__w/FlagCX/FlagCX
          source test/script/_gpu_check.sh
          wait_for_gpu

      - name: "Perf tests (homoRunner)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reducescatter -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reduce -b 128M -e 1G -f 2 -r 0 -p 1

      - name: "Perf tests (uniRunner)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1

      - name: "Registration -R 1 (homoRunner)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1 -R 1

      - name: "Registration -R 1 (uniRunner P2P)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 -R 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 -R 1

      - name: "Registration -R 1 (uniRunner NET)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_P2P_DISABLE=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 -R 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_P2P_DISABLE=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 -R 1

      - name: "RMA one-sided (put/get)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 2 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_put -b 8 -e 1M -f 2 -R 2
          mpirun -np 2 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_get -b 8 -e 1M -f 2 -R 2

      - name: "Device API perf (intra-node)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          mpirun -np 8 --allow-run-as-root \
            -x FLAGCX_USE_HETERO_COMM=1 \
            -x FLAGCX_MEM_ENABLE=1 \
            -x FLAGCX_VMM_ENABLE=0 \
            -x FLAGCX_P2P_DISABLE=1 \
            -x FLAGCX_DEBUG=TRACE -x FLAGCX_DEBUG_SUBSYS=ALL \
            -x LD_LIBRARY_PATH \
            $DEVICE_API_BIN/perf_allreduce_intranode -b 1M -e 64M -f 2 -R 1

      - name: "Device API perf (inter-node one-sided)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          COMMON_ENV="-x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_P2P_DISABLE=1 -x LD_LIBRARY_PATH"
          NODE1_FLAG="-x CUDA_VISIBLE_DEVICES=0,1,2,3 -x FLAGCX_HOSTID=node0 -x FLAGCX_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3"
          NODE2_FLAG="-x CUDA_VISIBLE_DEVICES=4,5,6,7 -x FLAGCX_HOSTID=node1 -x FLAGCX_IB_HCA=mlx5_4,mlx5_5,mlx5_6,mlx5_7"
          mpirun --allow-run-as-root \
            -np 4 ${COMMON_ENV} ${NODE1_FLAG} $DEVICE_API_BIN/perf_internode_onesided -b 1M -e 64M -f 2 -R 1 \
            : -np 4 ${COMMON_ENV} ${NODE2_FLAG} $DEVICE_API_BIN/perf_internode_onesided -b 1M -e 64M -f 2 -R 1

      - name: "P2P Engine perf (one-sided read/write)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          mpirun -np 2 --allow-run-as-root $PERF_BIN/perf_p2p_engine -b 4K -e 64M -f 2 -n 10