test.yml

name: Perf Tests in Container

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

env:
  MPI_HOME: /usr/local/mpi
  PERF_BIN: /__w/FlagCX/FlagCX/test/perf/host_api/build/bin
  DEVICE_API_BIN: /__w/FlagCX/FlagCX/test/perf/device_api/build/bin

jobs:
  perf-test:
    runs-on: [self-hosted, cx-build]
    container:
      image: localhost:5000/flagscale:cuda12.8.1-cudnn9.7.1-python3.12-torch2.7.0-time2507111538
      options: --gpus all --privileged --ipc=host --ulimit memlock=-1 --ulimit stack=67108864

    steps:
      - name: Checkout Code
        uses: actions/checkout@v7
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
          ssh-strict: true
          ssh-user: git
          persist-credentials: true
          clean: true
          sparse-checkout-cone-mode: true
          fetch-tags: false
          show-progress: true
          lfs: false
          submodules: true
          set-safe-directory: true

      # FORCE_DEFAULT_PATH only affects kernel object compilation (comm traits);
      # host-side library and vendor API dispatch are unchanged.
      - name: Build FlagCX
        run: |
          cd /__w/FlagCX/FlagCX
          make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1

      - name: Build perf tests
        run: |
          cd /__w/FlagCX/FlagCX/test/perf
          make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1

      - name: Wait for GPU
        shell: bash
        run: |
          cd /__w/FlagCX/FlagCX
          source test/script/_gpu_check.sh
          wait_for_gpu

      - name: "Perf tests (homoRunner)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reducescatter -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reduce -b 128M -e 1G -f 2 -r 0 -p 1

      - name: "Perf tests (uniRunner)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1

      - name: "Registration -R 1 (homoRunner)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1 -R 1

      - name: "Registration -R 1 (uniRunner P2P)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 -R 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 -R 1

      - name: "Registration -R 1 (uniRunner NET)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_P2P_DISABLE=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 -R 1
          mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_P2P_DISABLE=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 -R 1

      - name: "RMA one-sided (put/get)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 2 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_put -b 8 -e 1M -f 2 -R 2
          mpirun -np 2 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_get -b 8 -e 1M -f 2 -R 2

      - name: "Device API perf (intra-node)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          mpirun -np 8 --allow-run-as-root \
            -x FLAGCX_USE_HETERO_COMM=1 \
            -x FLAGCX_MEM_ENABLE=1 \
            -x FLAGCX_VMM_ENABLE=0 \
            -x FLAGCX_P2P_DISABLE=1 \
            -x FLAGCX_DEBUG=TRACE -x FLAGCX_DEBUG_SUBSYS=ALL \
            -x LD_LIBRARY_PATH \
            $DEVICE_API_BIN/perf_allreduce_intranode -b 1M -e 64M -f 2 -R 1

      - name: "Device API perf (inter-node one-sided)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          COMMON_ENV="-x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_P2P_DISABLE=1 -x LD_LIBRARY_PATH"
          NODE1_FLAG="-x CUDA_VISIBLE_DEVICES=0,1,2,3 -x FLAGCX_HOSTID=node0 -x FLAGCX_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3"
          NODE2_FLAG="-x CUDA_VISIBLE_DEVICES=4,5,6,7 -x FLAGCX_HOSTID=node1 -x FLAGCX_IB_HCA=mlx5_4,mlx5_5,mlx5_6,mlx5_7"
          mpirun --allow-run-as-root \
            -np 4 ${COMMON_ENV} ${NODE1_FLAG} $DEVICE_API_BIN/perf_internode_onesided -b 1M -e 64M -f 2 -R 1 \
            : -np 4 ${COMMON_ENV} ${NODE2_FLAG} $DEVICE_API_BIN/perf_internode_onesided -b 1M -e 64M -f 2 -R 1

      - name: "P2P Engine perf (one-sided read/write)"
        run: |
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
          mpirun -np 2 --allow-run-as-root $PERF_BIN/perf_p2p_engine -b 4K -e 64M -f 2 -n 10

  perf-test-hygon:
    name: perf-test (hygon)
    runs-on: [self-hosted, hg-8g-cicd-flagcx]
    container:
      image: harbor.baai.ac.cn/flagos-dev/flagcx:b079886-hygon-dev
      volumes:
        - /usr/local/hyhal:/opt/hyhal:ro
      options: >-
        --ipc=host
        --privileged=true
        --shm-size=100gb
        --ulimit memlock=-1
        --ulimit stack=67108864
        --device=/dev/kfd
        --device=/dev/dri
        --group-add video
    env:
      MPI_HOME: /opt/mpi
      PERF_BIN: /__w/FlagCX/FlagCX/test/perf/host_api/build/bin

    steps:
      - name: Checkout Code
        uses: actions/checkout@v7
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
          ssh-strict: true
          ssh-user: git
          persist-credentials: true
          clean: true
          sparse-checkout-cone-mode: true
          fetch-tags: false
          show-progress: true
          lfs: false
          submodules: true
          set-safe-directory: true

      - name: Prepare Hygon environment
        run: |
          source /opt/dtk/env.sh
          export CUDA_PATH=/opt/dtk/cuda/cuda-12
          export CUDA_HOME=$CUDA_PATH
          echo "CUDA_PATH=$CUDA_PATH" >> "$GITHUB_ENV"
          echo "CUDA_HOME=$CUDA_HOME" >> "$GITHUB_ENV"
          echo "PATH=/opt/mpi/bin:$PATH" >> "$GITHUB_ENV"
          echo "LD_LIBRARY_PATH=/opt/mpi/lib:/opt/dtk/lib:$CUDA_PATH/lib64:$LD_LIBRARY_PATH" >> "$GITHUB_ENV"
          hy-smi --showproductname
          python3 -c 'import torch; print("torch", torch.__version__, "devices", torch.cuda.device_count()); assert torch.cuda.device_count() >= 8'

      - name: Build FlagCX
        run: |
          cd /__w/FlagCX/FlagCX
          source /opt/dtk/env.sh
          make -j$(nproc) USE_DU=1 PLATFORM_EXTRA_SRCS=flagcx/adaptor/device_api/default_dev_api_backend.cc

      - name: Build perf tests
        run: |
          cd /__w/FlagCX/FlagCX/test/perf
          source /opt/dtk/env.sh
          make -j$(nproc) USE_DU=1 MPI_HOME=/opt/mpi CCL_HOME=/opt/dtk/cuda/cuda-12

      # Hygon currently supports the homogeneous single-node path here.
      # Heterogeneous, RMA, and device-API paths are covered only where the
      # backend advertises support; unsupported cases self-skip in the tests.
      - name: Perf tests (Hygon homogeneous 8-chip)
        run: |
          cd /__w/FlagCX/FlagCX
          source /opt/dtk/env.sh
          export PATH=/opt/mpi/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:/opt/mpi/lib:/opt/dtk/lib:/opt/dtk/cuda/cuda-12/lib64:$LD_LIBRARY_PATH
          set -e
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reducescatter -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reduce -b 128M -e 1G -f 2 -r 0 -p 1

  perf-test-cuda:
    name: perf-test (cuda)
    runs-on: [self-hosted, nv-8g-cicd-flagcx]
    container:
      image: harbor.baai.ac.cn/flagos-dev/flagcx:55c5c6f-cuda-dev
      options: >-
        --gpus all
        --privileged
        --ipc=host
        --ulimit memlock=-1
        --ulimit stack=67108864
    env:
      MPI_HOME: /usr/local/mpi
      PERF_BIN: /__w/FlagCX/FlagCX/test/perf/host_api/build/bin

    steps:
      - name: Checkout Code
        uses: actions/checkout@v7
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
          ssh-strict: true
          ssh-user: git
          persist-credentials: true
          clean: true
          sparse-checkout-cone-mode: true
          fetch-tags: false
          show-progress: true
          lfs: false
          submodules: true
          set-safe-directory: true

      - name: Prepare CUDA environment
        run: |
          source /__w/FlagCX/FlagCX/.github/scripts/set_env/cuda.sh
          echo "PATH=$MPI_HOME/bin:$PATH" >> "$GITHUB_ENV"
          echo "MPI_HOME=$MPI_HOME" >> "$GITHUB_ENV"
          echo "LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$MPI_HOME/lib:$LD_LIBRARY_PATH" >> "$GITHUB_ENV"
          nvidia-smi

      - name: Build FlagCX
        run: |
          cd /__w/FlagCX/FlagCX
          make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1

      - name: Build perf tests
        run: |
          cd /__w/FlagCX/FlagCX/test/perf
          make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1

      - name: Perf tests (CUDA homogeneous 8-GPU)
        run: |
          cd /__w/FlagCX/FlagCX
          source .github/scripts/set_env/cuda.sh
          export PATH=$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$MPI_HOME/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reducescatter -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reduce -b 128M -e 1G -f 2 -r 0 -p 1

  perf-test-metax:
    name: perf-test (metax)
    runs-on: [self-hosted, mx-8g-cicd-flagcx]
    container:
      image: harbor.baai.ac.cn/flagos-dev/flagcx:3016346-metax-dev
      options: >-
        --hostname=te-cicd
        --ipc=host
        --privileged=true
        --shm-size=100gb
        --ulimit memlock=-1
        --user root
        --ulimit nofile=65535:65535
        --group-add video
        -e PLATFORM=metax
        -e TORCH_DISTRIBUTED_BACKEND=mccl
        -e FLAGCX_IB_GID_INDEX=0
        -e LD_LIBRARY_PATH=/opt/maca/lib:/usr/local/lib:$LD_LIBRARY_PATH
    env:
      MPI_HOME: /usr/local/mpi
      PERF_BIN: /__w/FlagCX/FlagCX/test/perf/host_api/build/bin

    steps:
      - name: Checkout Code
        uses: actions/checkout@v7
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
          ssh-strict: true
          ssh-user: git
          persist-credentials: true
          clean: true
          sparse-checkout-cone-mode: true
          fetch-tags: false
          show-progress: true
          lfs: false
          submodules: true
          set-safe-directory: true

      - name: Prepare MetaX environment
        run: |
          source /__w/FlagCX/FlagCX/.github/scripts/set_env/metax.sh
          echo "PATH=/opt/maca/mxgpu_llvm/bin:$MPI_HOME/bin:$PATH" >> "$GITHUB_ENV"
          echo "MPI_HOME=$MPI_HOME" >> "$GITHUB_ENV"
          echo "LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:/opt/maca/lib:/usr/local/lib:$MPI_HOME/lib:$LD_LIBRARY_PATH" >> "$GITHUB_ENV"
          mx-smi 2>/dev/null || true

      - name: Build FlagCX
        run: |
          cd /__w/FlagCX/FlagCX
          make -j$(nproc) USE_METAX=1

      - name: Build perf tests
        run: |
          cd /__w/FlagCX/FlagCX/test/perf
          make -j$(nproc) USE_METAX=1 MPI_HOME=/usr/local/mpi CCL_HOME=/opt/maca

      - name: Perf tests (MetaX homogeneous 8-chip)
        run: |
          cd /__w/FlagCX/FlagCX
          source .github/scripts/set_env/metax.sh
          export PATH=/opt/maca/mxgpu_llvm/bin:$MPI_HOME/bin:$PATH
          export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:/opt/maca/lib:/usr/local/lib:$MPI_HOME/lib:$LD_LIBRARY_PATH
          set -e
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reducescatter -b 128M -e 1G -f 2 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1
          mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reduce -b 128M -e 1G -f 2 -r 0 -p 1