[CICD] Add Hygon DCU adaptation and CI workflow #2341
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Perf Tests in Container | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| env: | |
| MPI_HOME: /usr/local/mpi | |
| PERF_BIN: /__w/FlagCX/FlagCX/test/perf/host_api/build/bin | |
| DEVICE_API_BIN: /__w/FlagCX/FlagCX/test/perf/device_api/build/bin | |
| jobs: | |
| perf-test: | |
| runs-on: [self-hosted, cx-build] | |
| container: | |
| image: localhost:5000/flagscale:cuda12.8.1-cudnn9.7.1-python3.12-torch2.7.0-time2507111538 | |
| options: --gpus all --privileged --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| ssh-strict: true | |
| ssh-user: git | |
| persist-credentials: true | |
| clean: true | |
| sparse-checkout-cone-mode: true | |
| fetch-tags: false | |
| show-progress: true | |
| lfs: false | |
| submodules: true | |
| set-safe-directory: true | |
| # FORCE_DEFAULT_PATH only affects kernel object compilation (comm traits); | |
| # host-side library and vendor API dispatch are unchanged. | |
| - name: Build FlagCX | |
| run: | | |
| cd /__w/FlagCX/FlagCX | |
| make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1 | |
| - name: Build perf tests | |
| run: | | |
| cd /__w/FlagCX/FlagCX/test/perf | |
| make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1 | |
| - name: Wait for GPU | |
| shell: bash | |
| run: | | |
| cd /__w/FlagCX/FlagCX | |
| source test/script/_gpu_check.sh | |
| wait_for_gpu | |
| - name: "Perf tests (homoRunner)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reducescatter -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reduce -b 128M -e 1G -f 2 -r 0 -p 1 | |
| - name: "Perf tests (uniRunner)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1 | |
| - name: "Registration -R 1 (homoRunner)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1 -R 1 | |
| - name: "Registration -R 1 (uniRunner P2P)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 -R 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 -R 1 | |
| - name: "Registration -R 1 (uniRunner NET)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_P2P_DISABLE=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 -R 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_P2P_DISABLE=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 -R 1 | |
| - name: "RMA one-sided (put/get)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 2 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_put -b 8 -e 1M -f 2 -R 2 | |
| mpirun -np 2 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_get -b 8 -e 1M -f 2 -R 2 | |
| - name: "Device API perf (intra-node)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| mpirun -np 8 --allow-run-as-root \ | |
| -x FLAGCX_USE_HETERO_COMM=1 \ | |
| -x FLAGCX_MEM_ENABLE=1 \ | |
| -x FLAGCX_VMM_ENABLE=0 \ | |
| -x FLAGCX_P2P_DISABLE=1 \ | |
| -x FLAGCX_DEBUG=TRACE -x FLAGCX_DEBUG_SUBSYS=ALL \ | |
| -x LD_LIBRARY_PATH \ | |
| $DEVICE_API_BIN/perf_allreduce_intranode -b 1M -e 64M -f 2 -R 1 | |
| - name: "Device API perf (inter-node one-sided)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| COMMON_ENV="-x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_P2P_DISABLE=1 -x LD_LIBRARY_PATH" | |
| NODE1_FLAG="-x CUDA_VISIBLE_DEVICES=0,1,2,3 -x FLAGCX_HOSTID=node0 -x FLAGCX_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3" | |
| NODE2_FLAG="-x CUDA_VISIBLE_DEVICES=4,5,6,7 -x FLAGCX_HOSTID=node1 -x FLAGCX_IB_HCA=mlx5_4,mlx5_5,mlx5_6,mlx5_7" | |
| mpirun --allow-run-as-root \ | |
| -np 4 ${COMMON_ENV} ${NODE1_FLAG} $DEVICE_API_BIN/perf_internode_onesided -b 1M -e 64M -f 2 -R 1 \ | |
| : -np 4 ${COMMON_ENV} ${NODE2_FLAG} $DEVICE_API_BIN/perf_internode_onesided -b 1M -e 64M -f 2 -R 1 | |
| - name: "P2P Engine perf (one-sided read/write)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| mpirun -np 2 --allow-run-as-root $PERF_BIN/perf_p2p_engine -b 4K -e 64M -f 2 -n 10 | |
| perf-test-hygon: | |
| name: perf-test (hygon) | |
| runs-on: [self-hosted, hg-8g-cicd-flagcx] | |
| container: | |
| image: harbor.baai.ac.cn/flagos-dev/flagcx:b079886-hygon-dev | |
| volumes: | |
| - /usr/local/hyhal:/opt/hyhal:ro | |
| options: >- | |
| --ipc=host | |
| --privileged=true | |
| --shm-size=100gb | |
| --ulimit memlock=-1 | |
| --ulimit stack=67108864 | |
| --device=/dev/kfd | |
| --device=/dev/dri | |
| --group-add video | |
| env: | |
| MPI_HOME: /opt/mpi | |
| PERF_BIN: /__w/FlagCX/FlagCX/test/perf/host_api/build/bin | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| ssh-strict: true | |
| ssh-user: git | |
| persist-credentials: true | |
| clean: true | |
| sparse-checkout-cone-mode: true | |
| fetch-tags: false | |
| show-progress: true | |
| lfs: false | |
| submodules: true | |
| set-safe-directory: true | |
| - name: Prepare Hygon environment | |
| run: | | |
| source /opt/dtk/env.sh | |
| export CUDA_PATH=/opt/dtk/cuda/cuda-12 | |
| export CUDA_HOME=$CUDA_PATH | |
| echo "CUDA_PATH=$CUDA_PATH" >> "$GITHUB_ENV" | |
| echo "CUDA_HOME=$CUDA_HOME" >> "$GITHUB_ENV" | |
| echo "PATH=/opt/mpi/bin:$PATH" >> "$GITHUB_ENV" | |
| echo "LD_LIBRARY_PATH=/opt/mpi/lib:/opt/dtk/lib:$CUDA_PATH/lib64:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" | |
| hy-smi --showproductname | |
| python3 -c 'import torch; print("torch", torch.__version__, "devices", torch.cuda.device_count()); assert torch.cuda.device_count() >= 8' | |
| - name: Build FlagCX | |
| run: | | |
| cd /__w/FlagCX/FlagCX | |
| source /opt/dtk/env.sh | |
| make -j$(nproc) USE_DU=1 PLATFORM_EXTRA_SRCS=flagcx/adaptor/device_api/default_dev_api_backend.cc | |
| - name: Build perf tests | |
| run: | | |
| cd /__w/FlagCX/FlagCX/test/perf | |
| source /opt/dtk/env.sh | |
| make -j$(nproc) USE_DU=1 MPI_HOME=/opt/mpi CCL_HOME=/opt/dtk/cuda/cuda-12 | |
| # Hygon currently supports the homogeneous single-node path here. | |
| # Heterogeneous, RMA, and device-API paths are covered only where the | |
| # backend advertises support; unsupported cases self-skip in the tests. | |
| - name: Perf tests (Hygon homogeneous 8-chip) | |
| run: | | |
| cd /__w/FlagCX/FlagCX | |
| source /opt/dtk/env.sh | |
| export PATH=/opt/mpi/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:/opt/mpi/lib:/opt/dtk/lib:/opt/dtk/cuda/cuda-12/lib64:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reducescatter -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reduce -b 128M -e 1G -f 2 -r 0 -p 1 | |
| perf-test-cuda: | |
| name: perf-test (cuda) | |
| runs-on: [self-hosted, nv-8g-cicd-flagcx] | |
| container: | |
| image: harbor.baai.ac.cn/flagos-dev/flagcx:55c5c6f-cuda-dev | |
| options: >- | |
| --gpus all | |
| --privileged | |
| --ipc=host | |
| --ulimit memlock=-1 | |
| --ulimit stack=67108864 | |
| env: | |
| MPI_HOME: /usr/local/mpi | |
| PERF_BIN: /__w/FlagCX/FlagCX/test/perf/host_api/build/bin | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| ssh-strict: true | |
| ssh-user: git | |
| persist-credentials: true | |
| clean: true | |
| sparse-checkout-cone-mode: true | |
| fetch-tags: false | |
| show-progress: true | |
| lfs: false | |
| submodules: true | |
| set-safe-directory: true | |
| - name: Prepare CUDA environment | |
| run: | | |
| source /__w/FlagCX/FlagCX/.github/scripts/set_env/cuda.sh | |
| echo "PATH=$MPI_HOME/bin:$PATH" >> "$GITHUB_ENV" | |
| echo "MPI_HOME=$MPI_HOME" >> "$GITHUB_ENV" | |
| echo "LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$MPI_HOME/lib:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" | |
| nvidia-smi | |
| - name: Build FlagCX | |
| run: | | |
| cd /__w/FlagCX/FlagCX | |
| make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1 | |
| - name: Build perf tests | |
| run: | | |
| cd /__w/FlagCX/FlagCX/test/perf | |
| make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1 | |
| - name: Perf tests (CUDA homogeneous 8-GPU) | |
| run: | | |
| cd /__w/FlagCX/FlagCX | |
| source .github/scripts/set_env/cuda.sh | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$MPI_HOME/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reducescatter -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reduce -b 128M -e 1G -f 2 -r 0 -p 1 | |
| perf-test-metax: | |
| name: perf-test (metax) | |
| runs-on: [self-hosted, mx-8g-cicd-flagcx] | |
| container: | |
| image: harbor.baai.ac.cn/flagos-dev/flagcx:3016346-metax-dev | |
| options: >- | |
| --hostname=te-cicd | |
| --ipc=host | |
| --privileged=true | |
| --shm-size=100gb | |
| --ulimit memlock=-1 | |
| --user root | |
| --ulimit nofile=65535:65535 | |
| --group-add video | |
| -e PLATFORM=metax | |
| -e TORCH_DISTRIBUTED_BACKEND=mccl | |
| -e FLAGCX_IB_GID_INDEX=0 | |
| -e LD_LIBRARY_PATH=/opt/maca/lib:/usr/local/lib:$LD_LIBRARY_PATH | |
| env: | |
| MPI_HOME: /usr/local/mpi | |
| PERF_BIN: /__w/FlagCX/FlagCX/test/perf/host_api/build/bin | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| ssh-strict: true | |
| ssh-user: git | |
| persist-credentials: true | |
| clean: true | |
| sparse-checkout-cone-mode: true | |
| fetch-tags: false | |
| show-progress: true | |
| lfs: false | |
| submodules: true | |
| set-safe-directory: true | |
| - name: Prepare MetaX environment | |
| run: | | |
| source /__w/FlagCX/FlagCX/.github/scripts/set_env/metax.sh | |
| echo "PATH=/opt/maca/mxgpu_llvm/bin:$MPI_HOME/bin:$PATH" >> "$GITHUB_ENV" | |
| echo "MPI_HOME=$MPI_HOME" >> "$GITHUB_ENV" | |
| echo "LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:/opt/maca/lib:/usr/local/lib:$MPI_HOME/lib:$LD_LIBRARY_PATH" >> "$GITHUB_ENV" | |
| mx-smi 2>/dev/null || true | |
| - name: Build FlagCX | |
| run: | | |
| cd /__w/FlagCX/FlagCX | |
| make -j$(nproc) USE_METAX=1 | |
| - name: Build perf tests | |
| run: | | |
| cd /__w/FlagCX/FlagCX/test/perf | |
| make -j$(nproc) USE_METAX=1 MPI_HOME=/usr/local/mpi CCL_HOME=/opt/maca | |
| - name: Perf tests (MetaX homogeneous 8-chip) | |
| run: | | |
| cd /__w/FlagCX/FlagCX | |
| source .github/scripts/set_env/metax.sh | |
| export PATH=/opt/maca/mxgpu_llvm/bin:$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:/opt/maca/lib:/usr/local/lib:$MPI_HOME/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reducescatter -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reduce -b 128M -e 1G -f 2 -r 0 -p 1 |