[PAL] Add PPU support (#512) #244
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Perf Tests in Container | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| env: | |
| MPI_HOME: /usr/local/mpi | |
| PERF_BIN: /__w/FlagCX/FlagCX/test/perf/host_api/build/bin | |
| DEVICE_API_BIN: /__w/FlagCX/FlagCX/test/perf/device_api/build/bin | |
| jobs: | |
| perf-test: | |
| runs-on: [self-hosted, cx-build] | |
| container: | |
| image: localhost:5000/flagscale:cuda12.8.1-cudnn9.7.1-python3.12-torch2.7.0-time2507111538 | |
| options: --gpus all --privileged --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| ssh-strict: true | |
| ssh-user: git | |
| persist-credentials: true | |
| clean: true | |
| sparse-checkout-cone-mode: true | |
| fetch-tags: false | |
| show-progress: true | |
| lfs: false | |
| submodules: true | |
| set-safe-directory: true | |
| # FORCE_DEFAULT_PATH only affects kernel object compilation (comm traits); | |
| # host-side library and vendor API dispatch are unchanged. | |
| - name: Build FlagCX | |
| run: | | |
| cd /__w/FlagCX/FlagCX | |
| make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1 | |
| - name: Build perf tests | |
| run: | | |
| cd /__w/FlagCX/FlagCX/test/perf | |
| make -j$(nproc) USE_NVIDIA=1 COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1 | |
| - name: Wait for GPU | |
| shell: bash | |
| run: | | |
| cd /__w/FlagCX/FlagCX | |
| source test/script/_gpu_check.sh | |
| wait_for_gpu | |
| - name: "Perf tests (homoRunner)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reducescatter -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_reduce -b 128M -e 1G -f 2 -r 0 -p 1 | |
| - name: "Perf tests (uniRunner)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoallv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_allgather -b 128M -e 1G -f 2 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_broadcast -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_gather -b 128M -e 1G -f 2 -r 0 -p 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_scatter -b 128M -e 1G -f 2 -r 0 -p 1 | |
| - name: "Registration -R 1 (homoRunner)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| mpirun -np 8 --allow-run-as-root $PERF_BIN/perf_allreduce -b 128M -e 1G -f 2 -p 1 -R 1 | |
| - name: "Registration -R 1 (uniRunner P2P)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 -R 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 -R 1 | |
| - name: "Registration -R 1 (uniRunner NET)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_P2P_DISABLE=1 $PERF_BIN/perf_sendrecv -b 128M -e 1G -f 2 -p 1 -R 1 | |
| mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_P2P_DISABLE=1 $PERF_BIN/perf_alltoall -b 128M -e 1G -f 2 -p 1 -R 1 | |
| - name: "RMA one-sided (put/get)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| set -e | |
| mpirun -np 2 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_put -b 8 -e 1M -f 2 -R 2 | |
| mpirun -np 2 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_USE_HETERO_COMM=1 $PERF_BIN/perf_get -b 8 -e 1M -f 2 -R 2 | |
| - name: "Device API perf (intra-node)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| mpirun -np 8 --allow-run-as-root \ | |
| -x FLAGCX_USE_HETERO_COMM=1 \ | |
| -x FLAGCX_MEM_ENABLE=1 \ | |
| -x FLAGCX_VMM_ENABLE=0 \ | |
| -x FLAGCX_P2P_DISABLE=1 \ | |
| -x FLAGCX_DEBUG=TRACE -x FLAGCX_DEBUG_SUBSYS=ALL \ | |
| -x LD_LIBRARY_PATH \ | |
| $DEVICE_API_BIN/perf_allreduce_intranode -b 1M -e 64M -f 2 -R 1 | |
| # - name: "Device API perf (inter-node two-sided)" | |
| # run: | | |
| # export PATH=$MPI_HOME/bin:$PATH | |
| # export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| # mpirun -np 8 --allow-run-as-root \ | |
| # $DEVICE_API_BIN/perf_internode_twosided -b 1M -e 64M -f 2 -R 1 | |
| # - name: "Device API perf (inter-node one-sided)" | |
| # run: | | |
| # export PATH=$MPI_HOME/bin:$PATH | |
| # export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| # mpirun -np 8 --allow-run-as-root \ | |
| # $DEVICE_API_BIN/perf_internode_onesided -b 1M -e 64M -f 2 -R 2 | |
| - name: "P2P Engine perf (one-sided read/write)" | |
| run: | | |
| export PATH=$MPI_HOME/bin:$PATH | |
| export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH | |
| mpirun -np 2 --allow-run-as-root $PERF_BIN/perf_p2p_engine -b 4K -e 64M -f 2 -n 10 |