Skip to content

Commit 83d9728

Browse files
authored
[CICD] Refactor and stabilize the Device API / Device IR MPI-based unit tests (#529)
1 parent 27e71b0 commit 83d9728

35 files changed

Lines changed: 4273 additions & 1968 deletions

.github/workflows/test.yml

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -128,19 +128,16 @@ jobs:
128128
-x LD_LIBRARY_PATH \
129129
$DEVICE_API_BIN/perf_allreduce_intranode -b 1M -e 64M -f 2 -R 1
130130
131-
# - name: "Device API perf (inter-node two-sided)"
132-
# run: |
133-
# export PATH=$MPI_HOME/bin:$PATH
134-
# export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
135-
# mpirun -np 8 --allow-run-as-root \
136-
# $DEVICE_API_BIN/perf_internode_twosided -b 1M -e 64M -f 2 -R 1
137-
138-
# - name: "Device API perf (inter-node one-sided)"
139-
# run: |
140-
# export PATH=$MPI_HOME/bin:$PATH
141-
# export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
142-
# mpirun -np 8 --allow-run-as-root \
143-
# $DEVICE_API_BIN/perf_internode_onesided -b 1M -e 64M -f 2 -R 2
131+
- name: "Device API perf (inter-node one-sided)"
132+
run: |
133+
export PATH=$MPI_HOME/bin:$PATH
134+
export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
135+
COMMON_ENV="-x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_P2P_DISABLE=1 -x LD_LIBRARY_PATH"
136+
NODE1_FLAG="-x CUDA_VISIBLE_DEVICES=0,1,2,3 -x FLAGCX_HOSTID=node0 -x FLAGCX_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3"
137+
NODE2_FLAG="-x CUDA_VISIBLE_DEVICES=4,5,6,7 -x FLAGCX_HOSTID=node1 -x FLAGCX_IB_HCA=mlx5_4,mlx5_5,mlx5_6,mlx5_7"
138+
mpirun --allow-run-as-root \
139+
-np 4 ${COMMON_ENV} ${NODE1_FLAG} $DEVICE_API_BIN/perf_internode_onesided -b 1M -e 64M -f 2 -R 1 \
140+
: -np 4 ${COMMON_ENV} ${NODE2_FLAG} $DEVICE_API_BIN/perf_internode_onesided -b 1M -e 64M -f 2 -R 1
144141
145142
- name: "P2P Engine perf (one-sided read/write)"
146143
run: |

.github/workflows/unit-test.yml

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -108,30 +108,3 @@ jobs:
108108
# export PATH=$MPI_HOME/bin:$PATH
109109
# mpirun -np 8 --allow-run-as-root -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_P2P_DISABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_DEBUG=TRACE -x FLAGCX_DEBUG_SUBSYS=ALL ./build/bin/kernel_mpi_tests
110110

111-
- name: Run intra-node device_api tests with mpirun
112-
run: |
113-
cd /__w/FlagCX/FlagCX/test/unittest/device_api
114-
export MPI_HOME=/usr/local/mpi
115-
export PATH=$MPI_HOME/bin:$PATH
116-
export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
117-
COMMON_ENV="-x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_P2P_DISABLE=1 -x LD_LIBRARY_PATH"
118-
FLAGS="-b 1M -e 4M -f 2 -R 2"
119-
mpirun -np 8 --allow-run-as-root ${COMMON_ENV} build/bin/test_device_api ${FLAGS}
120-
mpirun -np 8 --allow-run-as-root ${COMMON_ENV} build/bin/test_device_ir_intra ${FLAGS}
121-
122-
- name: Run inter-node device_api tests with mpirun (simulate 2 nodes)
123-
run: |
124-
cd /__w/FlagCX/FlagCX/test/unittest/device_api
125-
export MPI_HOME=/usr/local/mpi
126-
export PATH=$MPI_HOME/bin:$PATH
127-
export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
128-
COMMON_ENV="-x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_P2P_DISABLE=1 -x LD_LIBRARY_PATH"
129-
FLAGS="-b 1M -e 4M -f 2 -R 2"
130-
NODE1_FLAG="-x CUDA_VISIBLE_DEVICES=0,1,2,3 -x FLAGCX_HOSTID=node0 -x FLAGCX_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3"
131-
NODE2_FLAG="-x CUDA_VISIBLE_DEVICES=4,5,6,7 -x FLAGCX_HOSTID=node1 -x FLAGCX_IB_HCA=mlx5_4,mlx5_5,mlx5_6,mlx5_7"
132-
mpirun --allow-run-as-root \
133-
-np 4 ${COMMON_ENV} ${NODE1_FLAG} build/bin/test_device_api ${FLAGS} \
134-
: -np 4 ${COMMON_ENV} ${NODE2_FLAG} build/bin/test_device_api ${FLAGS}
135-
mpirun --allow-run-as-root \
136-
-np 4 ${COMMON_ENV} ${NODE1_FLAG} build/bin/test_device_ir_inter ${FLAGS} \
137-
: -np 4 ${COMMON_ENV} ${NODE2_FLAG} build/bin/test_device_ir_inter ${FLAGS}

.github/workflows/unittest-device-api.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@ jobs:
5252
export PATH=$MPI_HOME/bin:$PATH
5353
export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
5454
COMMON_ENV="-x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_P2P_DISABLE=1 -x LD_LIBRARY_PATH"
55-
FLAGS="-b 1M -e 4M -f 2 -R 2"
56-
mpirun -np 8 --allow-run-as-root ${COMMON_ENV} build/bin/test_device_api ${FLAGS}
55+
FLAGS="-b 1M -e 4M -f 2 -R 1"
56+
mpirun -np 8 --allow-run-as-root ${COMMON_ENV} build/bin/test_device_api_intra ${FLAGS}
5757
mpirun -np 8 --allow-run-as-root ${COMMON_ENV} build/bin/test_device_ir_intra ${FLAGS}
5858
5959
- name: Run inter-node tests (simulate 2 nodes, 4 GPUs each)
@@ -63,12 +63,12 @@ jobs:
6363
export PATH=$MPI_HOME/bin:$PATH
6464
export LD_LIBRARY_PATH=/__w/FlagCX/FlagCX/build/lib:$LD_LIBRARY_PATH
6565
COMMON_ENV="-x FLAGCX_USE_HETERO_COMM=1 -x FLAGCX_MEM_ENABLE=1 -x FLAGCX_VMM_ENABLE=0 -x FLAGCX_P2P_DISABLE=1 -x LD_LIBRARY_PATH"
66-
FLAGS="-b 1M -e 4M -f 2 -R 2"
66+
FLAGS="-b 1M -e 4M -f 2 -R 1"
6767
NODE1_FLAG="-x CUDA_VISIBLE_DEVICES=0,1,2,3 -x FLAGCX_HOSTID=node0 -x FLAGCX_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3"
6868
NODE2_FLAG="-x CUDA_VISIBLE_DEVICES=4,5,6,7 -x FLAGCX_HOSTID=node1 -x FLAGCX_IB_HCA=mlx5_4,mlx5_5,mlx5_6,mlx5_7"
6969
mpirun --allow-run-as-root \
70-
-np 4 ${COMMON_ENV} ${NODE1_FLAG} build/bin/test_device_api ${FLAGS} \
71-
: -np 4 ${COMMON_ENV} ${NODE2_FLAG} build/bin/test_device_api ${FLAGS}
70+
-np 4 ${COMMON_ENV} ${NODE1_FLAG} build/bin/test_device_api_inter ${FLAGS} \
71+
: -np 4 ${COMMON_ENV} ${NODE2_FLAG} build/bin/test_device_api_inter ${FLAGS}
7272
mpirun --allow-run-as-root \
7373
-np 4 ${COMMON_ENV} ${NODE1_FLAG} build/bin/test_device_ir_inter ${FLAGS} \
7474
: -np 4 ${COMMON_ENV} ${NODE2_FLAG} build/bin/test_device_ir_inter ${FLAGS}

bindings/ir/flagcx_device_scalar_ir_impl.h

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,41 @@ flagcxMakeTeamFromKind(const flagcxDevComm &comm, flagcxTeamKind_t kind) {
7373
}
7474
}
7575

76+
/* ================================================================
77+
* Internal helper: grid-wide barrier (sense-reversing)
78+
*
79+
* Synchronizes all blocks on the same GPU. Uses a monotonic
80+
* sense-flip protocol so it's safely reusable across iterations.
81+
* gridSyncState[0] = arrive counter, gridSyncState[1] = sense.
82+
* ================================================================ */
83+
84+
static FLAGCX_DEVICE_INLINE_DECORATOR void
85+
flagcxGridSync(unsigned int *gridSyncState) {
86+
__syncthreads();
87+
if (threadIdx.x == 0) {
88+
unsigned int curSense = *(volatile unsigned int *)&gridSyncState[1];
89+
unsigned int arrived = atomicAdd(&gridSyncState[0], 1u) + 1;
90+
if (arrived == gridDim.x) {
91+
// Last block: reset counter, flip sense to release others
92+
atomicExch(&gridSyncState[0], 0u);
93+
__threadfence();
94+
atomicExch(&gridSyncState[1], 1u - curSense);
95+
} else {
96+
// Spin until sense flips (all blocks arrived)
97+
#ifndef NDEBUG
98+
unsigned int __spins = 0;
99+
#endif
100+
while (*(volatile unsigned int *)&gridSyncState[1] == curSense) {
101+
#ifndef NDEBUG
102+
if (++__spins >= 100000000u)
103+
__trap(); // grid barrier timeout — likely a hang
104+
#endif
105+
}
106+
}
107+
}
108+
__syncthreads();
109+
}
110+
76111
/* ================================================================
77112
* Category 2: Scalar Cooperative Group (6)
78113
* ================================================================ */
@@ -231,6 +266,9 @@ flagcxIntraBarrierSyncS(const void *commOpaque, flagcxCoopKind_t coopKind,
231266
flagcxDevBarrier<flagcxTeamTagIntra, flagcxCoopAny> bar(coop, *comm, team,
232267
index, multimem);
233268
bar.sync(order);
269+
if (comm->_gridBarrierState) {
270+
flagcxGridSync(comm->_gridBarrierState);
271+
}
234272
}
235273

236274
/* ================================================================
@@ -274,6 +312,9 @@ flagcxInterBarrierSyncS(const void *netOpaque, flagcxCoopKind_t coopKind,
274312
flagcxDevBarrier<flagcxTeamTagInter, flagcxCoopAny> bar(coop, *net, team,
275313
index);
276314
bar.sync(order, fence);
315+
if (net->_gridBarrierState) {
316+
flagcxGridSync(net->_gridBarrierState);
317+
}
277318
}
278319

279320
/* ================================================================
@@ -317,6 +358,9 @@ flagcxWorldBarrierSyncS(const void *netOpaque, flagcxCoopKind_t coopKind,
317358
flagcxDevBarrier<flagcxTeamTagWorld, flagcxCoopAny> bar(
318359
coop, flagcxTeamTagWorld{}, *net, index, multimem);
319360
bar.sync(order, fence);
361+
if (net->_gridBarrierState) {
362+
flagcxGridSync(net->_gridBarrierState);
363+
}
320364
}
321365

322366
/* ================================================================

0 commit comments

Comments
 (0)