Skip to content

Commit 6fffaa2

Browse files
MC952-archMC952-arch
authored
Add a '-m [DEC/HEX/BIN]' argument to test/perf for specifying the MPI communication split mode (#252)
1 parent f511001 commit 6fffaa2

14 files changed

Lines changed: 192 additions & 160 deletions

‎test/perf/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,4 +135,4 @@ print_var:
135135
@echo "MPI_INCLUDE: $(MPI_INCLUDE)"
136136
@echo "MPI_LIB: $(MPI_LIB)"
137137
@echo "COMPILER: $(COMPILER)"
138-
@echo "EXTRA_COMPILER_FLAG: $(EXTRA_COMPILER_FLAG)"
138+
@echo "EXTRA_COMPILER_FLAG: $(EXTRA_COMPILER_FLAG)"

‎test/perf/include/tools.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
1+
#include "mpi.h"
12
#include <cstddef>
23
#include <cstdint>
34

5+
void initMpiEnv(int argc, char **argv, int &worldRank, int &worldSize,
6+
int &proc, int &totalProcs, int &color, MPI_Comm &splitComm,
7+
uint64_t splitMask);
8+
49
class timer {
510
public:
611
timer();
@@ -20,6 +25,7 @@ class parser {
2025
int getTestIters() const { return testIters; }
2126
bool isPrintBuffer() const { return printBuffer == 1; }
2227
int getRootRank() const { return root; }
28+
uint64_t getSplitMask() const { return splitMask; }
2329

2430
size_t minBytes;
2531
size_t maxBytes;
@@ -28,4 +34,5 @@ class parser {
2834
int testIters;
2935
int printBuffer;
3036
int root;
31-
};
37+
uint64_t splitMask;
38+
};

‎test/perf/test_allgather.cpp‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#include "flagcx.h"
2-
#include "mpi.h"
32
#include "tools.h"
43
#include <cstring>
54
#include <iostream>
@@ -14,27 +13,28 @@ int main(int argc, char *argv[]) {
1413
int num_warmup_iters = args.getWarmupIters();
1514
int num_iters = args.getTestIters();
1615
int print_buffer = args.isPrintBuffer();
17-
18-
int totalProcs, proc;
19-
MPI_Init(&argc, &argv);
20-
MPI_Comm_size(MPI_COMM_WORLD, &totalProcs);
21-
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
22-
printf("I am %d of %d\n", proc, totalProcs);
16+
uint64_t split_mask = args.getSplitMask();
2317

2418
flagcxHandlerGroup_t handler;
2519
flagcxHandleInit(&handler);
2620
flagcxUniqueId_t &uniqueId = handler->uniqueId;
2721
flagcxComm_t &comm = handler->comm;
2822
flagcxDeviceHandle_t &devHandle = handler->devHandle;
2923

24+
int color = 0;
25+
int worldSize = 1, worldRank = 0;
26+
int totalProcs = 1, proc = 0;
27+
MPI_Comm splitComm;
28+
initMpiEnv(argc, argv, worldRank, worldSize, proc, totalProcs, color,
29+
splitComm, split_mask);
30+
3031
int nGpu;
3132
devHandle->getDeviceCount(&nGpu);
32-
devHandle->setDevice(proc % nGpu);
33+
devHandle->setDevice(worldRank % nGpu);
3334

3435
if (proc == 0)
3536
flagcxGetUniqueId(&uniqueId);
36-
MPI_Bcast((void *)uniqueId, sizeof(flagcxUniqueId), MPI_BYTE, 0,
37-
MPI_COMM_WORLD);
37+
MPI_Bcast((void *)uniqueId, sizeof(flagcxUniqueId), MPI_BYTE, 0, splitComm);
3838
MPI_Barrier(MPI_COMM_WORLD);
3939

4040
flagcxCommInitRank(&comm, totalProcs, uniqueId, proc);
@@ -76,7 +76,7 @@ int main(int argc, char *argv[]) {
7676
devHandle->deviceMemcpy(sendbuff, hello, size / totalProcs,
7777
flagcxMemcpyHostToDevice, NULL);
7878

79-
if ((proc == 0 || proc == totalProcs - 1) && print_buffer) {
79+
if ((proc == 0 || proc == totalProcs - 1) && color == 0 && print_buffer) {
8080
printf("sendbuff = ");
8181
printf("%f\n", ((float *)hello)[0]);
8282
}
@@ -93,13 +93,13 @@ int main(int argc, char *argv[]) {
9393
double elapsed_time = tim.elapsed() / num_iters;
9494
MPI_Allreduce(MPI_IN_PLACE, (void *)&elapsed_time, 1, MPI_DOUBLE, MPI_SUM,
9595
MPI_COMM_WORLD);
96-
elapsed_time /= totalProcs;
96+
elapsed_time /= worldSize;
9797

9898
double base_bw = (double)(size) / 1.0E9 / elapsed_time;
9999
double alg_bw = base_bw;
100100
double factor = ((double)(totalProcs - 1)) / ((double)totalProcs);
101101
double bus_bw = base_bw * factor;
102-
if (proc == 0) {
102+
if (proc == 0 && color == 0) {
103103
printf("Comm size: %zu bytes; Elapsed time: %lf sec; Algo bandwidth: %lf "
104104
"GB/s; Bus bandwidth: %lf GB/s\n",
105105
size, elapsed_time, alg_bw, bus_bw);
@@ -110,7 +110,7 @@ int main(int argc, char *argv[]) {
110110
devHandle->deviceMemset(hello, 0, size, flagcxMemHost, NULL);
111111
devHandle->deviceMemcpy(hello, recvbuff, size, flagcxMemcpyDeviceToHost,
112112
NULL);
113-
if ((proc == 0 || proc == totalProcs - 1) && print_buffer) {
113+
if ((proc == 0 || proc == totalProcs - 1) && color == 0 && print_buffer) {
114114
printf("recvbuff = ");
115115
for (int i = 0; i < totalProcs; i++) {
116116
printf("%f ", ((float *)hello)[i * (count / totalProcs)]);

‎test/perf/test_allreduce.cpp‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#include "flagcx.h"
2-
#include "mpi.h"
32
#include "tools.h"
43
#include <cstring>
54
#include <iostream>
@@ -14,27 +13,28 @@ int main(int argc, char *argv[]) {
1413
int num_warmup_iters = args.getWarmupIters();
1514
int num_iters = args.getTestIters();
1615
int print_buffer = args.isPrintBuffer();
17-
18-
int totalProcs, proc;
19-
MPI_Init(&argc, &argv);
20-
MPI_Comm_size(MPI_COMM_WORLD, &totalProcs);
21-
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
22-
printf("I am %d of %d\n", proc, totalProcs);
16+
uint64_t split_mask = args.getSplitMask();
2317

2418
flagcxHandlerGroup_t handler;
2519
flagcxHandleInit(&handler);
2620
flagcxUniqueId_t &uniqueId = handler->uniqueId;
2721
flagcxComm_t &comm = handler->comm;
2822
flagcxDeviceHandle_t &devHandle = handler->devHandle;
2923

24+
int color = 0;
25+
int worldSize = 1, worldRank = 0;
26+
int totalProcs = 1, proc = 0;
27+
MPI_Comm splitComm;
28+
initMpiEnv(argc, argv, worldRank, worldSize, proc, totalProcs, color,
29+
splitComm, split_mask);
30+
3031
int nGpu;
3132
devHandle->getDeviceCount(&nGpu);
32-
devHandle->setDevice(proc % nGpu);
33+
devHandle->setDevice(worldRank % nGpu);
3334

3435
if (proc == 0)
3536
flagcxGetUniqueId(&uniqueId);
36-
MPI_Bcast((void *)uniqueId, sizeof(flagcxUniqueId), MPI_BYTE, 0,
37-
MPI_COMM_WORLD);
37+
MPI_Bcast((void *)uniqueId, sizeof(flagcxUniqueId), MPI_BYTE, 0, splitComm);
3838
MPI_Barrier(MPI_COMM_WORLD);
3939

4040
flagcxCommInitRank(&comm, totalProcs, uniqueId, proc);
@@ -75,7 +75,7 @@ int main(int argc, char *argv[]) {
7575
devHandle->deviceMemcpy(sendbuff, hello, size, flagcxMemcpyHostToDevice,
7676
NULL);
7777

78-
if (proc == 0 && print_buffer) {
78+
if (proc == 0 && color == 0 && print_buffer) {
7979
printf("sendbuff = ");
8080
for (size_t i = 0; i < 10; i++) {
8181
printf("%f ", ((float *)hello)[i]);
@@ -95,13 +95,13 @@ int main(int argc, char *argv[]) {
9595
double elapsed_time = tim.elapsed() / num_iters;
9696
MPI_Allreduce(MPI_IN_PLACE, (void *)&elapsed_time, 1, MPI_DOUBLE, MPI_SUM,
9797
MPI_COMM_WORLD);
98-
elapsed_time /= totalProcs;
98+
elapsed_time /= worldSize;
9999

100100
double base_bw = (double)(size) / 1.0E9 / elapsed_time;
101101
double alg_bw = base_bw;
102102
double factor = ((double)(2 * (totalProcs - 1))) / ((double)(totalProcs));
103103
double bus_bw = base_bw * factor;
104-
if (proc == 0) {
104+
if (proc == 0 && color == 0) {
105105
printf("Comm size: %zu bytes; Elapsed time: %lf sec; Algo bandwidth: %lf "
106106
"GB/s; Bus bandwidth: %lf GB/s\n",
107107
size, elapsed_time, alg_bw, bus_bw);
@@ -112,7 +112,7 @@ int main(int argc, char *argv[]) {
112112
devHandle->deviceMemset(hello, 0, size, flagcxMemHost, NULL);
113113
devHandle->deviceMemcpy(hello, recvbuff, size, flagcxMemcpyDeviceToHost,
114114
NULL);
115-
if (proc == 0 && print_buffer) {
115+
if (proc == 0 && color == 0 && print_buffer) {
116116
printf("recvbuff = ");
117117
for (size_t i = 0; i < 10; i++) {
118118
printf("%f ", ((float *)hello)[i]);

‎test/perf/test_alltoall.cpp‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#include "flagcx.h"
2-
#include "mpi.h"
32
#include "tools.h"
43
#include <cstring>
54
#include <iostream>
@@ -14,27 +13,28 @@ int main(int argc, char *argv[]) {
1413
int num_warmup_iters = args.getWarmupIters();
1514
int num_iters = args.getTestIters();
1615
int print_buffer = args.isPrintBuffer();
17-
18-
int totalProcs, proc;
19-
MPI_Init(&argc, &argv);
20-
MPI_Comm_size(MPI_COMM_WORLD, &totalProcs);
21-
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
22-
printf("I am %d of %d\n", proc, totalProcs);
16+
uint64_t split_mask = args.getSplitMask();
2317

2418
flagcxHandlerGroup_t handler;
2519
flagcxHandleInit(&handler);
2620
flagcxUniqueId_t &uniqueId = handler->uniqueId;
2721
flagcxComm_t &comm = handler->comm;
2822
flagcxDeviceHandle_t &devHandle = handler->devHandle;
2923

24+
int color = 0;
25+
int worldSize = 1, worldRank = 0;
26+
int totalProcs = 1, proc = 0;
27+
MPI_Comm splitComm;
28+
initMpiEnv(argc, argv, worldRank, worldSize, proc, totalProcs, color,
29+
splitComm, split_mask);
30+
3031
int nGpu;
3132
devHandle->getDeviceCount(&nGpu);
32-
devHandle->setDevice(proc % nGpu);
33+
devHandle->setDevice(worldRank % nGpu);
3334

3435
if (proc == 0)
3536
flagcxGetUniqueId(&uniqueId);
36-
MPI_Bcast((void *)uniqueId, sizeof(flagcxUniqueId), MPI_BYTE, 0,
37-
MPI_COMM_WORLD);
37+
MPI_Bcast((void *)uniqueId, sizeof(flagcxUniqueId), MPI_BYTE, 0, splitComm);
3838
MPI_Barrier(MPI_COMM_WORLD);
3939

4040
flagcxCommInitRank(&comm, totalProcs, uniqueId, proc);
@@ -75,7 +75,7 @@ int main(int argc, char *argv[]) {
7575
devHandle->deviceMemcpy(sendbuff, hello, size, flagcxMemcpyHostToDevice,
7676
NULL);
7777

78-
if ((proc == 0 || proc == totalProcs - 1) && print_buffer) {
78+
if ((proc == 0 || proc == totalProcs - 1) && color == 0 && print_buffer) {
7979
printf("sendbuff = ");
8080
for (int i = 0; i < totalProcs; i++) {
8181
printf("%f ", ((float *)hello)[i * (count / totalProcs)]);
@@ -95,13 +95,13 @@ int main(int argc, char *argv[]) {
9595
double elapsed_time = tim.elapsed() / num_iters;
9696
MPI_Allreduce(MPI_IN_PLACE, (void *)&elapsed_time, 1, MPI_DOUBLE, MPI_SUM,
9797
MPI_COMM_WORLD);
98-
elapsed_time /= totalProcs;
98+
elapsed_time /= worldSize;
9999

100100
double base_bw = (double)(size) / 1.0E9 / elapsed_time;
101101
double alg_bw = base_bw;
102102
double factor = ((double)(totalProcs - 1)) / ((double)(totalProcs));
103103
double bus_bw = base_bw * factor;
104-
if (proc == 0) {
104+
if (proc == 0 && color == 0) {
105105
printf("Comm size: %zu bytes; Elapsed time: %lf sec; Algo bandwidth: %lf "
106106
"GB/s; Bus bandwidth: %lf GB/s\n",
107107
size, elapsed_time, alg_bw, bus_bw);
@@ -112,7 +112,7 @@ int main(int argc, char *argv[]) {
112112
devHandle->deviceMemset(hello, 0, size, flagcxMemHost, NULL);
113113
devHandle->deviceMemcpy(hello, recvbuff, size, flagcxMemcpyDeviceToHost,
114114
NULL);
115-
if ((proc == 0 || proc == totalProcs - 1) && print_buffer) {
115+
if ((proc == 0 || proc == totalProcs - 1) && color == 0 && print_buffer) {
116116
printf("recvbuff = ");
117117
for (int i = 0; i < totalProcs; i++) {
118118
printf("%f ", ((float *)hello)[i * (count / totalProcs)]);

‎test/perf/test_alltoallv.cpp‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#include "flagcx.h"
2-
#include "mpi.h"
32
#include "tools.h"
43
#include <cstring>
54
#include <iostream>
@@ -14,27 +13,28 @@ int main(int argc, char *argv[]) {
1413
int num_warmup_iters = args.getWarmupIters();
1514
int num_iters = args.getTestIters();
1615
int print_buffer = args.isPrintBuffer();
17-
18-
int totalProcs, proc;
19-
MPI_Init(&argc, &argv);
20-
MPI_Comm_size(MPI_COMM_WORLD, &totalProcs);
21-
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
22-
printf("I am %d of %d\n", proc, totalProcs);
16+
uint64_t split_mask = args.getSplitMask();
2317

2418
flagcxHandlerGroup_t handler;
2519
flagcxHandleInit(&handler);
2620
flagcxUniqueId_t &uniqueId = handler->uniqueId;
2721
flagcxComm_t &comm = handler->comm;
2822
flagcxDeviceHandle_t &devHandle = handler->devHandle;
2923

24+
int color = 0;
25+
int worldSize = 1, worldRank = 0;
26+
int totalProcs = 1, proc = 0;
27+
MPI_Comm splitComm;
28+
initMpiEnv(argc, argv, worldRank, worldSize, proc, totalProcs, color,
29+
splitComm, split_mask);
30+
3031
int nGpu;
3132
devHandle->getDeviceCount(&nGpu);
32-
devHandle->setDevice(proc % nGpu);
33+
devHandle->setDevice(worldRank % nGpu);
3334

3435
if (proc == 0)
3536
flagcxGetUniqueId(&uniqueId);
36-
MPI_Bcast((void *)uniqueId, sizeof(flagcxUniqueId), MPI_BYTE, 0,
37-
MPI_COMM_WORLD);
37+
MPI_Bcast((void *)uniqueId, sizeof(flagcxUniqueId), MPI_BYTE, 0, splitComm);
3838
MPI_Barrier(MPI_COMM_WORLD);
3939

4040
flagcxCommInitRank(&comm, totalProcs, uniqueId, proc);
@@ -170,7 +170,7 @@ int main(int argc, char *argv[]) {
170170
devHandle->deviceMemcpy(sendbuff, hello, size, flagcxMemcpyHostToDevice,
171171
NULL);
172172

173-
if ((proc == 0 || proc == totalProcs - 1) && print_buffer) {
173+
if ((proc == 0 || proc == totalProcs - 1) && color == 0 && print_buffer) {
174174
printf("sendbuff = ");
175175
for (int i = 0; i < totalProcs; i++) {
176176
printf("%f ", ((float *)hello)[i * count]);
@@ -218,7 +218,7 @@ int main(int argc, char *argv[]) {
218218
}
219219
}
220220

221-
if ((proc == 0 || proc == totalProcs - 1) && print_buffer) {
221+
if ((proc == 0 || proc == totalProcs - 1) && color == 0 && print_buffer) {
222222
printf("h_sendcounts = ");
223223
for (int i = 0; i < totalProcs; i++) {
224224
printf("%ld ", h_sendcounts[i]);
@@ -253,13 +253,13 @@ int main(int argc, char *argv[]) {
253253
double elapsed_time = tim.elapsed() / num_iters;
254254
MPI_Allreduce(MPI_IN_PLACE, (void *)&elapsed_time, 1, MPI_DOUBLE, MPI_SUM,
255255
MPI_COMM_WORLD);
256-
elapsed_time /= totalProcs;
256+
elapsed_time /= worldSize;
257257

258258
double base_bw = (double)(size) / 1.0E9 / elapsed_time;
259259
double alg_bw = base_bw;
260260
double factor = ((double)(totalProcs - 1)) / ((double)(totalProcs));
261261
double bus_bw = base_bw * factor;
262-
if (proc == 0) {
262+
if (proc == 0 && color == 0) {
263263
printf("Comm size: %zu bytes; Elapsed time: %lf sec; Algo bandwidth: %lf "
264264
"GB/s; Bus bandwidth: %lf GB/s\n",
265265
size, elapsed_time, alg_bw, bus_bw);
@@ -270,7 +270,7 @@ int main(int argc, char *argv[]) {
270270
devHandle->deviceMemset(hello, 0, size, flagcxMemHost, NULL);
271271
devHandle->deviceMemcpy(hello, recvbuff, size, flagcxMemcpyDeviceToHost,
272272
NULL);
273-
if ((proc == 0 || proc == totalProcs - 1) && print_buffer) {
273+
if ((proc == 0 || proc == totalProcs - 1) && color == 0 && print_buffer) {
274274
printf("recvbuff = ");
275275
for (int i = 0; i < totalProcs; i++) {
276276
printf("%f ", ((float *)hello)[i * count]);

0 commit comments

Comments
 (0)