1+ trigger :
2+ branches :
3+ include :
4+ - main
5+ - release/*
6+ paths :
7+ exclude :
8+ - .devcontainer/**
9+ - .github/**
10+ - docker/**
11+ - docs/**
12+ - ' **/*.md'
13+
14+ # Do not run multi-nodes-test for PR, we can trigger it manually
15+ pr : none
16+
17+ parameters :
18+ - name : vmssName
19+ type : string
20+ default : ' mnnvl' # unused on pilot (StartVMSS gated off); kept only to satisfy deploy.yml
21+ - name : hostEntries
22+ type : string
23+ default : |
24+ 10.0.5.145 DSM121082302011
25+ 10.0.5.150 DSM121082302014
26+ 10.0.5.177 DSM121082301013
27+ 10.0.5.225 DSM121082301032
28+ # Subnet (or interface name) passed to Open MPI's btl_tcp_if_include so every
29+ # rank selects the routable interface matching the node IPs in hostEntries.
30+ - name : mpiTcpIfInclude
31+ type : string
32+ default : ' 10.0.5.0/24'
33+
34+ variables :
35+ - group : mscclpp
36+
37+ jobs :
38+ - job : MultiNodesTest
39+ displayName : Multi nodes test
40+ strategy :
41+ matrix :
42+ cuda13 :
43+ containerImage : ghcr.io/microsoft/mscclpp/mscclpp:base-dev-cuda13.0
44+ pool :
45+ name : msccl-ci-gb200
46+ container :
47+ image : $[ variables['containerImage'] ]
48+
49+ steps :
50+ - task : Bash@3
51+ displayName : Node connectivity check
52+ inputs :
53+ targetType : ' inline'
54+ script : |
55+ set -x
56+ echo "Agent egress IP:"; curl -s -m 5 ifconfig.me; echo
57+ while IFS= read -r line; do
58+ ip=$(awk 'NF>=1{print $1}' <<< "$line")
59+ [ -z "$ip" ] && continue
60+ timeout 10 bash -c "cat < /dev/null > /dev/tcp/${ip}/22" \
61+ && echo "${ip}:22 OPEN" || echo "${ip}:22 UNREACHABLE"
62+ done <<< "${{ parameters.hostEntries }}"
63+
64+ task : Bash@3
65+ displayName : Add HostEntry
66+ inputs :
67+ targetType : ' inline'
68+ script : |
69+ while IFS= read -r line; do
70+ [ -z "$line" ] && continue
71+ if ! grep -qxF "$line" /etc/hosts; then
72+ echo "Adding to /etc/hosts: $line"
73+ echo "$line" | sudo tee -a /etc/hosts
74+ else
75+ echo "Entry already exists: $line"
76+ fi
77+ done <<< "${{ parameters.hostEntries }}"
78+
79+ task : Bash@3
80+ displayName : Generate deploy files
81+ inputs :
82+ targetType : ' inline'
83+ script : |
84+ set -e
85+ DEPLOY_DIR="$(System.DefaultWorkingDirectory)/test/deploy"
86+ # First NVLink pair (2 nodes) from hostEntries; use $1 (the IP) so the
87+ # names resolve inside the container too (the DSM hostnames are only in
88+ # the agent's /etc/hosts). Bump to more nodes by parsing further lines.
89+ NODE0=$(awk 'NF>=2{print $1; exit}' <<< "${{ parameters.hostEntries }}")
90+ NODE1=$(awk 'NF>=2{c++} c==2{print $1; exit}' <<< "${{ parameters.hostEntries }}")
91+
92+ echo "Host ${NODE0}
93+ Port 22345
94+ IdentityFile /root/mscclpp/sshkey
95+ StrictHostKeyChecking no
96+ Host ${NODE1}
97+ Port 22345
98+ IdentityFile /root/mscclpp/sshkey
99+ StrictHostKeyChecking no" > "${DEPLOY_DIR}/config"
100+
101+ printf '%s\n%s\n' "azhpcuser@${NODE0}" "azhpcuser@${NODE1}" > "${DEPLOY_DIR}/hostfile"
102+
103+ printf '%s\n%s\n' "${NODE0}" "${NODE1}" > "${DEPLOY_DIR}/hostfile_mpi"
104+
105+ # Publish the head node so the run steps can target it without hardcoding.
106+ echo "##vso[task.setvariable variable=headNode]${NODE0}"
107+
108+ template : templates/deploy.yml
109+ parameters :
110+ subscription : mscclpp-ci-h100
111+ vmssName : ${{ parameters.vmssName }}
112+ pilot : true
113+ resourceGroup : mscclpp
114+ gpuArch : ' 100a'
115+ cmakeArgs : ' -DMSCCLPP_USE_IB=OFF'
116+ deployArgs : ' multi-node-test false cuda'
117+
118+ - template : templates/ut-gb200.yml
119+ parameters :
120+ user : azhpcuser
121+ headNode : $(headNode)
122+ hostfile : $(System.DefaultWorkingDirectory)/test/deploy/hostfile
123+ gpuArch : ' 100a'
124+ mpiTcpIfInclude : ${{ parameters.mpiTcpIfInclude }}
125+
126+ - template : templates/run-remote-task.yml
127+ parameters :
128+ name : RunCollectivebenchmarks
129+ displayName : Run Collective Benchmarks
130+ runRemoteArgs : ' --hostfile $(System.DefaultWorkingDirectory)/test/deploy/hostfile --host $(headNode) --user azhpcuser'
131+ remoteScript : |
132+ mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include ${{ parameters.mpiTcpIfInclude }} -np 8 -npernode 4 \
133+ -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
134+ /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allreduce --dtype float16 --symmetric-memory
135+ mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include ${{ parameters.mpiTcpIfInclude }} -np 8 -npernode 4 \
136+ -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
137+ /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allgather --dtype float16 --symmetric-memory
138+ mpirun --allow-run-as-root --bind-to numa -hostfile /root/mscclpp/test/deploy/hostfile_mpi -mca btl_tcp_if_include ${{ parameters.mpiTcpIfInclude }} -np 8 -npernode 4 \
139+ -x MSCCLPP_DEBUG=WARN -x LD_LIBRARY_PATH=/root/mscclpp/build/lib:$LD_LIBRARY_PATH -x MSCCLPP_HOME=/root/mscclpp \
140+ /root/venv/bin/python3 -m mscclpp_benchmark.bench_collective --collective allgather --dtype float16 --symmetric-memory --buffer-mode out-of-place
0 commit comments