Skip to content

Commit dc7e499

Browse files
AlexMa616Darryl233
AlexMa616
and
Darryl233
authored
[CICD] Add configurable CUDA unit test pipeline (flagos-ai#527)
--------- Co-authored-by: liyuzhuo <lee.yuzhuo233@gmail.com>
1 parent 032ffb6 commit dc7e499

16 files changed

Lines changed: 535 additions & 591 deletions

‎.github/configs/cuda.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# CUDA Hardware Configuration
2+
hardware_name: cuda
3+
display_name: CUDA Tests
4+
5+
ci_image: harbor.baai.ac.cn/flagos-dev/flagcx:55c5c6f-cuda-dev
6+
7+
runner_labels:
8+
- nv-8g-cicd-flagcx
9+
10+
container_volumes: []
11+
12+
container_options: >-
13+
--gpus all
14+
--privileged
15+
--ipc=host
16+
--ulimit memlock=-1
17+
--ulimit stack=67108864
18+
19+
set_env: .github/scripts/set_env/cuda.sh
20+
21+
unit_test_suites:
22+
- adaptor
23+
- core
24+
- device_api
25+
- p2p
26+
- rma
27+
- runner
28+
- service
29+
- symmem

‎.github/scripts/ci/load_platform_config.rb‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/usr/bin/env ruby
2+
# frozen_string_literal: true
3+
4+
require "json"
5+
require "yaml"
6+
7+
platform = ARGV.fetch(0) { abort "Usage: #{$PROGRAM_NAME} <platform>" }
8+
config_path = ".github/configs/#{platform}.yml"
9+
abort "Platform config not found: #{config_path}" unless File.file?(config_path)
10+
11+
contents = File.read(config_path)
12+
config = begin
13+
YAML.safe_load(
14+
contents,
15+
permitted_classes: [],
16+
permitted_symbols: [],
17+
aliases: false,
18+
filename: config_path
19+
)
20+
rescue ArgumentError
21+
YAML.safe_load(contents, [], [], false, config_path)
22+
end
23+
24+
required_keys = %w[
25+
hardware_name
26+
display_name
27+
ci_image
28+
runner_labels
29+
container_volumes
30+
container_options
31+
set_env
32+
unit_test_suites
33+
]
34+
missing = required_keys.reject { |key| config.key?(key) }
35+
abort "#{config_path}: missing required keys: #{missing.join(', ')}" unless missing.empty?
36+
37+
hardware_name = config.fetch("hardware_name")
38+
abort "#{config_path}: hardware_name must be #{platform}" unless hardware_name == platform
39+
40+
runner_labels = config.fetch("runner_labels")
41+
abort "#{config_path}: runner_labels must be a non-empty array" unless runner_labels.is_a?(Array) && !runner_labels.empty?
42+
43+
container_volumes = config.fetch("container_volumes")
44+
abort "#{config_path}: container_volumes must be an array" unless container_volumes.is_a?(Array)
45+
46+
suites = config.fetch("unit_test_suites")
47+
abort "#{config_path}: unit_test_suites must be a non-empty array" unless suites.is_a?(Array) && !suites.empty?
48+
49+
set_env = config.fetch("set_env")
50+
abort "#{config_path}: set_env does not exist: #{set_env}" unless File.file?(set_env)
51+
52+
outputs = {
53+
"display_name" => config.fetch("display_name"),
54+
"ci_image" => config.fetch("ci_image"),
55+
"runs_on" => JSON.generate(runner_labels),
56+
"container_volumes" => JSON.generate(container_volumes),
57+
"container_options" => config.fetch("container_options"),
58+
"set_env" => set_env,
59+
"unit_test_suites" => JSON.generate(suites)
60+
}
61+
62+
if ENV["GITHUB_OUTPUT"] && !ENV["GITHUB_OUTPUT"].empty?
63+
File.open(ENV.fetch("GITHUB_OUTPUT"), "a") do |output|
64+
outputs.each do |key, value|
65+
delimiter = "FLAGCX_#{key.upcase}"
66+
output.puts "#{key}<<#{delimiter}"
67+
output.puts value
68+
output.puts delimiter
69+
end
70+
end
71+
else
72+
puts JSON.pretty_generate(outputs)
73+
end

‎.github/scripts/ci/load_platform_matrix.rb‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env ruby
2+
# frozen_string_literal: true
3+
4+
require "json"
5+
require "yaml"
6+
7+
config_dir = ARGV.fetch(0, ".github/configs")
8+
config_files = Dir.glob(File.join(config_dir, "*.yml")).sort
9+
abort "No platform configs found in #{config_dir}" if config_files.empty?
10+
11+
platforms = config_files.map do |path|
12+
contents = File.read(path)
13+
config = begin
14+
YAML.safe_load(
15+
contents,
16+
permitted_classes: [],
17+
permitted_symbols: [],
18+
aliases: false,
19+
filename: path
20+
)
21+
rescue ArgumentError
22+
# Compatibility with the older Ruby/Psych available on some self-hosted
23+
# runners, whose safe_load API only accepts positional arguments.
24+
YAML.safe_load(contents, [], [], false, path)
25+
end
26+
required_keys = %w[hardware_name display_name]
27+
missing = required_keys.reject { |key| config.key?(key) }
28+
abort "#{path}: missing required keys: #{missing.join(', ')}" unless missing.empty?
29+
30+
platform = File.basename(path, ".yml")
31+
hardware_name = config.fetch("hardware_name")
32+
abort "#{path}: hardware_name must match file name #{platform}" unless hardware_name == platform
33+
34+
{
35+
"platform" => platform,
36+
"display_name" => config.fetch("display_name")
37+
}
38+
end
39+
40+
duplicates = platforms.group_by { |platform| platform.fetch("platform") }
41+
.select { |_name, entries| entries.length > 1 }
42+
abort "Duplicate platform configs: #{duplicates.keys.join(', ')}" unless duplicates.empty?
43+
44+
matrix = JSON.generate({ "include" => platforms })
45+
puts matrix
46+
47+
if ENV["GITHUB_OUTPUT"] && !ENV["GITHUB_OUTPUT"].empty?
48+
File.open(ENV.fetch("GITHUB_OUTPUT"), "a") do |output|
49+
output.puts "matrix=#{matrix}"
50+
end
51+
end

‎.github/scripts/ci/run_unit_test.sh‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
if [[ $# -ne 2 ]]; then
6+
echo "Usage: $0 <set-env-script> <suite>" >&2
7+
exit 2
8+
fi
9+
10+
SET_ENV_SCRIPT=$1
11+
SUITE=$2
12+
PROJECT_ROOT=${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}
13+
14+
if [[ ! -f "$SET_ENV_SCRIPT" ]]; then
15+
echo "Platform environment script not found: $SET_ENV_SCRIPT" >&2
16+
exit 1
17+
fi
18+
19+
# The platform script owns accelerator-specific compiler flags and device
20+
# topology. It is sourced (rather than executed) so it can provide arrays and
21+
# hook functions without unsafe string evaluation.
22+
# shellcheck source=/dev/null
23+
source "$SET_ENV_SCRIPT"
24+
25+
if declare -F flagcx_ci_configure_suite >/dev/null; then
26+
flagcx_ci_configure_suite "$SUITE"
27+
fi
28+
29+
: "${MPI_HOME:?The platform set_env script must define MPI_HOME}"
30+
declare -p FLAGCX_CI_PROJECT_MAKE_ARGS >/dev/null 2>&1 || {
31+
echo "The platform set_env script must define FLAGCX_CI_PROJECT_MAKE_ARGS" >&2
32+
exit 1
33+
}
34+
declare -p FLAGCX_CI_TEST_MAKE_ARGS >/dev/null 2>&1 || {
35+
echo "The platform set_env script must define FLAGCX_CI_TEST_MAKE_ARGS" >&2
36+
exit 1
37+
}
38+
39+
export PATH="$MPI_HOME/bin:$PATH"
40+
export LD_LIBRARY_PATH="$PROJECT_ROOT/build/lib:${LD_LIBRARY_PATH:-}"
41+
42+
if declare -F flagcx_ci_prepare >/dev/null; then
43+
flagcx_ci_prepare "$SUITE"
44+
fi
45+
46+
build_googletest() {
47+
cmake -S "$PROJECT_ROOT/third-party/googletest" \
48+
-B "$PROJECT_ROOT/third-party/googletest/build"
49+
cmake --build "$PROJECT_ROOT/third-party/googletest/build" --parallel "$(nproc)"
50+
}
51+
52+
build_project() {
53+
local -a args=("${FLAGCX_CI_PROJECT_MAKE_ARGS[@]}")
54+
make -C "$PROJECT_ROOT" --jobs="$(nproc)" "${args[@]}"
55+
}
56+
57+
build_suite() {
58+
local suite_dir="$PROJECT_ROOT/test/unittest/$SUITE"
59+
local -a args=("${FLAGCX_CI_TEST_MAKE_ARGS[@]}")
60+
make -C "$suite_dir" --jobs="$(nproc)" "${args[@]}"
61+
}
62+
63+
run_device_api() {
64+
local suite_dir="$PROJECT_ROOT/test/unittest/device_api"
65+
local -a common_env=(
66+
-x FLAGCX_USE_HETERO_COMM=1
67+
-x FLAGCX_MEM_ENABLE=1
68+
-x FLAGCX_VMM_ENABLE=0
69+
-x FLAGCX_P2P_DISABLE=1
70+
-x LD_LIBRARY_PATH
71+
)
72+
local -a flags=(-b 1M -e 4M -f 2 -R 1)
73+
74+
declare -p FLAGCX_CI_NODE1_MPI_ARGS >/dev/null 2>&1 || {
75+
echo "The platform set_env script must define FLAGCX_CI_NODE1_MPI_ARGS" >&2
76+
exit 1
77+
}
78+
declare -p FLAGCX_CI_NODE2_MPI_ARGS >/dev/null 2>&1 || {
79+
echo "The platform set_env script must define FLAGCX_CI_NODE2_MPI_ARGS" >&2
80+
exit 1
81+
}
82+
: "${FLAGCX_CI_INTRA_NP:?The platform set_env script must define FLAGCX_CI_INTRA_NP}"
83+
: "${FLAGCX_CI_NODE_NP:?The platform set_env script must define FLAGCX_CI_NODE_NP}"
84+
85+
cd "$suite_dir"
86+
mpirun -np "$FLAGCX_CI_INTRA_NP" --allow-run-as-root "${common_env[@]}" \
87+
build/bin/test_device_api_intra "${flags[@]}"
88+
mpirun -np "$FLAGCX_CI_INTRA_NP" --allow-run-as-root "${common_env[@]}" \
89+
build/bin/test_device_ir_intra "${flags[@]}"
90+
91+
mpirun --allow-run-as-root \
92+
-np "$FLAGCX_CI_NODE_NP" "${common_env[@]}" "${FLAGCX_CI_NODE1_MPI_ARGS[@]}" \
93+
build/bin/test_device_api_inter "${flags[@]}" \
94+
: -np "$FLAGCX_CI_NODE_NP" "${common_env[@]}" "${FLAGCX_CI_NODE2_MPI_ARGS[@]}" \
95+
build/bin/test_device_api_inter "${flags[@]}"
96+
mpirun --allow-run-as-root \
97+
-np "$FLAGCX_CI_NODE_NP" "${common_env[@]}" "${FLAGCX_CI_NODE1_MPI_ARGS[@]}" \
98+
build/bin/test_device_ir_inter "${flags[@]}" \
99+
: -np "$FLAGCX_CI_NODE_NP" "${common_env[@]}" "${FLAGCX_CI_NODE2_MPI_ARGS[@]}" \
100+
build/bin/test_device_ir_inter "${flags[@]}"
101+
}
102+
103+
run_suite() {
104+
local suite_dir="$PROJECT_ROOT/test/unittest/$SUITE"
105+
local -a args=("${FLAGCX_CI_TEST_MAKE_ARGS[@]}")
106+
case "$SUITE" in
107+
adaptor|core|service)
108+
make -C "$suite_dir" run-unit "${args[@]}"
109+
;;
110+
p2p)
111+
FLAGCX_USE_HETERO_COMM=1 FLAGCX_MEM_ENABLE=1 FLAGCX_VMM_ENABLE=0 \
112+
make -C "$suite_dir" run-unit "${args[@]}"
113+
;;
114+
rma)
115+
make -C "$suite_dir" run-mpi "${args[@]}"
116+
;;
117+
runner)
118+
: "${FLAGCX_CI_RUNNER_NP:?The platform set_env script must define FLAGCX_CI_RUNNER_NP}"
119+
make -C "$suite_dir" run-unit "${args[@]}"
120+
cd "$suite_dir"
121+
mpirun -np "$FLAGCX_CI_RUNNER_NP" --allow-run-as-root \
122+
./build/bin/runner_mpi_tests
123+
mpirun -np "$FLAGCX_CI_RUNNER_NP" --allow-run-as-root \
124+
-x FLAGCX_MEM_ENABLE=1 \
125+
-x FLAGCX_CLUSTER_SPLIT_LIST=2 \
126+
./build/bin/runner_mpi_tests
127+
mpirun -np "$FLAGCX_CI_RUNNER_NP" --allow-run-as-root \
128+
-x FLAGCX_MEM_ENABLE=1 \
129+
-x FLAGCX_CLUSTER_SPLIT_LIST=2 \
130+
-x FLAGCX_P2P_DISABLE=1 \
131+
-x FLAGCX_VMM_ENABLE=0 \
132+
./build/bin/runner_mpi_tests
133+
;;
134+
symmem)
135+
bash "$PROJECT_ROOT/test/script/symmem_test.sh"
136+
;;
137+
device_api)
138+
run_device_api
139+
;;
140+
*)
141+
echo "Unsupported unit test suite: $SUITE" >&2
142+
exit 2
143+
;;
144+
esac
145+
}
146+
147+
case "$SUITE" in
148+
device_api|symmem)
149+
;;
150+
adaptor|core|p2p|rma|runner|service)
151+
build_googletest
152+
;;
153+
*)
154+
echo "Unsupported unit test suite: $SUITE" >&2
155+
exit 2
156+
;;
157+
esac
158+
159+
build_project
160+
build_suite
161+
run_suite

‎.github/scripts/set_env/cuda.sh‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env bash
2+
3+
# Everything in this file is CUDA/NVIDIA specific. The common workflow and
4+
# runner deliberately do not inspect the platform name.
5+
6+
FLAGCX_CI_MPI_BASE_HOME=${MPI_HOME:-/usr/local/mpi}
7+
8+
# The FlagScale training image ships a launcher wrapper that passes "$@" as a
9+
# literal argument. Use the real OpenMPI launcher through an isolated MPI_HOME
10+
# so Makefiles and test scripts can keep invoking plain `mpirun`.
11+
if [[ -x "$FLAGCX_CI_MPI_BASE_HOME/bin/mpirun.real" ]]; then
12+
FLAGCX_CI_MPI_HOME=$(mktemp -d)
13+
mkdir -p "$FLAGCX_CI_MPI_HOME/bin"
14+
ln -s "$FLAGCX_CI_MPI_BASE_HOME/bin/mpirun.real" \
15+
"$FLAGCX_CI_MPI_HOME/bin/mpirun"
16+
ln -s "$FLAGCX_CI_MPI_BASE_HOME/include" "$FLAGCX_CI_MPI_HOME/include"
17+
ln -s "$FLAGCX_CI_MPI_BASE_HOME/lib" "$FLAGCX_CI_MPI_HOME/lib"
18+
export MPI_HOME=$FLAGCX_CI_MPI_HOME
19+
else
20+
export MPI_HOME=$FLAGCX_CI_MPI_BASE_HOME
21+
fi
22+
23+
FLAGCX_CI_PROJECT_MAKE_ARGS=(USE_NVIDIA=1)
24+
FLAGCX_CI_TEST_MAKE_ARGS=(USE_NVIDIA=1)
25+
FLAGCX_CI_INTRA_NP=8
26+
FLAGCX_CI_NODE_NP=4
27+
FLAGCX_CI_RUNNER_NP=8
28+
export NP=8
29+
30+
# Two logical four-GPU nodes on the eight-GPU CUDA runner.
31+
FLAGCX_CI_NODE1_MPI_ARGS=(
32+
-x CUDA_VISIBLE_DEVICES=0,1,2,3
33+
-x FLAGCX_HOSTID=node0
34+
-x FLAGCX_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3
35+
)
36+
FLAGCX_CI_NODE2_MPI_ARGS=(
37+
-x CUDA_VISIBLE_DEVICES=4,5,6,7
38+
-x FLAGCX_HOSTID=node1
39+
-x FLAGCX_IB_HCA=mlx5_4,mlx5_5,mlx5_6,mlx5_7
40+
)
41+
42+
flagcx_ci_configure_suite() {
43+
local suite=$1
44+
case "$suite" in
45+
runner)
46+
FLAGCX_CI_PROJECT_MAKE_ARGS+=(COMPILE_KERNEL=1)
47+
;;
48+
device_api)
49+
FLAGCX_CI_PROJECT_MAKE_ARGS+=(COMPILE_KERNEL=1 FORCE_DEFAULT_PATH=1)
50+
FLAGCX_CI_TEST_MAKE_ARGS+=(FORCE_DEFAULT_PATH=1)
51+
;;
52+
esac
53+
}
54+
55+
flagcx_ci_prepare() {
56+
local suite=$1
57+
echo "Preparing CUDA environment for unit-test suite: $suite"
58+
command -v nvcc
59+
command -v mpirun
60+
mpirun --version
61+
nvidia-smi
62+
}

0 commit comments

Comments
 (0)