Skip to content

Commit 1538669

Browse files
HermiaHuanHermiaHuan
authored
[CICD] Add Hygon BW1000 smoke CI workflow (#257)
## Summary This PR adds the first-stage Hygon/BW1000 CI smoke workflow. Main changes: - Add Hygon platform configuration. - Add Hygon runner, Docker image, device mount, and container options. - Add Hygon environment check and setup scripts. - Add Hygon platform test matrix. - Enable the first-stage smoke loop: - unit - functional - Qwen3.6-27B TP=2 eager inference - Qwen3.6-27B TP=2 eager serving - benchmark serve smoke ## Validation Environment validation has been completed on Hygon BW1000. Validated items: - Hygon Docker image startup: passed - Hygon device check: passed - unit tests: passed - functional tests: passed - Qwen3.6-27B TP=2 eager inference: passed - Qwen3.6-27B TP=2 eager serving: passed - benchmark serve smoke: passed ## Notes This PR keeps the first-stage CI scope minimal and only enables validated smoke cases by default. Larger model cases such as Qwen3.6-35B-A3B can be added to the default matrix after the model is provisioned and verified on the runner.
1 parent dbfe3be commit 1538669

11 files changed

Lines changed: 285 additions & 1 deletion

File tree

‎.github/configs/hygon.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Hygon Hardware Configuration
2+
# This file defines CI/CD settings for Hygon DCU testing.
3+
4+
platform: hygon
5+
6+
# Docker image for this hardware.
7+
ci_image: harbor.sourcefind.cn:5443/dcu/admin/base/custom:vllm0.20.0-ubuntu22.04-dtk26.04-py3.10-MiniCPM-V-4.6
8+
9+
# Runner labels for this hardware.
10+
runner_labels:
11+
- hg-cicd-vllm-plugin
12+
13+
# Container volumes (hardware-specific paths).
14+
container_volumes:
15+
- /opt/hyhal:/opt/hyhal
16+
- /data:/data
17+
- /workspace:/workspace
18+
19+
# Container options (hardware-specific settings).
20+
container_options: >-
21+
--hostname vllm-plugin-fl
22+
--ipc=host
23+
--env DTK_HOME=/opt/dtk
24+
--env GEMS_VENDOR=hygon
25+
--env ROCM_PATH=/opt/dtk
26+
--env HIP_PATH=/opt/dtk/hip
27+
--env HSA_PATH=/opt/dtk/hsa
28+
--env HIP_CLANG_PATH=/opt/dtk/llvm/bin
29+
--env DEVICE_LIB_PATH=/opt/dtk/amdgcn/bitcode
30+
--env PATH=/opt/dtk/bin:/opt/dtk/hip/bin:/opt/dtk/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
31+
--env LD_LIBRARY_PATH=/opt/hyhal/lib/criu:/opt/hyhal/lib/rocprofiler:/opt/hyhal/lib:/opt/dtk/hip/lib:/opt/dtk/lib:/opt/dtk/llvm/lib:/opt/dtk/dcc/lib:/opt/dtk/aillvm/lib:/opt/dtk/hsa/lib
32+
--device=/dev/kfd
33+
--device=/dev/mkfd
34+
--device=/dev/dri
35+
--group-add video
36+
--cap-add=SYS_PTRACE
37+
--security-opt seccomp=unconfined

‎.github/configs/platforms.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,5 @@ platforms:
1010
enabled: true
1111
ascend:
1212
enabled: false
13+
hygon:
14+
enabled: true

‎.github/scripts/hygon/check.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
# Copyright (c) 2025 BAAI. All rights reserved.
3+
# Check Hygon DCU availability.
4+
set -euo pipefail
5+
6+
echo "Current time: $(date '+%Y-%m-%d %H:%M:%S')"
7+
echo "=== Checking Hygon DCU availability ==="
8+
9+
HY_SMI_BIN=""
10+
if command -v hy-smi >/dev/null 2>&1; then
11+
HY_SMI_BIN="$(command -v hy-smi)"
12+
elif [[ -x /opt/hyhal/bin/hy-smi ]]; then
13+
HY_SMI_BIN="/opt/hyhal/bin/hy-smi"
14+
fi
15+
16+
if [[ -n "${HY_SMI_BIN}" ]]; then
17+
echo "Using hy-smi: ${HY_SMI_BIN}"
18+
"${HY_SMI_BIN}"
19+
"${HY_SMI_BIN}" --showmeminfo vram || true
20+
else
21+
echo "::warning::hy-smi not found in PATH or /opt/hyhal/bin; skipping SMI output."
22+
fi
23+
24+
test -e /dev/kfd
25+
test -e /dev/mkfd
26+
test -d /dev/dri
27+
test -d /opt/hyhal

‎.github/scripts/hygon/setup.sh‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#!/bin/bash
2+
# Copyright (c) 2025 BAAI. All rights reserved.
3+
# Setup script for Hygon DCU CI environment.
4+
set -euo pipefail
5+
6+
git config --global --add safe.directory "$(pwd)"
7+
8+
export GEMS_VENDOR="${GEMS_VENDOR:-hygon}"
9+
export DTK_HOME="${DTK_HOME:-/opt/dtk}"
10+
export ROCM_PATH="${ROCM_PATH:-${DTK_HOME}}"
11+
export HIP_PATH="${HIP_PATH:-${DTK_HOME}/hip}"
12+
export HSA_PATH="${HSA_PATH:-${DTK_HOME}/hsa}"
13+
export HIP_CLANG_PATH="${HIP_CLANG_PATH:-${DTK_HOME}/llvm/bin}"
14+
export DEVICE_LIB_PATH="${DEVICE_LIB_PATH:-${DTK_HOME}/amdgcn/bitcode}"
15+
16+
DTK_PATH="${DTK_HOME}/bin:${HIP_PATH}/bin:${HIP_CLANG_PATH}"
17+
DTK_LIBRARY_PATH="/opt/hyhal/lib/criu:/opt/hyhal/lib/rocprofiler:/opt/hyhal/lib:${HIP_PATH}/lib:${DTK_HOME}/lib:${DTK_HOME}/llvm/lib:${DTK_HOME}/dcc/lib:${DTK_HOME}/aillvm/lib:${HSA_PATH}/lib"
18+
export PATH="${DTK_PATH}:${PATH}"
19+
export LD_LIBRARY_PATH="${DTK_LIBRARY_PATH}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
20+
21+
if [[ -n "${GITHUB_ENV:-}" ]]; then
22+
for name in \
23+
GEMS_VENDOR \
24+
DTK_HOME \
25+
ROCM_PATH \
26+
HIP_PATH \
27+
HSA_PATH \
28+
HIP_CLANG_PATH \
29+
DEVICE_LIB_PATH \
30+
PATH \
31+
LD_LIBRARY_PATH; do
32+
echo "${name}=${!name}" >> "${GITHUB_ENV}"
33+
done
34+
fi
35+
36+
echo "DTK_HOME=${DTK_HOME}"
37+
echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}"
38+
test -e "${HIP_PATH}/lib/libgalaxyhip.so.5"
39+
test -e "${DTK_HOME}/llvm/lib/libomp.so"
40+
41+
TEST_DEPS=(
42+
pytest
43+
pytest-cov
44+
pytest-timeout
45+
pytest-json-report
46+
scikit-build-core==0.11
47+
pybind11
48+
ninja
49+
cmake
50+
numpy
51+
requests
52+
openai
53+
decorator
54+
pyyaml
55+
sqlalchemy
56+
)
57+
58+
FLAGGEMS_REF="8d23621eb1381ae96b315a9287ce3cc555433824"
59+
FLAGGEMS_SOURCE="${FLAGGEMS_PATH:-/workspace/FlagGems}"
60+
61+
if command -v uv >/dev/null 2>&1; then
62+
uv pip install --system --upgrade pip
63+
uv pip install --system --no-build-isolation -e . --no-deps
64+
uv pip install --system "${TEST_DEPS[@]}"
65+
else
66+
python -m pip install --upgrade pip
67+
python -m pip install --no-build-isolation -e . --no-deps
68+
python -m pip install "${TEST_DEPS[@]}"
69+
fi
70+
71+
test -d "${FLAGGEMS_SOURCE}/src/flag_gems"
72+
git config --global --add safe.directory "${FLAGGEMS_SOURCE}"
73+
74+
FLAGGEMS_HEAD="$(git -C "${FLAGGEMS_SOURCE}" rev-parse HEAD)"
75+
if [[ "${FLAGGEMS_HEAD}" != "${FLAGGEMS_REF}" ]]; then
76+
echo "Unexpected FlagGems commit: ${FLAGGEMS_HEAD}; expected ${FLAGGEMS_REF}."
77+
exit 1
78+
fi
79+
80+
if ! grep -q 'kwargs.pop("num_ldmatrixes", None)' \
81+
"${FLAGGEMS_SOURCE}/src/flag_gems/runtime/configloader.py"; then
82+
echo "FlagGems num_ldmatrixes compatibility patch is missing."
83+
exit 1
84+
fi
85+
86+
FLAGGEMS_DIR="$(mktemp -d)/FlagGems"
87+
cp -a "${FLAGGEMS_SOURCE}" "${FLAGGEMS_DIR}"
88+
89+
if command -v uv >/dev/null 2>&1; then
90+
uv pip install --system --no-build-isolation -e "${FLAGGEMS_DIR}" --no-deps
91+
else
92+
python -m pip install --no-build-isolation -e "${FLAGGEMS_DIR}" --no-deps
93+
fi
94+
95+
python - <<'PY'
96+
import flag_gems
97+
import torch
98+
99+
print(f"FlagGems import ok: {getattr(flag_gems, '__version__', 'unknown')}")
100+
print(f"Torch import ok: {torch.__version__}")
101+
print(f"Accelerator available: {torch.cuda.is_available()}")
102+
print(f"Accelerator count: {torch.cuda.device_count()}")
103+
PY

‎.github/workflows/ci.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,3 +98,14 @@ jobs:
9898
with:
9999
platform: ascend
100100
secrets: inherit
101+
102+
# ============================================================
103+
# Job 4c: Hygon platform testing
104+
# ============================================================
105+
test-hygon:
106+
needs: discover
107+
if: contains(fromJson(needs.discover.outputs.platforms), 'hygon')
108+
uses: ./.github/workflows/_platform_test.yml
109+
with:
110+
platform: hygon
111+
secrets: inherit

‎tests/e2e_tests/serving/test_serving_smoke.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ def server():
6363
tp_size=_CFG.engine.get("tensor_parallel_size", 1),
6464
api_key=serve.api_key,
6565
served_model_name=serve.served_model_name,
66+
max_retries=serve.startup_retries,
6667
extra_args=extra_args,
6768
) as srv:
6869
yield srv

‎tests/models/qwen3_6/27b_tp2_eager.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Qwen3.6-27B Hygon smoke configuration (TP=2, eager).
2+
3+
llm:
4+
model: "/workspace/Qwen3.6-27B"
5+
tensor_parallel_size: 2
6+
pipeline_parallel_size: 1
7+
max_model_len: 8192
8+
gpu_memory_utilization: 0.95
9+
enforce_eager: true
10+
trust_remote_code: false
11+
disable_custom_all_reduce: true
12+
13+
generate:
14+
prompts:
15+
- "Introduce yourself,please"
16+
sampling:
17+
max_tokens: 100
18+
temperature: 0.0
19+
20+
serve:
21+
served_model_name: "qwen"
22+
endpoints: ["chat"]
23+
stream: false
24+
max_tokens: 256
25+
chat_messages:
26+
- role: "user"
27+
content: "Introduce yourself,please"
28+
sampling:
29+
temperature: 0.0

‎tests/models/qwen3_6/35b_a3b_tp2_eager.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Qwen3.6-35B-A3B Hygon smoke configuration (TP=2, eager).
2+
3+
llm:
4+
model: "/workspace/Qwen3.6-35B-A3B"
5+
tensor_parallel_size: 2
6+
pipeline_parallel_size: 1
7+
max_model_len: 8192
8+
gpu_memory_utilization: 0.95
9+
enforce_eager: true
10+
trust_remote_code: false
11+
disable_custom_all_reduce: true
12+
13+
generate:
14+
prompts:
15+
- "Introduce yourself,please"
16+
sampling:
17+
max_tokens: 100
18+
temperature: 0.0
19+
20+
serve:
21+
served_model_name: "qwen"
22+
startup_retries: 120
23+
endpoints: ["chat"]
24+
stream: false
25+
max_tokens: 256
26+
chat_messages:
27+
- role: "user"
28+
content: "Introduce yourself,please"
29+
sampling:
30+
temperature: 0.0

‎tests/platforms/hygon.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
platform: hygon
2+
vendor: hygon
3+
4+
device_types:
5+
bw1000:
6+
memory_gb: 64
7+
tags: [bf16, fp16, fp32]
8+
9+
tolerance:
10+
inference:
11+
default:
12+
rtol: 1e-5
13+
atol: 1e-8
14+
bfloat16:
15+
rtol: 1e-3
16+
atol: 1e-3
17+
18+
env_defaults:
19+
GEMS_VENDOR: "hygon"
20+
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
21+
VLLM_FL_FLAGOS_BLACKLIST: "pow_scalar,pow_tensor_scalar,pow_tensor_tensor,pow_tensor_scalar_,pow_tensor_tensor_,cat"
22+
23+
bw1000:
24+
name: "bw1000"
25+
tests:
26+
e2e:
27+
inference:
28+
qwen3_6: ["27b_tp2_eager"]
29+
serving:
30+
qwen3_6: ["35b_a3b_tp2_eager"]
31+
functional:
32+
include: "*"
33+
exclude: []
34+
unit:
35+
include: "*"
36+
exclude: []
37+
benchmark:
38+
enabled: true
39+
smoke: ["serve"]

‎tests/unit_tests/flaggems/test_gems_whitelist.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
# On some platforms (e.g. Ascend), get_flagos_blacklist() returns a non-empty
1717
# default blacklist which would interfere with env-var-only assertions.
1818
_no_platform_blacklist = patch(
19-
"vllm_fl.dispatch.config.get_flagos_blacklist", return_value=None
19+
"vllm_fl.dispatch.config.get_flagos_blacklist", new=lambda: None
2020
)
2121

2222

0 commit comments

Comments
 (0)