Skip to content

Commit cb3f404

Browse files
hooray03hooray03
committed
ci(ascend): enable workflow validation
1 parent 63f7c17 commit cb3f404

16 files changed

Lines changed: 254 additions & 80 deletions

File tree

‎.github/configs/ascend.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ display_name: "Ascend NPU Tests"
2121

2222
# Docker image for this hardware
2323
ci_image: harbor.baai.ac.cn/flagscale/vllm-plugin-fl:v0.1.0-ascend-ci
24-
ci_train_image: harbor.baai.ac.cn/flagscale/vllm-plugin-fl:v0.1.0-ascend-ci
24+
ci_train_image: harbor.baai.ac.cn/flagos-dev/flagscale:manual-20260724-ascend-train-426ad116-te771a7488
2525
ci_inference_image: harbor.baai.ac.cn/flagscale/vllm-plugin-fl:v0.1.0-ascend-ci
2626

2727
# Runner labels for this hardware

‎.github/workflows/all_tests.yml‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -87,13 +87,12 @@ jobs:
8787
with:
8888
platform: cuda
8989

90-
# Disable util the new devices ready
91-
# ascend_tests:
92-
# name: Ascend tests
93-
# if: github.event_name != 'workflow_dispatch' || inputs.platform == 'all' || inputs.platform == 'ascend'
94-
# uses: ./.github/workflows/all_tests_common.yml
95-
# with:
96-
# platform: ascend
90+
ascend_tests:
91+
name: Ascend tests
92+
if: github.event_name != 'workflow_dispatch' || inputs.platform == 'all' || inputs.platform == 'ascend'
93+
uses: ./.github/workflows/all_tests_common.yml
94+
with:
95+
platform: ascend
9796

9897
metax_tests:
9998
name: MetaX tests
@@ -107,7 +106,7 @@ jobs:
107106
needs:
108107
- check_docker_changes
109108
- cuda_tests
110-
# - ascend_tests
109+
- ascend_tests
111110
- metax_tests
112111
runs-on: ubuntu-latest
113112
if: always()
@@ -124,11 +123,11 @@ jobs:
124123
# failed=true
125124
# fi
126125
127-
# if [ "${{ needs.ascend_tests.result }}" != "success" ] && \
128-
# [ "${{ needs.ascend_tests.result }}" != "skipped" ]; then
129-
# echo "❌ Ascend tests failed"
130-
# failed=true
131-
# fi
126+
if [ "${{ needs.ascend_tests.result }}" != "success" ] && \
127+
[ "${{ needs.ascend_tests.result }}" != "skipped" ]; then
128+
echo "❌ Ascend tests failed"
129+
failed=true
130+
fi
132131
133132
if [ "${{ needs.metax_tests.result }}" != "success" ] && \
134133
[ "${{ needs.metax_tests.result }}" != "skipped" ]; then

‎.github/workflows/functional_tests_inference.yml‎

Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,6 @@ jobs:
142142
run: cd $PROJECT_ROOT && bash ./tests/test_utils/runners/check_env.sh
143143

144144
- name: Setup environment for inference
145-
if: inputs.platform == 'cuda'
146145
run: |
147146
set -euo pipefail
148147
cd $PROJECT_ROOT
@@ -192,27 +191,6 @@ jobs:
192191
193192
timeout-minutes: 5
194193

195-
- name: Install dependencies for inference ascend
196-
if: inputs.platform == 'ascend'
197-
run: |
198-
set -euo pipefail
199-
200-
# Install vllm-plugin-FL
201-
pip install vllm-plugin-fl==0.1.0+vllm0.13.0 \
202-
--extra-index-url https://resource.flagos.net/repository/flagos-pypi-hosted/simple \
203-
|| { echo "❌ vllm-plugin-FL install failed"; exit 1; }
204-
echo "✅ vllm-plugin-FL installed successfully"
205-
206-
# Install FlagScale and dependencies
207-
cd $PROJECT_ROOT
208-
pip install . --no-build-isolation --root-user-action=ignore || { echo "❌ FlagScale CLI install failed"; exit 1; }
209-
210-
# Verify installation
211-
command -v flagscale || { echo "❌ FlagScale CLI not found in PATH"; exit 1; }
212-
echo "✅ FlagScale CLI installed successfully: $(flagscale --version 2>/dev/null || echo 'version unknown')"
213-
echo "✅ Environment ready for inference tests"
214-
timeout-minutes: 15
215-
216194
- name: Run functional tests
217195
id: functional_test
218196
run: |
@@ -255,7 +233,7 @@ jobs:
255233
fi
256234
;;
257235
pip)
258-
echo "ℹ️ Running tests with pip/system Python"
236+
echo "Running tests with pip/system Python"
259237
;;
260238
esac
261239

‎.github/workflows/functional_tests_serve.yml‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,6 @@ jobs:
142142
run: cd $PROJECT_ROOT && bash ./tests/test_utils/runners/check_env.sh
143143

144144
- name: Install dependencies for serve
145-
if: inputs.platform == 'cuda'
146145
run: |
147146
set -euo pipefail
148147
cd $PROJECT_ROOT
@@ -191,27 +190,6 @@ jobs:
191190
echo "Environment ready for serve tests"
192191
timeout-minutes: 5
193192

194-
- name: Install dependencies for serve ascend
195-
if: inputs.platform == 'ascend'
196-
run: |
197-
set -euo pipefail
198-
199-
# Install vllm-plugin-FL
200-
pip install vllm-plugin-fl==0.1.0+vllm0.13.0 \
201-
--extra-index-url https://resource.flagos.net/repository/flagos-pypi-hosted/simple \
202-
|| { echo "❌ vllm-plugin-FL install failed"; exit 1; }
203-
echo "✅ vllm-plugin-FL installed successfully"
204-
205-
# Install FlagScale
206-
cd $PROJECT_ROOT
207-
pip install . --no-build-isolation --root-user-action=ignore || { echo "❌ FlagScale CLI install failed"; exit 1; }
208-
209-
# Verify installation
210-
command -v flagscale || { echo "❌ FlagScale CLI not found in PATH"; exit 1; }
211-
echo "✅ FlagScale CLI installed successfully: $(flagscale --version 2>/dev/null || echo 'version unknown')"
212-
echo "✅ Environment ready for serve tests"
213-
timeout-minutes: 15
214-
215193
- name: Run functional tests
216194
id: functional_test
217195
env:

‎pyproject.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,3 +135,6 @@ extend-ignore-identifiers-re = [
135135
"nd_*",
136136
"\\d+[Bb][-]?[Aa]\\d+(\\.\\d+)?[Bb]", # protect 30BA3B
137137
]
138+
139+
[tool.typos.default.extend-words]
140+
CANN = "CANN"

‎tests/functional_tests/inference/qwen3/conf/4b_tp2_ascend.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ experiment:
2828

2929
envs:
3030
HYDRA_FULL_ERROR: 1
31-
ASCEND_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15"
31+
ASCEND_VISIBLE_DEVICES: "2,3"
3232
HCCL_WHITELIST_DISABLE: 1
3333
HCCL_CONNECT_TIMEOUT: 600
3434
ASCEND_LAUNCH_BLOCKING: 1
@@ -37,8 +37,10 @@ experiment:
3737
MKL_NUM_THREADS: 1
3838
OMP_NUM_THREADS: 1
3939
NUMEXPR_NUM_THREADS: 1
40+
# Serialize Triton launcher compilation to avoid concurrent .gch writes.
41+
CC: ./tests/test_utils/runners/triton_gxx_wrapper.sh
4042
VLLM_TARGET_DEVICE: "npu"
41-
VLLM_PLATFORM_PLUGINS: "ascend"
43+
VLLM_PLUGINS: "fl"
4244

4345
action: run
4446

‎tests/functional_tests/serve/qwen3/conf/4b_tp2_ascend.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,12 @@ experiment:
2828
port: 6703
2929
use_fs_serve: false
3030
envs:
31-
ASCEND_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15"
31+
ASCEND_VISIBLE_DEVICES: "4,5"
3232
HCCL_WHITELIST_DISABLE: 1
33+
# Serialize Triton launcher compilation to avoid concurrent .gch writes.
34+
CC: ./tests/test_utils/runners/triton_gxx_wrapper.sh
3335
VLLM_TARGET_DEVICE: "npu"
36+
VLLM_PLUGINS: "fl"
3437
no_proxy: "127.0.0.1,localhost"
3538

3639
action: run

‎tests/functional_tests/train/qwen3/conf/0_6b_ascend_smoke.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Copyright 2026 FlagOS Contributors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
defaults:
16+
- _self_
17+
- train: 0_6b_ascend_smoke
18+
19+
experiment:
20+
exp_name: 0_6b_ascend_smoke
21+
exp_dir: tests/functional_tests/train/qwen3/test_results/0_6b_ascend_smoke
22+
task:
23+
type: train
24+
backend: megatron
25+
entrypoint: tools/install/ascend/launch_megatron_gpt.py
26+
runner:
27+
ssh_port: null
28+
shell_cmds: null
29+
envs:
30+
HYDRA_FULL_ERROR: 1
31+
FS_PLATFORM: npu
32+
ASCEND_RT_VISIBLE_DEVICES: "0,1"
33+
HCCL_CONNECT_TIMEOUT: "1800"
34+
NCCL_ALGO: "Ring"
35+
NVTE_ALLOW_NONDETERMINISTIC_ALGO: "0"
36+
cmds:
37+
before_start: ''
38+
39+
action: run
40+
41+
hydra:
42+
run:
43+
dir: ${experiment.exp_dir}/hydra

‎tests/functional_tests/train/qwen3/conf/train/0_6b_ascend_smoke.yaml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# Copyright 2026 FlagOS Contributors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
defaults:
16+
- data
17+
18+
data:
19+
data_path: /opt/flagscale/test_data/pile_wikipedia_demo
20+
tokenizer:
21+
tokenizer_path: /home/gitlab-runner/data/Qwen3-4B
22+
23+
system:
24+
# torch_npu maps Megatron's supported NCCL choice to HCCL on Ascend.
25+
distributed_backend: nccl
26+
tensor_model_parallel_size: 1
27+
pipeline_model_parallel_size: 1
28+
disable_bias_linear: true
29+
reset_position_ids: true
30+
reset_attention_mask: true
31+
qk_layernorm: true
32+
use_distributed_optimizer: false
33+
precision:
34+
bf16: true
35+
attention_softmax_in_fp32: true
36+
accumulate_allreduce_grads_in_fp32: true
37+
logging:
38+
log_interval: 1
39+
no_log_loss_scale_to_tensorboard: true
40+
checkpoint:
41+
no_save_optim: true
42+
no_save_rng: true
43+
no_load_optim: true
44+
no_load_rng: true
45+
save_interval: 100000
46+
ckpt_format: torch
47+
tensorboard_log_interval: 999999
48+
49+
model:
50+
deterministic_mode: true
51+
transformer_impl: transformer_engine
52+
attention_backend: unfused
53+
te_fl_prefer: reference
54+
te_fl_per_op: "rmsnorm_fwd=reference"
55+
te_fl_allow_vendors: "npu"
56+
te_fl_deny_vendors: "cuda"
57+
enable_flag_gems: false
58+
no_persist_layer_norm: true
59+
no_masked_softmax_fusion: true
60+
no_check_for_nan_in_loss_and_grad: false
61+
no_gradient_accumulation_fusion: true
62+
num_layers: 28
63+
hidden_size: 1024
64+
ffn_hidden_size: 3072
65+
kv_channels: 128
66+
num_attention_heads: 16
67+
group_query_attention: true
68+
num_query_groups: 8
69+
seq_length: 256
70+
max_position_embeddings: 40960
71+
norm_epsilon: 1e-6
72+
use_rotary_position_embeddings: true
73+
rotary_base: 1000000
74+
no_rope_fusion: true
75+
swiglu: true
76+
normalization: RMSNorm
77+
untie_embeddings_and_output_weights: false
78+
no_position_embedding: true
79+
init_method_std: 0.006
80+
attention_dropout: 0.0
81+
hidden_dropout: 0.0
82+
clip_grad: 1.0
83+
train_iters: 10
84+
eval_iters: 0
85+
eval_interval: 1000
86+
micro_batch_size: 1
87+
global_batch_size: 2
88+
seed: 42
89+
90+
optimizer:
91+
weight_decay: 0.1
92+
adam_beta1: 0.9
93+
adam_beta2: 0.95
94+
lr_scheduler:
95+
lr: 1.0e-5
96+
min_lr: 1.0e-6
97+
lr_warmup_samples: 0
98+
lr_warmup_fraction: 0.01
99+
lr_decay_style: cosine

‎tests/functional_tests/train/qwen3/gold_values/0_6b_ascend_smoke.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"lm loss:": {"values": [11.93016, 11.87934, 11.78204, 11.84415, 11.64288, 11.57093, 11.56676, 11.75107, 11.77425, 11.44834], "rtol": 0.05, "atol": 0.1}}

0 commit comments

Comments
 (0)