Skip to content

Commit 93b5dc4

Browse files
马函廷马函廷
authored andcommitted
test(ascend): enable real TE training smoke in CI
1 parent 9c76dcc commit 93b5dc4

8 files changed

Lines changed: 269 additions & 41 deletions

File tree

.github/configs/ascend.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ display_name: "Ascend NPU Tests"
2121

2222
# Docker image for this hardware
2323
ci_image: harbor.baai.ac.cn/flagscale/vllm-plugin-fl:v0.1.0-ascend-ci
24-
ci_train_image: harbor.baai.ac.cn/flagscale/vllm-plugin-fl:v0.1.0-ascend-ci
24+
ci_train_image: harbor.baai.ac.cn/flagos-dev/flagscale:manual-20260724-ascend-train-426ad116-te771a7488
2525
ci_inference_image: harbor.baai.ac.cn/flagscale/vllm-plugin-fl:v0.1.0-ascend-ci
2626

2727
# Runner labels for this hardware

.github/workflows/all_tests.yml

Lines changed: 24 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,20 +20,28 @@ on:
2020
paths-ignore:
2121
- 'docs/**'
2222
- 'docker/cuda/**'
23+
- 'docker/ascend/**'
2324
- 'docker/build.sh'
24-
- 'tools/install/**'
25-
- 'requirements/**'
25+
- 'tools/install/cuda/**'
26+
- 'tools/install/ascend/**'
27+
- 'requirements/cuda/**'
28+
- 'requirements/ascend/**'
2629
- '.github/workflows/build_image_cuda.yml'
30+
- '.github/workflows/build_image_ascend.yml'
2731
- '.github/workflows/push_image_harbor.yml'
2832
pull_request:
2933
branches: ["main"]
3034
paths-ignore:
3135
- 'docs/**'
3236
- 'docker/cuda/**'
37+
- 'docker/ascend/**'
3338
- 'docker/build.sh'
34-
- 'tools/install/**'
35-
- 'requirements/**'
39+
- 'tools/install/cuda/**'
40+
- 'tools/install/ascend/**'
41+
- 'requirements/cuda/**'
42+
- 'requirements/ascend/**'
3643
- '.github/workflows/build_image_cuda.yml'
44+
- '.github/workflows/build_image_ascend.yml'
3745
- '.github/workflows/push_image_harbor.yml'
3846
workflow_dispatch:
3947
inputs:
@@ -87,13 +95,12 @@ jobs:
8795
with:
8896
platform: cuda
8997

90-
# Disable util the new devices ready
91-
# ascend_tests:
92-
# name: Ascend tests
93-
# if: github.event_name != 'workflow_dispatch' || inputs.platform == 'all' || inputs.platform == 'ascend'
94-
# uses: ./.github/workflows/all_tests_common.yml
95-
# with:
96-
# platform: ascend
98+
ascend_tests:
99+
name: Ascend tests
100+
if: github.event_name != 'workflow_dispatch' || inputs.platform == 'all' || inputs.platform == 'ascend'
101+
uses: ./.github/workflows/all_tests_common.yml
102+
with:
103+
platform: ascend
97104

98105
metax_tests:
99106
name: MetaX tests
@@ -107,7 +114,7 @@ jobs:
107114
needs:
108115
- check_docker_changes
109116
- cuda_tests
110-
# - ascend_tests
117+
- ascend_tests
111118
- metax_tests
112119
runs-on: ubuntu-latest
113120
if: always()
@@ -124,11 +131,11 @@ jobs:
124131
# failed=true
125132
# fi
126133
127-
# if [ "${{ needs.ascend_tests.result }}" != "success" ] && \
128-
# [ "${{ needs.ascend_tests.result }}" != "skipped" ]; then
129-
# echo "❌ Ascend tests failed"
130-
# failed=true
131-
# fi
134+
if [ "${{ needs.ascend_tests.result }}" != "success" ] && \
135+
[ "${{ needs.ascend_tests.result }}" != "skipped" ]; then
136+
echo "❌ Ascend tests failed"
137+
failed=true
138+
fi
132139
133140
if [ "${{ needs.metax_tests.result }}" != "success" ] && \
134141
[ "${{ needs.metax_tests.result }}" != "skipped" ]; then
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright 2026 FlagOS Contributors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
defaults:
16+
- _self_
17+
- train: 0_6b_ascend_smoke
18+
19+
experiment:
20+
exp_name: 0_6b_ascend_smoke
21+
exp_dir: tests/functional_tests/train/qwen3/test_results/0_6b_ascend_smoke
22+
task:
23+
type: train
24+
backend: megatron
25+
entrypoint: tools/install/ascend/launch_megatron_gpt.py
26+
runner:
27+
ssh_port: null
28+
shell_cmds: null
29+
envs:
30+
HYDRA_FULL_ERROR: 1
31+
FS_PLATFORM: npu
32+
ASCEND_RT_VISIBLE_DEVICES: "0,1"
33+
HCCL_CONNECT_TIMEOUT: "1800"
34+
cmds:
35+
before_start: ''
36+
37+
# No Ascend golden curve is checked until one has been recorded on the target
38+
# runner. The case still fails unless training produces finite loss values.
39+
test:
40+
training_smoke:
41+
enabled: true
42+
metric: "lm loss:"
43+
min_values: 3
44+
45+
action: run
46+
47+
hydra:
48+
run:
49+
dir: ${experiment.exp_dir}/hydra
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Copyright 2026 FlagOS Contributors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
data:
16+
# Keep the accelerator smoke test independent of runner-local .idx/.bin files.
17+
# MockGPTDataset still exercises tokenization, forward/backward, HCCL, and the optimizer.
18+
mock_data: true
19+
tokenizer:
20+
tokenizer_type: QwenTokenizerFS
21+
tokenizer_path: /home/gitlab-runner/tokenizers/qwentokenizer
22+
vocab_size: 151936
23+
make_vocab_size_divisible_by: 64
24+
25+
system:
26+
distributed_backend: hccl
27+
tensor_model_parallel_size: 1
28+
pipeline_model_parallel_size: 1
29+
disable_bias_linear: true
30+
reset_position_ids: true
31+
reset_attention_mask: true
32+
qk_layernorm: true
33+
use_distributed_optimizer: false
34+
precision:
35+
bf16: true
36+
attention_softmax_in_fp32: true
37+
accumulate_allreduce_grads_in_fp32: true
38+
logging:
39+
log_interval: 1
40+
no_log_loss_scale_to_tensorboard: true
41+
checkpoint:
42+
no_save_optim: true
43+
no_save_rng: true
44+
no_load_optim: true
45+
no_load_rng: true
46+
save_interval: 100000
47+
ckpt_format: torch
48+
tensorboard_log_interval: 999999
49+
50+
model:
51+
transformer_impl: transformer_engine
52+
te_fl_prefer: reference
53+
te_fl_per_op: "rmsnorm_fwd=reference"
54+
te_fl_allow_vendors: "npu"
55+
te_fl_deny_vendors: "cuda"
56+
enable_flag_gems: false
57+
no_persist_layer_norm: true
58+
no_masked_softmax_fusion: true
59+
no_check_for_nan_in_loss_and_grad: false
60+
no_gradient_accumulation_fusion: true
61+
num_layers: 28
62+
hidden_size: 1024
63+
ffn_hidden_size: 3072
64+
kv_channels: 128
65+
num_attention_heads: 16
66+
group_query_attention: true
67+
num_query_groups: 8
68+
seq_length: 256
69+
max_position_embeddings: 40960
70+
norm_epsilon: 1e-6
71+
use_rotary_position_embeddings: true
72+
rotary_base: 1000000
73+
no_rope_fusion: true
74+
swiglu: true
75+
normalization: RMSNorm
76+
untie_embeddings_and_output_weights: false
77+
no_position_embedding: true
78+
init_method_std: 0.006
79+
attention_dropout: 0.0
80+
hidden_dropout: 0.0
81+
clip_grad: 1.0
82+
train_iters: 3
83+
eval_iters: 0
84+
eval_interval: 1000
85+
micro_batch_size: 1
86+
global_batch_size: 2
87+
seed: 42
88+
89+
optimizer:
90+
weight_decay: 0.1
91+
adam_beta1: 0.9
92+
adam_beta2: 0.95
93+
lr_scheduler:
94+
lr: 1.0e-5
95+
min_lr: 1.0e-6
96+
lr_warmup_samples: 0
97+
lr_warmup_fraction: 0.01
98+
lr_decay_style: cosine

tests/test_utils/config/platforms/ascend.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ ascend910:
3232
tests:
3333
functional:
3434
# Add or remove test cases per model
35-
train: {}
35+
train:
36+
qwen3: ["0_6b_ascend_smoke"]
3637
hetero_train: {}
3738
benchmark: {}
3839
inference:

tests/test_utils/runners/check_results.py

Lines changed: 62 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,25 @@
2121
import requests
2222
from omegaconf import OmegaConf
2323

24+
ANSI_ESCAPE_RE = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
25+
26+
27+
def _extract_metric_value(line, key):
28+
"""Extract a metric value from a log line, tolerating formatting variations."""
29+
cleaned_line = ANSI_ESCAPE_RE.sub("", line)
30+
pattern = re.compile(
31+
rf"{re.escape(key.rstrip(':'))}\s*:\s*([+-]?\d+\.?\d*(?:[eE][+-]?\d+)?)",
32+
re.IGNORECASE,
33+
)
34+
match = pattern.search(cleaned_line)
35+
if not match:
36+
return None
37+
38+
try:
39+
return float(match.group(1))
40+
except ValueError:
41+
return None
42+
2443

2544
def find_directory(start_path, target_dir_name):
2645
"""Recursively find directory by name."""
@@ -67,25 +86,10 @@ def extract_metrics_from_log(lines, metric_keys=None):
6786
results = {key: {"values": []} for key in metric_keys}
6887

6988
for line in lines:
70-
# Skip non-iteration lines
71-
if "iteration" not in line:
72-
continue
73-
74-
# Split by | and extract key-value pairs
75-
parts = line.split("|")
76-
for part in parts:
77-
part = part.strip()
78-
for key in metric_keys:
79-
# Match "lm loss: 1.161108E+01" format
80-
if part.startswith(key.rstrip(":")):
81-
# Extract the value after the colon
82-
match = re.search(r":\s*([+-]?\d+\.?\d*(?:[eE][+-]?\d+)?)", part)
83-
if match:
84-
try:
85-
value = float(match.group(1))
86-
results[key]["values"].append(value)
87-
except ValueError:
88-
continue
89+
for key in metric_keys:
90+
value = _extract_metric_value(line, key)
91+
if value is not None:
92+
results[key]["values"].append(value)
8993

9094
return results
9195

@@ -129,7 +133,8 @@ def find_latest_stdout_log(start_path):
129133

130134
# Sort attempt directories numerically (attempt_0, attempt_1, ...)
131135
attempt_dirs.sort(
132-
key=lambda x: int(x.split("_")[1]) if x.split("_")[1].isdigit() else -1, reverse=True
136+
key=lambda x: int(x.split("_")[1]) if x.split("_")[1].isdigit() else -1,
137+
reverse=True,
133138
)
134139
latest_attempt = os.path.join(latest_folder, attempt_dirs[0])
135140

@@ -172,6 +177,38 @@ def test_train_equal(path, task, model, case):
172177
with open(result_path, "r", errors="replace") as file:
173178
lines = file.readlines()
174179

180+
# A case may explicitly opt into smoke validation. This is useful when a
181+
# new accelerator is first enabled and no trustworthy, platform-specific
182+
# golden loss curve has been recorded yet. Smoke mode still requires a
183+
# completed run log and finite loss values; it never invents golden data.
184+
config_path = os.path.join(path, task, model, "conf", case + ".yaml")
185+
smoke_config = {}
186+
if os.path.exists(config_path):
187+
case_config = OmegaConf.load(config_path)
188+
training_smoke = case_config.get("test", {}).get("training_smoke", {})
189+
smoke_config = (
190+
OmegaConf.to_container(training_smoke, resolve=True)
191+
if OmegaConf.is_config(training_smoke)
192+
else training_smoke
193+
)
194+
195+
if smoke_config and smoke_config.get("enabled", False):
196+
metric_key = smoke_config.get("metric", "lm loss:")
197+
min_values = int(smoke_config.get("min_values", 1))
198+
result_values = extract_metrics_from_log(lines, [metric_key])[metric_key]["values"]
199+
200+
print("\nTraining smoke validation")
201+
print(f"Metric: {metric_key}")
202+
print(f"Values: {result_values}")
203+
assert len(result_values) >= min_values, (
204+
f"Expected at least {min_values} values for '{metric_key}', "
205+
f"but extracted {len(result_values)}"
206+
)
207+
assert np.all(np.isfinite(result_values)), (
208+
f"Metric '{metric_key}' contains NaN or Inf: {result_values}"
209+
)
210+
return
211+
175212
# Load gold values first to determine which metrics to extract
176213
gold_value_path = os.path.join(path, task, model, "gold_values", case + ".json")
177214
assert os.path.exists(gold_value_path), f"Failed to find gold result JSON at {gold_value_path}"
@@ -333,7 +370,11 @@ def test_inference_equal(path, task, model, case):
333370
print("\nResult checking")
334371
print("Result: ", result_lines)
335372
print("Gold Result: ", gold_value_lines)
336-
print("len(result_lines), (gold_value_lines): ", len(result_lines), len(gold_value_lines))
373+
print(
374+
"len(result_lines), (gold_value_lines): ",
375+
len(result_lines),
376+
len(gold_value_lines),
377+
)
337378

338379
assert len(result_lines) == len(gold_value_lines)
339380

tests/test_utils/runners/setup_training_test_env.sh

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,18 @@ setup_metax_training_env() {
141141

142142
setup_ascend_training_env() {
143143
python -m pip install datasets==4.5.0 omegaconf==2.3.0 diffusers==0.36.0 hydra-core==1.3.2
144-
echo "Ascend CI image is expected to provide platform runtime dependencies"
144+
145+
# Keep the legacy Ascend CI image usable while the newly built training
146+
# image is rolling out. The training stack requires both Megatron-LM-FL
147+
# and TransformerEngine-FL, so validate the full integration before
148+
# skipping either source dependency.
149+
./tools/install/install.sh \
150+
--platform ascend \
151+
--task train \
152+
--pkg-mgr pip \
153+
--no-system --no-dev --no-base --no-task \
154+
--src-deps transformer-engine,megatron-lm \
155+
--retry-count 3
145156

146157
apt-get update
147158
apt-get install -y curl

0 commit comments

Comments
 (0)