Skip to content

Commit e8183b9

Browse files
zhzhcookiezhzhcookie
authored
Merge branch 'main' into fix/sparse_mla
2 parents 64cf310 + 7d9180c commit e8183b9

5 files changed

Lines changed: 131 additions & 44 deletions

File tree

‎.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/all_perf.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
SERVER_HOST = "127.0.0.1"
4242
SERVER_PORT = 8000
4343
MODEL_NAME = "qwen36"
44-
TOKENIZER_PATH = "/root/flagrelease/qwen3.6/Qwen3.6-35B-A3B-nomtp"
44+
TOKENIZER_PATH = "/root/flagrelease/qwen3.6/Qwen3.6-27B"
4545

4646
# =============================================================================
4747
# 默认测试参数

‎.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/perf.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,14 @@ start=$(date +%s)
4545
nvidia-smi -i "$GPU_DEVICE_LIST" -lgc 1830,1830
4646
nvidia-smi --query-gpu=index,name,clocks.gr,clocks.mem,utilization.gpu --format=csv
4747

48-
#numactl --cpunodebind=1 --membind=1 \
49-
#python3 ${SCRIPT_DIR}/all_perf.py --input-len=32768 --output-len=1024 --concurrency=64
48+
# numactl --cpunodebind=1 --membind=1 \
49+
# python3 ${SCRIPT_DIR}/all_perf.py --input-len=32768 --output-len=1024 --concurrency=16
5050

5151
numactl --cpunodebind=1 --membind=1 \
52-
python3 ${SCRIPT_DIR}/all_perf.py --input-len=16384 --output-len=1024 --concurrency=64
52+
python3 ${SCRIPT_DIR}/all_perf.py --input-len=16384 --output-len=1024 --concurrency=16
5353

5454
numactl --cpunodebind=1 --membind=1 \
55-
python3 ${SCRIPT_DIR}/all_perf.py --input-len=4096 --output-len=1024 --concurrency=64
55+
python3 ${SCRIPT_DIR}/all_perf.py --input-len=4096 --output-len=1024 --concurrency=16
5656

5757
nvidia-smi -i "$GPU_DEVICE_LIST" -rgc
5858
nvidia-smi --query-gpu=index,name,clocks.gr,clocks.mem,utilization.gpu --format=csv

‎.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/ref-35B.txt‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
H100 * 4
2+
vllm==0.16.0rc2.dev221+ga5ccc85c8
3+
torch==2.10.0+cu128
4+
cuda==13.1
5+
6+
[BaseLine] triton==3.6.0
7+
# 服务冷启动时间: 1分43秒
8+
# 输入长度: 32K, 输出长度: 1K, 并发数: 64
9+
Output token throughput (tok/s): 1218.17
10+
Total token throughput (tok/s): 40199.48
11+
Median TTFT (ms): 17179.61
12+
Median TPOT (ms): 33.82
13+
Median ITL (ms): 34.69
14+
# 输入长度: 16K, 输出长度: 1K, 并发数: 64
15+
Output token throughput (tok/s): 1846.06
16+
Total token throughput (tok/s): 31383.00
17+
Median TTFT (ms): 8332.61
18+
Median TPOT (ms): 25.63
19+
Median ITL (ms): 31.13
20+
# 输入长度: 4K, 输出长度: 1K, 并发数: 64
21+
Output token throughput (tok/s): 3129.82
22+
Total token throughput (tok/s): 15649.09
23+
Median TTFT (ms): 2077.27
24+
Median TPOT (ms): 18.29
25+
Median ITL (ms): 19.39
26+
27+
[2026/07/11] flag_gems==5.3.0rc2 (2026/05), flagtree===0.6.0 (2026/06/26)
28+
# 服务冷启动时间: 7分13秒, 服务热启动时间: 2分3秒
29+
# 输入长度: 32K, 输出长度: 1K, 并发数: 64
30+
Output token throughput (tok/s): 827.43
31+
Total token throughput (tok/s): 27305.25
32+
Median TTFT (ms): 29391.30
33+
Median TPOT (ms): 46.98
34+
Median ITL (ms): 32.13
35+
# 输入长度: 16K, 输出长度: 1K, 并发数: 64
36+
Output token throughput (tok/s): 1355.01
37+
Total token throughput (tok/s): 23035.15
38+
Median TTFT (ms): 14824.90
39+
Median TPOT (ms): 32.08
40+
Median ITL (ms): 23.82
41+
# 输入长度: 4K, 输出长度: 1K, 并发数: 64
42+
Output token throughput (tok/s): 2345.05
43+
Total token throughput (tok/s): 11725.28
44+
Median TTFT (ms): 4025.38
45+
Median TPOT (ms): 23.45
46+
Median ITL (ms): 23.40

‎.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/ref.txt‎

Lines changed: 77 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,88 @@
1-
H100 * 4
1+
H100 * 2
22
vllm==0.16.0rc2.dev221+ga5ccc85c8
33
torch==2.10.0+cu128
44
cuda==13.1
55

6-
[BaseLine] triton==3.6.0
7-
# 服务冷启动时间: 1分43秒
6+
[BaseLine] USE_FLAGGEMS=0, triton==3.6.0
7+
# 服务冷启动时间: 2分23秒, 服务热启动时间: 2分4秒
88
# 输入长度: 32K, 输出长度: 1K, 并发数: 64
9-
Output token throughput (tok/s): 1218.17
10-
Total token throughput (tok/s): 40199.48
11-
Median TTFT (ms): 17179.61
12-
Median TPOT (ms): 33.82
13-
Median ITL (ms): 34.69
9+
Output token throughput (tok/s): 319.46
10+
Total token throughput (tok/s): 10542.09
11+
Median TTFT (ms): 15798.41
12+
Median TPOT (ms): 34.20
13+
Median ITL (ms): 22.15
1414
# 输入长度: 16K, 输出长度: 1K, 并发数: 64
15-
Output token throughput (tok/s): 1846.06
16-
Total token throughput (tok/s): 31383.00
17-
Median TTFT (ms): 8332.61
18-
Median TPOT (ms): 25.63
19-
Median ITL (ms): 31.13
15+
Output token throughput (tok/s): 492.25
16+
Total token throughput (tok/s): 8368.25
17+
Median TTFT (ms): 7761.14
18+
Median TPOT (ms): 24.72
19+
Median ITL (ms): 20.41
2020
# 输入长度: 4K, 输出长度: 1K, 并发数: 64
21-
Output token throughput (tok/s): 3129.82
22-
Total token throughput (tok/s): 15649.09
23-
Median TTFT (ms): 2077.27
24-
Median TPOT (ms): 18.29
25-
Median ITL (ms): 19.39
21+
Output token throughput (tok/s): 734.11
22+
Total token throughput (tok/s): 3670.52
23+
Median TTFT (ms): 2117.86
24+
Median TPOT (ms): 19.74
25+
Median ITL (ms): 19.32
2626

27-
[2026/07/11] flag_gems==5.3.0rc2 (2026/05), flagtree===0.6.0 (2026/06/26)
28-
# 服务冷启动时间: 7分13秒, 服务热启动时间: 2分3秒
27+
[2026/08/11] flag_gems==5.3.0rc2 (2026/05), triton==3.6.0
28+
# 服务冷启动时间: 9分13秒, 服务热启动时间: 2分2秒
2929
# 输入长度: 32K, 输出长度: 1K, 并发数: 64
30-
Output token throughput (tok/s): 827.43
31-
Total token throughput (tok/s): 27305.25
32-
Median TTFT (ms): 29391.30
33-
Median TPOT (ms): 46.98
34-
Median ITL (ms): 32.13
30+
Output token throughput (tok/s): 264.37
31+
Total token throughput (tok/s): 8724.08
32+
Median TTFT (ms): 19197.68
33+
Median TPOT (ms): 41.21
34+
Median ITL (ms): 26.92
3535
# 输入长度: 16K, 输出长度: 1K, 并发数: 64
36-
Output token throughput (tok/s): 1355.01
37-
Total token throughput (tok/s): 23035.15
38-
Median TTFT (ms): 14824.90
39-
Median TPOT (ms): 32.08
40-
Median ITL (ms): 23.82
36+
Output token throughput (tok/s): 402.67
37+
Total token throughput (tok/s): 6845.39
38+
Median TTFT (ms): 9507.34
39+
Median TPOT (ms): 30.22
40+
Median ITL (ms): 22.38
4141
# 输入长度: 4K, 输出长度: 1K, 并发数: 64
42-
Successful requests: 64.00
43-
Output token throughput (tok/s): 2345.05
44-
Total token throughput (tok/s): 11725.28
45-
Median TTFT (ms): 4025.38
46-
Median TPOT (ms): 23.45
47-
Median ITL (ms): 23.40
42+
Output token throughput (tok/s): 610.43
43+
Total token throughput (tok/s): 3052.17
44+
Median TTFT (ms): 2613.72
45+
Median TPOT (ms): 23.68
46+
Median ITL (ms): 20.30
47+
48+
[2026/08/11] flag_gems==5.3.0rc2 (2026/05), flagtree===0.6.0 (2026/06/26)
49+
# 服务冷启动时间: 8分20秒, 服务热启动时间: 2分2秒
50+
# 输入长度: 32K, 输出长度: 1K, 并发数: 16
51+
Output token throughput (tok/s): 262.29
52+
Total token throughput (tok/s): 8655.47
53+
Median TTFT (ms): 20106.92
54+
Median TPOT (ms): 40.85
55+
Median ITL (ms): 24.00
56+
# 输入长度: 16K, 输出长度: 1K, 并发数: 16
57+
Output token throughput (tok/s): 398.28
58+
Total token throughput (tok/s): 6770.73
59+
Median TTFT (ms): 9918.11
60+
Median TPOT (ms): 30.24
61+
Median ITL (ms): 22.97
62+
# 输入长度: 4K, 输出长度: 1K, 并发数: 16
63+
Output token throughput (tok/s): 600.23
64+
Total token throughput (tok/s): 3001.12
65+
Median TTFT (ms): 2707.72
66+
Median TPOT (ms): 24.01
67+
Median ITL (ms): 25.02
68+
69+
[2026/08/11] flag_gems==5.3.0rc2 (2026/05), flagtree===0.6.1 (2026/08/07)
70+
# 服务冷启动时间: 8分30秒, 服务热启动时间: 2分2秒
71+
# 输入长度: 32K, 输出长度: 1K, 并发数: 16
72+
Output token throughput (tok/s): 261.04
73+
Total token throughput (tok/s): 8614.48
74+
Median TTFT (ms): 18658.96
75+
Median TPOT (ms): 42.50
76+
Median ITL (ms): 26.28
77+
# 输入长度: 16K, 输出长度: 1K, 并发数: 16
78+
Output token throughput (tok/s): 418.64
79+
Total token throughput (tok/s): 7116.86
80+
Median TTFT (ms): 9186.20
81+
Median TPOT (ms): 28.97
82+
Median ITL (ms): 22.67
83+
# 输入长度: 4K, 输出长度: 1K, 并发数: 16
84+
Output token throughput (tok/s): 624.05
85+
Total token throughput (tok/s): 3120.26
86+
Median TTFT (ms): 2507.47
87+
Median TPOT (ms): 23.17
88+
Median ITL (ms): 21.83

‎.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/start.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,14 @@ export USE_FLAGGEMS=1
4949
export USE_RESHAPE_AND_CACHE_FLASH=1
5050

5151
numactl --cpunodebind=1 --membind=1 \
52-
nohup vllm serve ./Qwen3.6-35B-A3B-nomtp/ \
53-
--tensor-parallel-size 4 \
52+
nohup vllm serve ./Qwen3.6-27B/ \
53+
--tensor-parallel-size 2 \
5454
--port 8000 \
5555
--served-model-name qwen36 \
5656
--mm-encoder-tp-mode data \
5757
--mm-processor-cache-type shm \
5858
--block-size 256 \
59-
--gpu-memory-utilization 0.7 \
59+
--gpu-memory-utilization 0.8 \
6060
--dtype bfloat16 2>&1 >vllm.log &
6161
echo "$!" >pid.txt
6262

0 commit comments

Comments
 (0)