Merge branch 'main' into feature/tle_fix_extract_tile

Author: zhzhcookie

.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/all_perf.py

@@ -41,7 +41,7 @@
 SERVER_HOST = "127.0.0.1"
 SERVER_PORT = 8000
 MODEL_NAME = "qwen36"
-TOKENIZER_PATH = "/root/flagrelease/qwen3.6/Qwen3.6-35B-A3B-nomtp"
+TOKENIZER_PATH = "/root/flagrelease/qwen3.6/Qwen3.6-27B"
 
 # =============================================================================
 # 默认测试参数

.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/perf.sh

@@ -45,14 +45,14 @@ start=$(date +%s)
 nvidia-smi -i "$GPU_DEVICE_LIST" -lgc 1830,1830
 nvidia-smi --query-gpu=index,name,clocks.gr,clocks.mem,utilization.gpu --format=csv
 
-#numactl --cpunodebind=1 --membind=1 \
-#python3 ${SCRIPT_DIR}/all_perf.py --input-len=32768 --output-len=1024 --concurrency=64
+# numactl --cpunodebind=1 --membind=1 \
+# python3 ${SCRIPT_DIR}/all_perf.py --input-len=32768 --output-len=1024 --concurrency=16
 
 numactl --cpunodebind=1 --membind=1 \
-python3 ${SCRIPT_DIR}/all_perf.py --input-len=16384 --output-len=1024 --concurrency=64
+python3 ${SCRIPT_DIR}/all_perf.py --input-len=16384 --output-len=1024 --concurrency=16
 
 numactl --cpunodebind=1 --membind=1 \
-python3 ${SCRIPT_DIR}/all_perf.py --input-len=4096  --output-len=1024 --concurrency=64
+python3 ${SCRIPT_DIR}/all_perf.py --input-len=4096  --output-len=1024 --concurrency=16
 
 nvidia-smi -i "$GPU_DEVICE_LIST" -rgc
 nvidia-smi --query-gpu=index,name,clocks.gr,clocks.mem,utilization.gpu --format=csv

.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/ref-35B.txt

@@ -0,0 +1,46 @@
+H100 * 4
+vllm==0.16.0rc2.dev221+ga5ccc85c8
+torch==2.10.0+cu128
+cuda==13.1
+
+[BaseLine] triton==3.6.0
+# 服务冷启动时间: 1分43秒
+# 输入长度: 32K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 1218.17
+  Total token throughput (tok/s): 40199.48
+  Median TTFT (ms): 17179.61
+  Median TPOT (ms): 33.82
+  Median ITL (ms): 34.69
+# 输入长度: 16K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 1846.06
+  Total token throughput (tok/s): 31383.00
+  Median TTFT (ms): 8332.61
+  Median TPOT (ms): 25.63
+  Median ITL (ms): 31.13
+# 输入长度: 4K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 3129.82
+  Total token throughput (tok/s): 15649.09
+  Median TTFT (ms): 2077.27
+  Median TPOT (ms): 18.29
+  Median ITL (ms): 19.39
+
+[2026/07/11] flag_gems==5.3.0rc2 (2026/05), flagtree===0.6.0 (2026/06/26)
+# 服务冷启动时间: 7分13秒, 服务热启动时间: 2分3秒
+# 输入长度: 32K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 827.43
+  Total token throughput (tok/s): 27305.25
+  Median TTFT (ms): 29391.30
+  Median TPOT (ms): 46.98
+  Median ITL (ms): 32.13
+# 输入长度: 16K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 1355.01
+  Total token throughput (tok/s): 23035.15
+  Median TTFT (ms): 14824.90
+  Median TPOT (ms): 32.08
+  Median ITL (ms): 23.82
+# 输入长度: 4K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 2345.05
+  Total token throughput (tok/s): 11725.28
+  Median TTFT (ms): 4025.38
+  Median TPOT (ms): 23.45
+  Median ITL (ms): 23.40

.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/ref.txt

@@ -1,47 +1,88 @@
-H100 * 4
+H100 * 2
 vllm==0.16.0rc2.dev221+ga5ccc85c8
 torch==2.10.0+cu128
 cuda==13.1
 
-[BaseLine] triton==3.6.0
-# 服务冷启动时间: 1分43秒
+[BaseLine] USE_FLAGGEMS=0, triton==3.6.0
+# 服务冷启动时间: 2分23秒, 服务热启动时间: 2分4秒
 # 输入长度: 32K, 输出长度: 1K, 并发数: 64
-  Output token throughput (tok/s): 1218.17
-  Total token throughput (tok/s): 40199.48
-  Median TTFT (ms): 17179.61
-  Median TPOT (ms): 33.82
-  Median ITL (ms): 34.69
+  Output token throughput (tok/s): 319.46
+  Total token throughput (tok/s): 10542.09
+  Median TTFT (ms): 15798.41
+  Median TPOT (ms): 34.20
+  Median ITL (ms): 22.15
 # 输入长度: 16K, 输出长度: 1K, 并发数: 64
-  Output token throughput (tok/s): 1846.06
-  Total token throughput (tok/s): 31383.00
-  Median TTFT (ms): 8332.61
-  Median TPOT (ms): 25.63
-  Median ITL (ms): 31.13
+  Output token throughput (tok/s): 492.25
+  Total token throughput (tok/s): 8368.25
+  Median TTFT (ms): 7761.14
+  Median TPOT (ms): 24.72
+  Median ITL (ms): 20.41
 # 输入长度: 4K, 输出长度: 1K, 并发数: 64
-  Output token throughput (tok/s): 3129.82
-  Total token throughput (tok/s): 15649.09
-  Median TTFT (ms): 2077.27
-  Median TPOT (ms): 18.29
-  Median ITL (ms): 19.39
+  Output token throughput (tok/s): 734.11
+  Total token throughput (tok/s): 3670.52
+  Median TTFT (ms): 2117.86
+  Median TPOT (ms): 19.74
+  Median ITL (ms): 19.32
 
-[2026/07/11] flag_gems==5.3.0rc2 (2026/05), flagtree===0.6.0 (2026/06/26)
-# 服务冷启动时间: 7分13秒, 服务热启动时间: 2分3秒
+[2026/08/11] flag_gems==5.3.0rc2 (2026/05), triton==3.6.0
+# 服务冷启动时间: 9分13秒, 服务热启动时间: 2分2秒
 # 输入长度: 32K, 输出长度: 1K, 并发数: 64
-  Output token throughput (tok/s): 827.43
-  Total token throughput (tok/s): 27305.25
-  Median TTFT (ms): 29391.30
-  Median TPOT (ms): 46.98
-  Median ITL (ms): 32.13
+Output token throughput (tok/s): 264.37
+  Total token throughput (tok/s): 8724.08
+  Median TTFT (ms): 19197.68
+  Median TPOT (ms): 41.21
+  Median ITL (ms): 26.92
 # 输入长度: 16K, 输出长度: 1K, 并发数: 64
-  Output token throughput (tok/s): 1355.01
-  Total token throughput (tok/s): 23035.15
-  Median TTFT (ms): 14824.90
-  Median TPOT (ms): 32.08
-  Median ITL (ms): 23.82
+  Output token throughput (tok/s): 402.67
+  Total token throughput (tok/s): 6845.39
+  Median TTFT (ms): 9507.34
+  Median TPOT (ms): 30.22
+  Median ITL (ms): 22.38
 # 输入长度: 4K, 输出长度: 1K, 并发数: 64
-  Successful requests: 64.00
-  Output token throughput (tok/s): 2345.05
-  Total token throughput (tok/s): 11725.28
-  Median TTFT (ms): 4025.38
-  Median TPOT (ms): 23.45
-  Median ITL (ms): 23.40
+  Output token throughput (tok/s): 610.43
+  Total token throughput (tok/s): 3052.17
+  Median TTFT (ms): 2613.72
+  Median TPOT (ms): 23.68
+  Median ITL (ms): 20.30
+
+[2026/08/11] flag_gems==5.3.0rc2 (2026/05), flagtree===0.6.0 (2026/06/26)
+# 服务冷启动时间: 8分20秒, 服务热启动时间: 2分2秒
+# 输入长度: 32K, 输出长度: 1K, 并发数: 16
+  Output token throughput (tok/s): 262.29
+  Total token throughput (tok/s): 8655.47
+  Median TTFT (ms): 20106.92
+  Median TPOT (ms): 40.85
+  Median ITL (ms): 24.00
+# 输入长度: 16K, 输出长度: 1K, 并发数: 16
+  Output token throughput (tok/s): 398.28
+  Total token throughput (tok/s): 6770.73
+  Median TTFT (ms): 9918.11
+  Median TPOT (ms): 30.24
+  Median ITL (ms): 22.97
+# 输入长度: 4K, 输出长度: 1K, 并发数: 16
+  Output token throughput (tok/s): 600.23
+  Total token throughput (tok/s): 3001.12
+  Median TTFT (ms): 2707.72
+  Median TPOT (ms): 24.01
+  Median ITL (ms): 25.02
+
+[2026/08/11] flag_gems==5.3.0rc2 (2026/05), flagtree===0.6.1 (2026/08/07)
+# 服务冷启动时间: 8分30秒, 服务热启动时间: 2分2秒
+# 输入长度: 32K, 输出长度: 1K, 并发数: 16
+  Output token throughput (tok/s): 261.04
+  Total token throughput (tok/s): 8614.48
+  Median TTFT (ms): 18658.96
+  Median TPOT (ms): 42.50
+  Median ITL (ms): 26.28
+# 输入长度: 16K, 输出长度: 1K, 并发数: 16
+  Output token throughput (tok/s): 418.64
+  Total token throughput (tok/s): 7116.86
+  Median TTFT (ms): 9186.20
+  Median TPOT (ms): 28.97
+  Median ITL (ms): 22.67
+# 输入长度: 4K, 输出长度: 1K, 并发数: 16
+  Output token throughput (tok/s): 624.05
+  Total token throughput (tok/s): 3120.26
+  Median TTFT (ms): 2507.47
+  Median TPOT (ms): 23.17
+  Median ITL (ms): 21.83

.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/start.sh

@@ -49,14 +49,14 @@ export USE_FLAGGEMS=1
 export USE_RESHAPE_AND_CACHE_FLASH=1
 
 numactl --cpunodebind=1 --membind=1 \
-nohup vllm serve ./Qwen3.6-35B-A3B-nomtp/  \
-    --tensor-parallel-size 4 \
+nohup vllm serve ./Qwen3.6-27B/  \
+    --tensor-parallel-size 2 \
     --port 8000  \
     --served-model-name qwen36 \
     --mm-encoder-tp-mode data \
     --mm-processor-cache-type shm \
     --block-size 256  \
-    --gpu-memory-utilization 0.7 \
+    --gpu-memory-utilization 0.8 \
     --dtype bfloat16 2>&1 >vllm.log &
 echo "$!" >pid.txt