Merge branch 'main' into fix/tle-raw-cache-key-optional-mlir

Author: zhzhcookie

.github/new-prs-labeler.yml

@@ -30,7 +30,6 @@ CORE:
           - 'cmake/**'
           - 'include/**'
           - 'lib/**'
-          - 'python/**'
           - 'scripts/**'
           - 'test/**'
           - 'unittest/**'
@@ -45,6 +44,19 @@ CORE:
           - 'setup.py'
           - '.github/actions/**'
           - '.github/workflows/code-format-check.yml'
+      - all-globs-to-any-file:
+          - 'python/**'
+          - '!python/setup_tools/utils/cambricon.py'
+          - '!python/setup_tools/utils/enflame.py'
+          - '!python/setup_tools/utils/hcu.py'
+          - '!python/setup_tools/utils/iluvatar.py'
+          - '!python/setup_tools/utils/metax.py'
+          - '!python/setup_tools/utils/mthreads.py'
+          - '!python/setup_tools/utils/ppu.py'
+          - '!python/setup_tools/utils/rpu.py'
+          - '!python/setup_tools/utils/sunrise.py'
+          - '!python/setup_tools/utils/thrive.py'
+          - '!python/setup_tools/utils/xpu.py'
 
 # ==========================================
 # 2. CI/CD Labels
@@ -128,6 +140,7 @@ cambricon:
           - 'third_party/cambricon/**'
           - '.github/workflows/cambricon*'
           - '.github/workflows/benchmark/cambricon/**'
+          - 'python/setup_tools/utils/cambricon.py'
 
 cpu:
   - changed-files:
@@ -142,40 +155,45 @@ enflame:
           - 'third_party/enflame/**'
           - '.github/workflows/enflame*'
           - '.github/workflows/benchmark/enflame/**'
+          - 'python/setup_tools/utils/enflame.py'
 
 hcu:
   - changed-files:
       - any-glob-to-any-file:
           - 'third_party/hcu/**'
           - '.github/workflows/hcu*'
           - '.github/workflows/benchmark/hcu/**'
+          - 'python/setup_tools/utils/hcu.py'
 
 iluvatar:
   - changed-files:
       - any-glob-to-any-file:
           - 'third_party/iluvatar/**'
           - '.github/workflows/iluvatar*'
           - '.github/workflows/benchmark/iluvatar/**'
+          - 'python/setup_tools/utils/iluvatar.py'
 
 metax:
   - changed-files:
       - any-glob-to-any-file:
           - 'third_party/metax/**'
           - '.github/workflows/metax*'
           - '.github/workflows/benchmark/metax/**'
+          - 'python/setup_tools/utils/metax.py'
 
 mthreads:
   - changed-files:
       - any-glob-to-any-file:
           - 'third_party/mthreads/**'
           - '.github/workflows/mthreads*'
           - '.github/workflows/benchmark/mthreads/**'
+          - 'python/setup_tools/utils/mthreads.py'
 
 nvidia:
   - changed-files:
       - any-glob-to-any-file:
           - 'third_party/nvidia/**'
-          - '.github/workflows/nv*'
+          - '.github/workflows/nvidia*'
           - '.github/workflows/benchmark/nvidia/**'
 
 ppu:
@@ -184,27 +202,31 @@ ppu:
           - 'third_party/ppu/**'
           - '.github/workflows/ppu*'
           - '.github/workflows/benchmark/ppu/**'
+          - 'python/setup_tools/utils/ppu.py'
 
 rpu:
   - changed-files:
       - any-glob-to-any-file:
           - 'third_party/rpu/**'
           - '.github/workflows/rpu*'
           - '.github/workflows/benchmark/rpu/**'
+          - 'python/setup_tools/utils/rpu.py'
 
 sunrise:
   - changed-files:
       - any-glob-to-any-file:
           - 'third_party/sunrise/**'
           - '.github/workflows/sunrise*'
           - '.github/workflows/benchmark/sunrise/**'
+          - 'python/setup_tools/utils/sunrise.py'
 
 thrive:
   - changed-files:
       - any-glob-to-any-file:
           - 'third_party/thrive/**'
           - '.github/workflows/thrive*'
           - '.github/workflows/benchmark/thrive/**'
+          - 'python/setup_tools/utils/thrive.py'
 
 tileir:
   - changed-files:
@@ -233,6 +255,7 @@ xpu:
           - 'third_party/xpu/**'
           - '.github/workflows/xpu*'
           - '.github/workflows/benchmark/xpu/**'
+          - 'python/setup_tools/utils/xpu.py'
 
 # ====================================================================================
 # 6. TLE Labels

.github/workflows/benchmark/mthreads/mthreads3.6-qwen3.6/clear_fuser_process.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Copyright 2025-     FlagOS Contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+set -euo pipefail
+
+if [[ -z "${MUSA_VISIBLE_DEVICES:-}" ]]; then
+  echo "[WARNING] MUSA_VISIBLE_DEVICES is unset; skipping GPU cleanup."
+  exit 0
+fi
+
+IFS=',' read -ra GPU_IDS <<< "$MUSA_VISIBLE_DEVICES"
+DEVICES=()
+for gpu_id in "${GPU_IDS[@]}"; do
+  # Allow whitespace; require non-negative integer IDs.
+  gpu_id="${gpu_id//[[:space:]]/}"
+  if [[ ! "$gpu_id" =~ ^[0-9]+$ ]]; then
+    echo "[FATAL] Invalid MUSA_VISIBLE_DEVICES: $MUSA_VISIBLE_DEVICES" >&2
+    exit 1
+  fi
+  DEVICES+=("/dev/mtgpu.${gpu_id}")
+done
+
+# Find PIDs using the devices.
+mapfile -t PIDS < <(
+  fuser "${DEVICES[@]}" 2>/dev/null \
+  | grep -oE '[0-9]+' \
+  | sort -u
+)
+
+if ((${#PIDS[@]} == 0)); then
+  echo "[INFO] No processes found on ${DEVICES[*]}."
+  exit 0
+fi
+
+echo "[INFO] Terminating processes: ${PIDS[*]}."
+ps -fp "${PIDS[@]}" || true
+kill "${PIDS[@]}" 2>/dev/null || true
+sleep 5
+
+# Force-kill remaining processes.
+mapfile -t REMAINING < <(
+  fuser "${DEVICES[@]}" 2>/dev/null \
+  | grep -oE '[0-9]+' \
+  | sort -u
+)
+
+if ((${#REMAINING[@]} > 0)); then
+  echo "[WARNING] Force-killing processes: ${REMAINING[*]}"
+  kill -9 "${REMAINING[@]}" 2>/dev/null || true
+fi
+
+echo "[INFO] Current usage:"
+fuser -v "${DEVICES[@]}" || true

.github/workflows/benchmark/mthreads/mthreads3.6-qwen3.6/start.sh

@@ -42,6 +42,7 @@ if [[ -f "$PID_FILE" ]]; then
         fi
     fi
 fi
+bash "${SCRIPT_DIR}/clear_fuser_process.sh"
 
 start=$(date +%s)
 

.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/all_perf.py

@@ -41,7 +41,7 @@
 SERVER_HOST = "127.0.0.1"
 SERVER_PORT = 8000
 MODEL_NAME = "qwen36"
-TOKENIZER_PATH = "/root/flagrelease/qwen3.6/Qwen3.6-35B-A3B-nomtp"
+TOKENIZER_PATH = "/root/flagrelease/qwen3.6/Qwen3.6-27B"
 
 # =============================================================================
 # 默认测试参数

.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/perf.sh

@@ -45,14 +45,14 @@ start=$(date +%s)
 nvidia-smi -i "$GPU_DEVICE_LIST" -lgc 1830,1830
 nvidia-smi --query-gpu=index,name,clocks.gr,clocks.mem,utilization.gpu --format=csv
 
-#numactl --cpunodebind=1 --membind=1 \
-#python3 ${SCRIPT_DIR}/all_perf.py --input-len=32768 --output-len=1024 --concurrency=64
+# numactl --cpunodebind=1 --membind=1 \
+# python3 ${SCRIPT_DIR}/all_perf.py --input-len=32768 --output-len=1024 --concurrency=16
 
 numactl --cpunodebind=1 --membind=1 \
-python3 ${SCRIPT_DIR}/all_perf.py --input-len=16384 --output-len=1024 --concurrency=64
+python3 ${SCRIPT_DIR}/all_perf.py --input-len=16384 --output-len=1024 --concurrency=16
 
 numactl --cpunodebind=1 --membind=1 \
-python3 ${SCRIPT_DIR}/all_perf.py --input-len=4096  --output-len=1024 --concurrency=64
+python3 ${SCRIPT_DIR}/all_perf.py --input-len=4096  --output-len=1024 --concurrency=16
 
 nvidia-smi -i "$GPU_DEVICE_LIST" -rgc
 nvidia-smi --query-gpu=index,name,clocks.gr,clocks.mem,utilization.gpu --format=csv

.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/ref-35B.txt

@@ -0,0 +1,46 @@
+H100 * 4
+vllm==0.16.0rc2.dev221+ga5ccc85c8
+torch==2.10.0+cu128
+cuda==13.1
+
+[BaseLine] triton==3.6.0
+# 服务冷启动时间: 1分43秒
+# 输入长度: 32K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 1218.17
+  Total token throughput (tok/s): 40199.48
+  Median TTFT (ms): 17179.61
+  Median TPOT (ms): 33.82
+  Median ITL (ms): 34.69
+# 输入长度: 16K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 1846.06
+  Total token throughput (tok/s): 31383.00
+  Median TTFT (ms): 8332.61
+  Median TPOT (ms): 25.63
+  Median ITL (ms): 31.13
+# 输入长度: 4K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 3129.82
+  Total token throughput (tok/s): 15649.09
+  Median TTFT (ms): 2077.27
+  Median TPOT (ms): 18.29
+  Median ITL (ms): 19.39
+
+[2026/07/11] flag_gems==5.3.0rc2 (2026/05), flagtree===0.6.0 (2026/06/26)
+# 服务冷启动时间: 7分13秒, 服务热启动时间: 2分3秒
+# 输入长度: 32K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 827.43
+  Total token throughput (tok/s): 27305.25
+  Median TTFT (ms): 29391.30
+  Median TPOT (ms): 46.98
+  Median ITL (ms): 32.13
+# 输入长度: 16K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 1355.01
+  Total token throughput (tok/s): 23035.15
+  Median TTFT (ms): 14824.90
+  Median TPOT (ms): 32.08
+  Median ITL (ms): 23.82
+# 输入长度: 4K, 输出长度: 1K, 并发数: 64
+  Output token throughput (tok/s): 2345.05
+  Total token throughput (tok/s): 11725.28
+  Median TTFT (ms): 4025.38
+  Median TPOT (ms): 23.45
+  Median ITL (ms): 23.40