Skip to content

Commit ab87d5a

Browse files
zhzhcookiezhzhcookie
committed
[CI][BENCHMARK] Add mtheads3.6 qwen benchmark to ci
1 parent 09e6c1b commit ab87d5a

5 files changed

Lines changed: 118 additions & 8 deletions

File tree

‎.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/all_perf.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
SERVER_HOST = "127.0.0.1"
4242
SERVER_PORT = 8000
4343
MODEL_NAME = "qwen36"
44-
TOKENIZER_PATH = "/data/zhengyang/flagrelease/Qwen3.6-35B-A3B-nomtp"
44+
TOKENIZER_PATH = "/root/flagrelease/qwen3.6/Qwen3.6-35B-A3B-nomtp"
4545

4646
# =============================================================================
4747
# 默认测试参数
@@ -243,6 +243,8 @@ def main():
243243
all_round_metrics = []
244244
for round_num in range(1, TOTAL_ROUNDS + 1):
245245
metrics = run_single_test(round_num, input_len, output_len, concurrency)
246+
if metrics is None:
247+
sys.exit(1)
246248
all_round_metrics.append(metrics)
247249

248250
avg_metrics = compute_average(all_round_metrics)

‎.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/ref.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ cuda==13.1
2525
Median ITL (ms): 19.39
2626

2727
[2026/07/11] flag_gems==5.3.0rc2 (2026/05), flagtree===0.6.0 (2026/06/26)
28-
# 服务冷启动时间: 7分13秒，服务热启动时间: 2分3秒
28+
# 服务冷启动时间: 7分13秒, 服务热启动时间: 2分3秒
2929
# 输入长度: 32K, 输出长度: 1K, 并发数: 64
3030
Output token throughput (tok/s): 827.43
3131
Total token throughput (tok/s): 27305.25

‎.github/workflows/benchmark/nvidia/nv3.6-qwen3.6/test.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
2424

2525
source "${SCRIPT_DIR}/disable_local_proxy.sh"
2626

27-
numactl --cpunodebind=0 --membind=0 \
2827
curl http://localhost:8000/v1/chat/completions \
2928
-H "Content-Type: application/json" \
3029
-d '{

‎.github/workflows/mthreads3.6-qwen-benchmark.yaml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
# Copyright 2025- FlagOS Contributors
2+
#
3+
# Permission is hereby granted, free of charge, to any person obtaining a copy
4+
# of this software and associated documentation files (the "Software"), to deal
5+
# in the Software without restriction, including without limitation the rights
6+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7+
# copies of the Software, and to permit persons to whom the Software is
8+
# furnished to do so, subject to the following conditions:
9+
#
10+
# The above copyright notice and this permission notice shall be included in all
11+
# copies or substantial portions of the Software.
12+
#
13+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19+
# SOFTWARE.
20+
21+
name: Mthreads3.6-Qwen-Benchmark
22+
23+
on:
24+
push:
25+
branches: [ "main", "triton_v3.6.x" ]
26+
pull_request:
27+
branches: [ "main", "triton_v3.6.x" ]
28+
types: [opened, synchronize, reopened, ready_for_review]
29+
30+
permissions:
31+
contents: read
32+
pull-requests: read
33+
34+
concurrency:
35+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
36+
cancel-in-progress: true
37+
38+
jobs:
39+
mthreads36x-qwen:
40+
runs-on: mthreads3.6-qwen
41+
if: ${{ github.repository == 'flagos-ai/flagtree' && github.event.pull_request.draft == false }}
42+
steps:
43+
- name: Setup environment
44+
shell: bash
45+
run: |
46+
source ~/env.sh
47+
env | grep -E '^(http_proxy|https_proxy|all_proxy|no_proxy)=' >> $GITHUB_ENV || true
48+
49+
- name: Smart Checkout
50+
uses: flagos-ai/FlagTree/.github/actions/smart-checkout@main
51+
with:
52+
checkout_version: 'v6'
53+
54+
- name: Check if backend-relevant files changed
55+
id: check_backend
56+
uses: flagos-ai/FlagTree/.github/actions/check-backend-changed@main
57+
with:
58+
backend: mthreads
59+
60+
- name: Cleanup FlagTree Environment
61+
if: ${{ steps.check_backend.outputs.should_skip != 'true' }}
62+
uses: flagos-ai/FlagTree/.github/actions/cleanup-flagtree-env@main
63+
64+
- name: Build FlagTree
65+
if: ${{ steps.check_backend.outputs.should_skip != 'true' }}
66+
shell: bash
67+
run: |
68+
set -x
69+
export FLAGTREE_BACKEND=mthreads
70+
MAX_JOBS=32 python3 -m pip install . --no-build-isolation -v
71+
72+
- name: Clear cache if ClearCache label present
73+
if: steps.check_backend.outputs.should_skip != 'true'
74+
id: clear_cache
75+
uses: flagos-ai/FlagTree/.github/actions/clear-cache-if-labeled@main
76+
with:
77+
github-token: ${{ secrets.GITHUB_TOKEN }}
78+
backend: mthreads-qwen3.6
79+
80+
- name: Start the service
81+
if: ${{ steps.check_backend.outputs.should_skip != 'true' }}
82+
shell: bash
83+
run: |
84+
export BENCH_SCRIPT_DIR="${PWD}/.github/workflows/benchmark/mthreads/mthreads3.6-qwen3.6"
85+
env | grep -E '^(BENCH_SCRIPT_DIR)=' >> $GITHUB_ENV || true
86+
cd ~/flagrelease/qwen3.6
87+
bash ${BENCH_SCRIPT_DIR}/start.sh
88+
89+
- name: Show the baseline
90+
if: ${{ steps.check_backend.outputs.should_skip != 'true' }}
91+
shell: bash
92+
run: |
93+
cd ~/flagrelease/qwen3.6
94+
cat ${BENCH_SCRIPT_DIR}/ref.txt
95+
96+
- name: Run the benchmark
97+
if: ${{ steps.check_backend.outputs.should_skip != 'true' }}
98+
shell: bash
99+
run: |
100+
set -x
101+
cd ~/flagrelease/qwen3.6
102+
bash ${BENCH_SCRIPT_DIR}/perf.sh
103+
104+
- name: Stop the service
105+
if: ${{ steps.check_backend.outputs.should_skip != 'true' }}
106+
shell: bash
107+
run: |
108+
cd ~/flagrelease/qwen3.6
109+
bash ${BENCH_SCRIPT_DIR}/stop.sh

‎.github/workflows/nv3.6-qwen-benchmark.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -77,35 +77,35 @@ jobs:
7777
uses: flagos-ai/FlagTree/.github/actions/clear-cache-if-labeled@main
7878
with:
7979
github-token: ${{ secrets.GITHUB_TOKEN }}
80-
backend: nvidia-qwen
80+
backend: nvidia-qwen3.6
8181

8282
- name: Start the service
8383
if: ${{ steps.check_backend.outputs.should_skip != 'true' }}
8484
shell: bash
8585
run: |
8686
export BENCH_SCRIPT_DIR="${PWD}/.github/workflows/benchmark/nvidia/nv3.6-qwen3.6"
8787
env | grep -E '^(BENCH_SCRIPT_DIR)=' >> $GITHUB_ENV || true
88-
cd ~/flagrelease
88+
cd ~/flagrelease/qwen3.6
8989
bash ${BENCH_SCRIPT_DIR}/start.sh
9090
9191
- name: Show the baseline
9292
if: ${{ steps.check_backend.outputs.should_skip != 'true' }}
9393
shell: bash
9494
run: |
95-
cd ~/flagrelease
95+
cd ~/flagrelease/qwen3.6
9696
cat ${BENCH_SCRIPT_DIR}/ref.txt
9797
9898
- name: Run the benchmark
9999
if: ${{ steps.check_backend.outputs.should_skip != 'true' }}
100100
shell: bash
101101
run: |
102102
set -x
103-
cd ~/flagrelease
103+
cd ~/flagrelease/qwen3.6
104104
bash ${BENCH_SCRIPT_DIR}/perf.sh
105105
106106
- name: Stop the service
107107
if: ${{ steps.check_backend.outputs.should_skip != 'true' }}
108108
shell: bash
109109
run: |
110-
cd ~/flagrelease
110+
cd ~/flagrelease/qwen3.6
111111
bash ${BENCH_SCRIPT_DIR}/stop.sh

0 commit comments

Comments
 (0)