Skip to content

Commit 07cdcfb

Browse files
RohitNagrajOseltamivir
RohitNagraj
and
Oseltamivir
authored
Add qwen3.5-fp4-b200-trt-mtp single-node TensorRT-LLM benchmark (#1894)
* Add qwen3.5-fp4-b200-trt-mtp single-node TensorRT-LLM benchmark Add the qwen3.5-fp4-b200-trt-mtp config (Qwen3.5-397B-A17B-NVFP4, B200, 1k/1k and 8k/1k) with MTP speculative decode across a TP/TEP/DEP parallelism sweep, the qwen3.5_fp4_b200_trt_mtp.sh benchmark script, and a perf-changelog entry. * Update perf-changelog pr-link for #1894 * Enable chat template for qwen3.5 fp4 b200 trt MTP benchmark MTP runs need --use-chat-template on run_benchmark_serving for meaningful acceptance, matching the other single-node MTP scripts. --------- Co-authored-by: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com>
1 parent 9ca3ef9 commit 07cdcfb

3 files changed

Lines changed: 203 additions & 0 deletions

File tree

‎.github/configs/nvidia-master.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12242,6 +12242,32 @@ qwen3.5-fp4-b200-trt:
1224212242
- { tp: 4, ep: 4, dp-attn: true, conc-list: [1024] }
1224312243
- { tp: 8, ep: 8, dp-attn: true, conc-list: [256, 512, 1024] }
1224412244

12245+
qwen3.5-fp4-b200-trt-mtp:
12246+
image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18
12247+
model: nvidia/Qwen3.5-397B-A17B-NVFP4
12248+
model-prefix: qwen3.5
12249+
runner: b200
12250+
precision: fp4
12251+
framework: trt
12252+
multinode: false
12253+
scenarios:
12254+
fixed-seq-len:
12255+
- isl: 1024
12256+
osl: 1024
12257+
search-space:
12258+
- { tp: 2, ep: 1, spec-decoding: "mtp", conc-list: [8] }
12259+
- { tp: 2, ep: 2, spec-decoding: "mtp", conc-list: [4] }
12260+
- { tp: 8, ep: 8, spec-decoding: "mtp", conc-list: [4] }
12261+
- { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [64, 128, 256, 512, 1024] }
12262+
- isl: 8192
12263+
osl: 1024
12264+
search-space:
12265+
- { tp: 2, ep: 1, spec-decoding: "mtp", conc-list: [4] }
12266+
- { tp: 2, ep: 2, spec-decoding: "mtp", conc-list: [8, 16] }
12267+
- { tp: 4, ep: 4, spec-decoding: "mtp", conc-list: [4] }
12268+
- { tp: 8, ep: 8, spec-decoding: "mtp", conc-list: [4] }
12269+
- { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] }
12270+
1224512271
# MiniMax-M3 day-zero (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3).
1224612272
# 427B total / 26B active MoE with MSA sparse attention; MXFP8 checkpoint
1224712273
# (MiniMaxAI/MiniMax-M3-MXFP8, ~444 GB) quantized by NVIDIA — native MX tensor

‎benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200_trt_mtp.sh‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
#!/usr/bin/env bash
2+
3+
source "$(dirname "$0")/../../benchmark_lib.sh"
4+
5+
check_env_vars \
6+
MODEL \
7+
TP \
8+
CONC \
9+
ISL \
10+
OSL \
11+
MAX_MODEL_LEN \
12+
RANDOM_RANGE_RATIO \
13+
RESULT_FILENAME \
14+
DP_ATTENTION \
15+
EP_SIZE
16+
17+
if [[ -n "$SLURM_JOB_ID" ]]; then
18+
echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
19+
fi
20+
21+
echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION"
22+
23+
# MTP (multi-token prediction) speculative decode requires the FlashInfer GDN
24+
# prefill path to be disabled.
25+
export TLLM_USE_FLASHINFER_GDN_PREFILL="0"
26+
27+
if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi
28+
29+
nvidia-smi
30+
31+
SERVER_LOG=/workspace/server.log
32+
EXTRA_CONFIG_FILE="qwen3.5-fp4-trt-mtp.yml"
33+
NUM_NEXTN_PREDICT_LAYERS=3
34+
35+
# Attention-DP layouts run CUTEDSL MoE; everything else runs the TRTLLM backend.
36+
# With MTP the served batch is much smaller than raw concurrency: attention-DP
37+
# runs at CONC/8, everything else at CONC. The KV-cache memory fraction is tuned
38+
# per layout (there is no single derivable rule).
39+
if [[ "$DP_ATTENTION" == "true" ]]; then
40+
MAX_BATCH_SIZE=$(( CONC / 8 ))
41+
MOE_BACKEND="CUTEDSL"
42+
# attention-DP: 0.9 up to conc 512, backed off to 0.8 at conc 1024.
43+
if (( CONC >= 1024 )); then KV_MEMORY_FRACTION=0.8; else KV_MEMORY_FRACTION=0.9; fi
44+
MODE_CONFIG="enable_attention_dp: true
45+
attention_dp_config:
46+
enable_balance: true
47+
batching_wait_iters: 10
48+
timeout_iters: 500"
49+
else
50+
MAX_BATCH_SIZE="$CONC"
51+
MOE_BACKEND="TRTLLM"
52+
# non-attention-DP fraction, tuned per (ISL, TP, EP) layout.
53+
case "${ISL}_tp${TP}_ep${EP_SIZE}" in
54+
1024_tp2_ep1) KV_MEMORY_FRACTION=0.6 ;;
55+
1024_tp2_ep2) KV_MEMORY_FRACTION=0.75 ;;
56+
1024_tp8_ep8) KV_MEMORY_FRACTION=0.8 ;;
57+
8192_tp2_ep1) KV_MEMORY_FRACTION=0.7 ;;
58+
8192_tp2_ep2) KV_MEMORY_FRACTION=0.6 ;;
59+
8192_tp4_ep4) KV_MEMORY_FRACTION=0.75 ;;
60+
8192_tp8_ep8) KV_MEMORY_FRACTION=0.8 ;;
61+
*) KV_MEMORY_FRACTION=0.8 ;;
62+
esac
63+
# Short-context runs hold less in flight, so they wait on a tighter token
64+
# ratio before flushing a batch.
65+
case "$ISL" in
66+
1024) BATCH_WAIT_MAX_TOKENS_RATIO=0.0625 ;;
67+
*) BATCH_WAIT_MAX_TOKENS_RATIO=0.45 ;;
68+
esac
69+
MODE_CONFIG="batch_wait_timeout_iters: 50
70+
batch_wait_max_tokens_ratio: $BATCH_WAIT_MAX_TOKENS_RATIO"
71+
fi
72+
73+
cat > "$EXTRA_CONFIG_FILE" << EOF
74+
backend: pytorch
75+
print_iter_log: true
76+
enable_layerwise_nvtx_marker: false
77+
disable_overlap_scheduler: false
78+
enable_iter_perf_stats: true
79+
enable_chunked_prefill: false
80+
stream_interval: 20
81+
num_postprocess_workers: 4
82+
scheduler_config:
83+
capacity_scheduler_policy: MAX_UTILIZATION
84+
context_chunking_policy: FIRST_COME_FIRST_SERVED
85+
kv_cache_config:
86+
free_gpu_memory_fraction: $KV_MEMORY_FRACTION
87+
enable_block_reuse: false
88+
dtype: fp8
89+
cuda_graph_config:
90+
enable_padding: true
91+
batch_sizes:
92+
- 1
93+
- 2
94+
- 4
95+
- 8
96+
- 16
97+
- 32
98+
- 64
99+
- 128
100+
moe_config:
101+
backend: $MOE_BACKEND
102+
use_low_precision_moe_combine: true
103+
speculative_config:
104+
decoding_type: MTP
105+
num_nextn_predict_layers: $NUM_NEXTN_PREDICT_LAYERS
106+
$MODE_CONFIG
107+
EOF
108+
109+
echo "Generated config file contents:"
110+
cat "$EXTRA_CONFIG_FILE"
111+
112+
MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 ))
113+
114+
case "${ISL}_${OSL}" in
115+
8192_1024) MAX_NUM_TOKENS=32768 ;;
116+
1024_1024) MAX_NUM_TOKENS=16384 ;;
117+
*)
118+
MAX_NUM_TOKENS=$(( ISL + OSL + 256 ))
119+
MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 ))
120+
;;
121+
esac
122+
123+
if [ "${EVAL_ONLY}" = "true" ]; then
124+
setup_eval_context
125+
MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN"
126+
MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN"
127+
fi
128+
129+
# Start GPU monitoring (power, temperature, clocks every second)
130+
start_gpu_monitor
131+
132+
set -x
133+
mpirun -n 1 --oversubscribe --allow-run-as-root \
134+
trtllm-serve "$MODEL" --port="$PORT" \
135+
--trust_remote_code \
136+
--backend=pytorch \
137+
--max_batch_size "$MAX_BATCH_SIZE" \
138+
--max_seq_len="$MAX_MODEL_LEN" \
139+
--max_num_tokens="$MAX_NUM_TOKENS" \
140+
--tp_size="$TP" --ep_size="$EP_SIZE" \
141+
--extra_llm_api_options="$EXTRA_CONFIG_FILE" \
142+
> "$SERVER_LOG" 2>&1 &
143+
144+
SERVER_PID=$!
145+
146+
# Wait for server to be ready
147+
wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
148+
149+
run_benchmark_serving \
150+
--model "$MODEL" \
151+
--port "$PORT" \
152+
--backend openai \
153+
--input-len "$ISL" \
154+
--output-len "$OSL" \
155+
--random-range-ratio "$RANDOM_RANGE_RATIO" \
156+
--num-prompts "$(( CONC * 10 ))" \
157+
--max-concurrency "$CONC" \
158+
--result-filename "$RESULT_FILENAME" \
159+
--result-dir /workspace/ \
160+
--use-chat-template
161+
162+
# After throughput, run evaluation only if RUN_EVAL is true
163+
if [ "${RUN_EVAL}" = "true" ]; then
164+
run_eval --framework lm-eval --port "$PORT"
165+
append_lm_eval_summary
166+
fi
167+
168+
# Stop GPU monitoring
169+
stop_gpu_monitor
170+
set +x

‎perf-changelog.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4123,6 +4123,13 @@
41234123
- "Allocate FlashInfer MNNVL workspace for one-shot TP8 all-reduce during CUDA graph capture"
41244124
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1897
41254125

4126+
- config-keys:
4127+
- qwen3.5-fp4-b200-trt-mtp
4128+
description:
4129+
- "Add Qwen3.5-397B-A17B-NVFP4 B200 single-node TensorRT-LLM benchmark with MTP speculative decode (1k/1k and 8k/1k) across a TP/TEP/DEP parallelism sweep"
4130+
- "Image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18"
4131+
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1894
4132+
41264133
- config-keys:
41274134
- dsr1-fp4-b200-sglang
41284135
description:

0 commit comments

Comments
 (0)