Skip to content

Commit 03ef25d

Browse files
wzhao18wzhao18
authored
Merge branch 'main' into wzhao/m3-b300-update-2
2 parents 4c22b14 + d527fc8 commit 03ef25d

5 files changed

Lines changed: 840 additions & 11 deletions

File tree

‎benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
base:
2+
name: gb200-fp4-glm5-mtp
3+
model:
4+
path: glm-5-fp4
5+
container: "lmsysorg/sglang:v0.5.13.post1-cu130"
6+
precision: fp4
7+
identity:
8+
model:
9+
repo: nvidia/GLM-5-NVFP4
10+
revision: dc54ff55a7e9e71b85db953d8bc22eca894b44c6
11+
frameworks:
12+
dynamo: 1.2.1
13+
sglang: 0.5.13.post1
14+
resources:
15+
gpu_type: gb200
16+
gpus_per_node: 4
17+
frontend:
18+
type: dynamo
19+
dynamo:
20+
version: 1.2.1
21+
backend:
22+
type: sglang
23+
prefill_environment:
24+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
25+
PYTHONUNBUFFERED: '1'
26+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
27+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
28+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
29+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
30+
MC_TE_METRIC: 'true'
31+
MC_FORCE_MNNVL: '1'
32+
NCCL_MNNVL_ENABLE: '1'
33+
NCCL_CUMEM_ENABLE: '1'
34+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
35+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
36+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
37+
decode_environment:
38+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
39+
PYTHONUNBUFFERED: '1'
40+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
41+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
42+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
43+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
44+
MC_TE_METRIC: 'true'
45+
MC_FORCE_MNNVL: '1'
46+
NCCL_MNNVL_ENABLE: '1'
47+
NCCL_CUMEM_ENABLE: '1'
48+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
49+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
50+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
51+
SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512'
52+
SGLANG_MOE_NVFP4_DISPATCH: '1'
53+
sglang_config:
54+
prefill:
55+
served-model-name: GLM-5-FP4
56+
trust-remote-code: true
57+
quantization: modelopt_fp4
58+
kv-cache-dtype: fp8_e4m3
59+
disaggregation-mode: prefill
60+
disaggregation-transfer-backend: nixl
61+
max-running-requests: 136
62+
cuda-graph-max-bs: 136
63+
mem-fraction-static: 0.812
64+
context-length: 9280
65+
chunked-prefill-size: 65536
66+
max-prefill-tokens: 16384
67+
tensor-parallel-size: 4
68+
data-parallel-size: 4
69+
expert-parallel-size: 4
70+
enable-dp-attention: true
71+
enable-dp-lm-head: true
72+
load-balance-method: total_tokens
73+
nsa-decode-backend: trtllm
74+
nsa-prefill-backend: trtllm
75+
moe-runner-backend: flashinfer_trtllm
76+
fp4-gemm-backend: flashinfer_cutlass
77+
enable-flashinfer-allreduce-fusion: true
78+
disable-radix-cache: true
79+
weight-loader-prefetch-checkpoints: true
80+
model-loader-extra-config: '{"enable_multithread_load": true}'
81+
decode:
82+
served-model-name: GLM-5-FP4
83+
trust-remote-code: true
84+
quantization: modelopt_fp4
85+
kv-cache-dtype: fp8_e4m3
86+
disaggregation-mode: decode
87+
disaggregation-transfer-backend: nixl
88+
speculative-algorithm: EAGLE
89+
speculative-num-steps: 2
90+
speculative-eagle-topk: 1
91+
speculative-num-draft-tokens: 3
92+
tensor-parallel-size: 4
93+
data-parallel-size: 1
94+
expert-parallel-size: 1
95+
enable-dp-attention: false
96+
enable-dp-lm-head: false
97+
load-balance-method: round_robin
98+
max-running-requests: 22
99+
cuda-graph-max-bs: 22
100+
chunked-prefill-size: 64
101+
context-length: 9280
102+
nsa-prefill-backend: trtllm
103+
nsa-decode-backend: trtllm
104+
moe-runner-backend: flashinfer_trtllm
105+
fp4-gemm-backend: flashinfer_cutlass
106+
mem-fraction-static: 0.918
107+
skip-tokenizer-init: true
108+
stream-interval: 30
109+
disable-radix-cache: true
110+
enable-flashinfer-allreduce-fusion: true
111+
weight-loader-prefetch-checkpoints: true
112+
model-loader-extra-config: '{"enable_multithread_load": true}'
113+
health_check:
114+
max_attempts: 360
115+
interval_seconds: 10
116+
benchmark:
117+
type: sa-bench
118+
req_rate: inf
119+
# ################# 8k1k #################
120+
zip_override_mtp_8k1k_hightpt:
121+
name: [8k1k-1p1d_dep16, 8k1k-2p1d_dep16, 8k1k-4p1d_dep16, 8k1k-8p1d_dep16]
122+
backend:
123+
decode_environment:
124+
SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1'
125+
sglang_config:
126+
decode:
127+
cuda-graph-max-bs: [91, 196, 451, 1354]
128+
data-parallel-size: 16
129+
deepep-config: /configs/deepep_config.json
130+
deepep-mode: low_latency
131+
enable-dp-attention: true
132+
enable-dp-lm-head: true
133+
enable-flashinfer-allreduce-fusion: false
134+
ep-dispatch-algorithm: static
135+
ep-num-redundant-experts: 0
136+
expert-parallel-size: 16
137+
load-balance-method: null
138+
max-running-requests: [91, 196, 451, 1354]
139+
mem-fraction-static: 0.858
140+
moe-a2a-backend: deepep
141+
moe-dense-tp-size: 1
142+
moe-runner-backend: flashinfer_cutedsl
143+
speculative-moe-a2a-backend: deepep
144+
speculative-moe-runner-backend: deep_gemm
145+
tensor-parallel-size: 16
146+
benchmark:
147+
concurrencies: [[109], [398], [853], [2132]]
148+
isl: 8192
149+
osl: 1024
150+
resources:
151+
decode_nodes: 4
152+
decode_workers: 1
153+
prefill_nodes: [1, 2, 4, 8]
154+
prefill_workers: [1, 2, 4, 8]
155+
zip_override_mtp_8k1k_lowlat:
156+
name: [8k1k-1p4d_tp4, 8k1k-1p4d_tp4-conc16, 8k1k-1p8d_tp4, 8k1k-1p8d_tp4-conc4, 8k1k-1p16d_tp4, 8k1k-1p16d_tp4-conc1]
157+
backend:
158+
sglang_config:
159+
decode:
160+
cuda-graph-max-bs: [22, 15, 9, 4, 4, 1]
161+
max-running-requests: [22, 15, 9, 4, 4, 1]
162+
benchmark:
163+
concurrencies: [[104], [82], [87], [47], [74], [26]]
164+
isl: 8192
165+
osl: 1024
166+
resources:
167+
decode_nodes: [4, 4, 8, 8, 16, 16]
168+
decode_workers: [4, 4, 8, 8, 16, 16]
169+
prefill_nodes: 1
170+
prefill_workers: 1
171+
# ################# 1k1k #################
172+
zip_override_mtp_1k1k_hightpt:
173+
name: [1k1k-1p1d_dep16, 1k1k-1p1d_dep16-conc1024, 1k1k-1p1d_dep16-conc256, 1k1k-1p1d_dep32]
174+
backend:
175+
decode_environment:
176+
SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1'
177+
sglang_config:
178+
decode:
179+
context-length: 2112
180+
cuda-graph-max-bs: [1627, 958, 239, 1122]
181+
data-parallel-size: [16, 16, 16, 32]
182+
deepep-config: /configs/deepep_config.json
183+
deepep-mode: low_latency
184+
enable-dp-attention: true
185+
enable-dp-lm-head: true
186+
enable-flashinfer-allreduce-fusion: false
187+
ep-dispatch-algorithm: static
188+
ep-num-redundant-experts: 0
189+
expert-parallel-size: [16, 16, 16, 32]
190+
max-running-requests: [1627, 958, 239, 1122]
191+
mem-fraction-static: [0.858, 0.858, 0.858, 0.845]
192+
moe-a2a-backend: deepep
193+
moe-dense-tp-size: 1
194+
moe-runner-backend: flashinfer_cutedsl
195+
speculative-moe-a2a-backend: deepep
196+
speculative-moe-runner-backend: deep_gemm
197+
tensor-parallel-size: [16, 16, 16, 32]
198+
prefill:
199+
context-length: 2112
200+
cuda-graph-max-bs: 512
201+
disable-cuda-graph: true
202+
max-running-requests: 512
203+
mem-fraction-static: 0.83
204+
benchmark:
205+
concurrencies: [[1742], [1150], [302], [1263]]
206+
isl: 1024
207+
osl: 1024
208+
resources:
209+
decode_nodes: [4, 4, 4, 8]
210+
decode_workers: 1
211+
prefill_nodes: 1
212+
prefill_workers: 1
213+
zip_override_mtp_1k1k_lowlat:
214+
name: [1k1k-1p16d_tp4-conc1, 1k1k-1p16d_tp4-conc16, 1k1k-1p16d_tp4-conc4]
215+
backend:
216+
sglang_config:
217+
decode:
218+
context-length: 2112
219+
cuda-graph-max-bs: [1, 16, 4]
220+
max-running-requests: [1, 16, 4]
221+
prefill:
222+
context-length: 2112
223+
cuda-graph-max-bs: 512
224+
disable-cuda-graph: true
225+
max-running-requests: 512
226+
mem-fraction-static: 0.83
227+
benchmark:
228+
concurrencies: [[26], [343], [94]]
229+
isl: 1024
230+
osl: 1024
231+
resources:
232+
decode_nodes: 16
233+
decode_workers: 16
234+
prefill_nodes: 1
235+
prefill_workers: 1

0 commit comments

Comments
 (0)