Skip to content

Commit 7db4551

Browse files
xinli-swxinli-sw
committed
redesign GB200 MTP sweep: 1P DEP4 prefill, 1D x TP4/TP8/DEP4/DEP8/DEP16/DEP32 decode, conc 1-1024
1 parent 34fec43 commit 7db4551

7 files changed

Lines changed: 502 additions & 27 deletions

File tree

benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml renamed to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k-eagle3"
1+
name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep16-fp8-8k1k-eagle3"
22

33
model:
44
path: "minimax-m3-mxfp8"
@@ -23,9 +23,9 @@ srun_options:
2323
resources:
2424
gpu_type: "gb200"
2525
gpus_per_node: 4
26-
prefill_nodes: 2
26+
prefill_nodes: 1
2727
decode_nodes: 4
28-
prefill_workers: 2
28+
prefill_workers: 1
2929
decode_workers: 1
3030
gpus_per_prefill: 4
3131
gpus_per_decode: 16
@@ -60,8 +60,8 @@ backend:
6060
prefill:
6161
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
6262
attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}'
63-
tensor-parallel-size: 2
64-
data-parallel-size: 2
63+
tensor-parallel-size: 1
64+
data-parallel-size: 4
6565
data-parallel-rpc-port: 13345
6666
enable-expert-parallel: true
6767
trust-remote-code: true
@@ -72,7 +72,7 @@ backend:
7272
language-model-only: true
7373
kv-cache-dtype: fp8
7474
speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}'
75-
stream-interval: 100
75+
stream-interval: 32
7676
max-cudagraph-capture-size: 2048
7777
max-num-batched-tokens: 16384
7878

@@ -100,7 +100,7 @@ benchmark:
100100
type: "sa-bench"
101101
isl: 8192
102102
osl: 1024
103-
concurrencies: "512x1024"
103+
concurrencies: "1x2x4x8x16x32x64x128x256x512x1024"
104104
req_rate: "inf"
105105
random_range_ratio: 0.8
106106
use_chat_template: true

‎benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep32-fp8-8k1k-eagle3"
2+
3+
model:
4+
path: "minimax-m3-mxfp8"
5+
container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de"
6+
precision: "fp8"
7+
8+
dynamo:
9+
install: true
10+
version: 1.3.0.dev20260710
11+
12+
health_check:
13+
max_attempts: 720
14+
interval_seconds: 10
15+
16+
sbatch_directives:
17+
mem: "0"
18+
cpus-per-task: "72"
19+
20+
srun_options:
21+
mem: "0"
22+
23+
resources:
24+
gpu_type: "gb200"
25+
gpus_per_node: 4
26+
prefill_nodes: 1
27+
decode_nodes: 8
28+
prefill_workers: 1
29+
decode_workers: 1
30+
gpus_per_prefill: 4
31+
gpus_per_decode: 32
32+
33+
frontend:
34+
type: dynamo
35+
enable_multiple_frontends: false
36+
37+
backend:
38+
type: vllm
39+
connector: null
40+
41+
prefill_environment:
42+
VLLM_ENGINE_READY_TIMEOUT_S: "3600"
43+
VLLM_FLOAT32_MATMUL_PRECISION: "high"
44+
VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl"
45+
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
46+
NCCL_CUMEM_ENABLE: "1"
47+
NCCL_MNNVL_ENABLE: "1"
48+
NCCL_NVLS_ENABLE: "1"
49+
50+
decode_environment:
51+
VLLM_ENGINE_READY_TIMEOUT_S: "3600"
52+
VLLM_FLOAT32_MATMUL_PRECISION: "high"
53+
VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl"
54+
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
55+
NCCL_CUMEM_ENABLE: "1"
56+
NCCL_MNNVL_ENABLE: "1"
57+
NCCL_NVLS_ENABLE: "1"
58+
59+
vllm_config:
60+
prefill:
61+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
62+
attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}'
63+
tensor-parallel-size: 1
64+
data-parallel-size: 4
65+
data-parallel-rpc-port: 13345
66+
enable-expert-parallel: true
67+
trust-remote-code: true
68+
no-enable-prefix-caching: true
69+
block-size: 128
70+
gpu-memory-utilization: 0.90
71+
max-model-len: 9472
72+
language-model-only: true
73+
kv-cache-dtype: fp8
74+
speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}'
75+
stream-interval: 32
76+
max-cudagraph-capture-size: 2048
77+
max-num-batched-tokens: 16384
78+
79+
decode:
80+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
81+
attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}'
82+
tensor-parallel-size: 1
83+
data-parallel-size: 32
84+
data-parallel-rpc-port: 13345
85+
enable-expert-parallel: true
86+
trust-remote-code: true
87+
no-enable-prefix-caching: true
88+
block-size: 128
89+
gpu-memory-utilization: 0.90
90+
max-model-len: 9472
91+
language-model-only: true
92+
kv-cache-dtype: fp8
93+
speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}'
94+
stream-interval: 100
95+
max-num-seqs: 1024
96+
max-num-batched-tokens: 16384
97+
max-cudagraph-capture-size: 2048
98+
99+
benchmark:
100+
type: "sa-bench"
101+
isl: 8192
102+
osl: 1024
103+
concurrencies: "1x2x4x8x16x32x64x128x256x512x1024"
104+
req_rate: "inf"
105+
random_range_ratio: 0.8
106+
use_chat_template: true

‎benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep4-fp8-8k1k-eagle3"
2+
3+
model:
4+
path: "minimax-m3-mxfp8"
5+
container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de"
6+
precision: "fp8"
7+
8+
dynamo:
9+
install: true
10+
version: 1.3.0.dev20260710
11+
12+
health_check:
13+
max_attempts: 720
14+
interval_seconds: 10
15+
16+
sbatch_directives:
17+
mem: "0"
18+
cpus-per-task: "72"
19+
20+
srun_options:
21+
mem: "0"
22+
23+
resources:
24+
gpu_type: "gb200"
25+
gpus_per_node: 4
26+
prefill_nodes: 1
27+
decode_nodes: 1
28+
prefill_workers: 1
29+
decode_workers: 1
30+
gpus_per_prefill: 4
31+
gpus_per_decode: 4
32+
33+
frontend:
34+
type: dynamo
35+
enable_multiple_frontends: false
36+
37+
backend:
38+
type: vllm
39+
connector: null
40+
41+
prefill_environment:
42+
VLLM_ENGINE_READY_TIMEOUT_S: "3600"
43+
VLLM_FLOAT32_MATMUL_PRECISION: "high"
44+
VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl"
45+
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
46+
NCCL_CUMEM_ENABLE: "1"
47+
NCCL_MNNVL_ENABLE: "1"
48+
NCCL_NVLS_ENABLE: "1"
49+
50+
decode_environment:
51+
VLLM_ENGINE_READY_TIMEOUT_S: "3600"
52+
VLLM_FLOAT32_MATMUL_PRECISION: "high"
53+
VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl"
54+
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
55+
NCCL_CUMEM_ENABLE: "1"
56+
NCCL_MNNVL_ENABLE: "1"
57+
NCCL_NVLS_ENABLE: "1"
58+
59+
vllm_config:
60+
prefill:
61+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
62+
attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}'
63+
tensor-parallel-size: 1
64+
data-parallel-size: 4
65+
data-parallel-rpc-port: 13345
66+
enable-expert-parallel: true
67+
trust-remote-code: true
68+
no-enable-prefix-caching: true
69+
block-size: 128
70+
gpu-memory-utilization: 0.90
71+
max-model-len: 9472
72+
language-model-only: true
73+
kv-cache-dtype: fp8
74+
speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}'
75+
stream-interval: 32
76+
max-cudagraph-capture-size: 2048
77+
max-num-batched-tokens: 16384
78+
79+
decode:
80+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
81+
attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}'
82+
tensor-parallel-size: 1
83+
data-parallel-size: 4
84+
data-parallel-rpc-port: 13345
85+
enable-expert-parallel: true
86+
trust-remote-code: true
87+
no-enable-prefix-caching: true
88+
block-size: 128
89+
gpu-memory-utilization: 0.90
90+
max-model-len: 9472
91+
language-model-only: true
92+
kv-cache-dtype: fp8
93+
speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}'
94+
stream-interval: 100
95+
max-num-seqs: 1024
96+
max-num-batched-tokens: 16384
97+
max-cudagraph-capture-size: 2048
98+
99+
benchmark:
100+
type: "sa-bench"
101+
isl: 8192
102+
osl: 1024
103+
concurrencies: "1x2x4x8x16x32x64x128x256x512x1024"
104+
req_rate: "inf"
105+
random_range_ratio: 0.8
106+
use_chat_template: true

‎benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep8-fp8-8k1k-eagle3"
2+
3+
model:
4+
path: "minimax-m3-mxfp8"
5+
container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de"
6+
precision: "fp8"
7+
8+
dynamo:
9+
install: true
10+
version: 1.3.0.dev20260710
11+
12+
health_check:
13+
max_attempts: 720
14+
interval_seconds: 10
15+
16+
sbatch_directives:
17+
mem: "0"
18+
cpus-per-task: "72"
19+
20+
srun_options:
21+
mem: "0"
22+
23+
resources:
24+
gpu_type: "gb200"
25+
gpus_per_node: 4
26+
prefill_nodes: 1
27+
decode_nodes: 2
28+
prefill_workers: 1
29+
decode_workers: 1
30+
gpus_per_prefill: 4
31+
gpus_per_decode: 8
32+
33+
frontend:
34+
type: dynamo
35+
enable_multiple_frontends: false
36+
37+
backend:
38+
type: vllm
39+
connector: null
40+
41+
prefill_environment:
42+
VLLM_ENGINE_READY_TIMEOUT_S: "3600"
43+
VLLM_FLOAT32_MATMUL_PRECISION: "high"
44+
VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl"
45+
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
46+
NCCL_CUMEM_ENABLE: "1"
47+
NCCL_MNNVL_ENABLE: "1"
48+
NCCL_NVLS_ENABLE: "1"
49+
50+
decode_environment:
51+
VLLM_ENGINE_READY_TIMEOUT_S: "3600"
52+
VLLM_FLOAT32_MATMUL_PRECISION: "high"
53+
VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl"
54+
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
55+
NCCL_CUMEM_ENABLE: "1"
56+
NCCL_MNNVL_ENABLE: "1"
57+
NCCL_NVLS_ENABLE: "1"
58+
59+
vllm_config:
60+
prefill:
61+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
62+
attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}'
63+
tensor-parallel-size: 1
64+
data-parallel-size: 4
65+
data-parallel-rpc-port: 13345
66+
enable-expert-parallel: true
67+
trust-remote-code: true
68+
no-enable-prefix-caching: true
69+
block-size: 128
70+
gpu-memory-utilization: 0.90
71+
max-model-len: 9472
72+
language-model-only: true
73+
kv-cache-dtype: fp8
74+
speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}'
75+
stream-interval: 32
76+
max-cudagraph-capture-size: 2048
77+
max-num-batched-tokens: 16384
78+
79+
decode:
80+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
81+
attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}'
82+
tensor-parallel-size: 1
83+
data-parallel-size: 8
84+
data-parallel-rpc-port: 13345
85+
enable-expert-parallel: true
86+
trust-remote-code: true
87+
no-enable-prefix-caching: true
88+
block-size: 128
89+
gpu-memory-utilization: 0.90
90+
max-model-len: 9472
91+
language-model-only: true
92+
kv-cache-dtype: fp8
93+
speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}'
94+
stream-interval: 100
95+
max-num-seqs: 1024
96+
max-num-batched-tokens: 16384
97+
max-cudagraph-capture-size: 2048
98+
99+
benchmark:
100+
type: "sa-bench"
101+
isl: 8192
102+
osl: 1024
103+
concurrencies: "1x2x4x8x16x32x64x128x256x512x1024"
104+
req_rate: "inf"
105+
random_range_ratio: 0.8
106+
use_chat_template: true

0 commit comments

Comments
 (0)