kimik2.5 fp4 b200: v0.26.0 image, TP/DEP sweep, tuning

Author: xinli-sw

benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh

@@ -12,6 +12,20 @@ check_env_vars \
     RANDOM_RANGE_RATIO \
     RESULT_FILENAME
 
+PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
+GMU=0.90
+PREFILL_SCHEDULE_ARGS=()
+if [ "${DP_ATTENTION:-false}" = "true" ]; then
+    PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
+    GMU=0.80
+    PREFILL_SCHEDULE_ARGS=(--prefill-schedule-interval 4)
+fi
+
+EP_ARGS=()
+if [ "${EP_SIZE:-1}" -gt 1 ]; then
+    EP_ARGS=(--enable-expert-parallel)
+fi
+
 if [[ -n "$SLURM_JOB_ID" ]]; then
   echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
 fi
@@ -22,6 +36,7 @@ nvidia-smi
 
 export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
+export VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS=""
 
 SERVER_LOG=/workspace/server.log
 
@@ -41,17 +56,22 @@ export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0
 
 set -x
 vllm serve $MODEL --host 0.0.0.0 --port $PORT \
---tensor-parallel-size=$TP \
---gpu-memory-utilization 0.90 \
+"${PARALLEL_ARGS[@]}" \
+"${EP_ARGS[@]}" \
+"${PREFILL_SCHEDULE_ARGS[@]}" \
+--gpu-memory-utilization "$GMU" \
 --max-model-len $MAX_MODEL_LEN \
 --max-num-seqs $CONC \
 --reasoning-parser kimi_k2 \
 --tool-call-parser kimi_k2 \
 --compilation_config.pass_config.fuse_allreduce_rms true \
 --kv-cache-dtype fp8 \
---max-cudagraph-capture-size 2048 \
+--max-cudagraph-capture-size "$CONC" \
 --max-num-batched-tokens "$((ISL * 2 ))" \
---stream-interval 20 --no-enable-prefix-caching \
+--stream-interval 32 \
+--attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \
+--linear-backend flashinfer_cutedsl \
+--no-enable-prefix-caching \
 --trust-remote-code > $SERVER_LOG 2>&1 &
 
 SERVER_PID=$!

configs/nvidia-master.yaml

@@ -1508,7 +1508,7 @@ kimik2.5-int4-h200-vllm-agentic:
 # Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available.
 
 kimik2.5-fp4-b200-vllm:
-  image: vllm/vllm-openai:v0.22.0
+  image: vllm/vllm-openai:nightly-e2fa28594f7baad142a426b0b6a2cfe2c79201c7
   model: nvidia/Kimi-K2.5-NVFP4
   model-prefix: kimik2.5
   runner: b200
@@ -1520,8 +1520,9 @@ kimik2.5-fp4-b200-vllm:
     - isl: 8192
       osl: 1024
       search-space:
-      - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 }
-      - { tp: 4, ep: 1, conc-start: 1, conc-end: 128 }
+      - { tp: 8, ep: 1, conc-start: 1, conc-end: 512 }
+      - { tp: 4, ep: 1, conc-start: 1, conc-end: 512 }
+      - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 512 }
 
 # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html
 # does not have a B300-specific recipe, so this config reuses the existing

perf-changelog.yaml

@@ -4835,7 +4835,7 @@
 - config-keys:
     - kimik2.5-fp4-b300-vllm-agentic-mtp
   description:
-    - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)."
+    - "Add EAGLE3 speculative-decoding arm for the Kimi K2.5 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)."
     - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2222
 
@@ -4917,7 +4917,7 @@
 - config-keys:
     - kimik2.5-fp4-b300-vllm-agentic-mtp
   description:
-    - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3.1-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)."
+    - "Add EAGLE3 speculative-decoding arm for the Kimi K2.5 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3.1-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)."
     - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2228
 
@@ -5343,7 +5343,7 @@
 - config-keys:
     - kimik2.6-fp4-b200-dynamo-vllm
   description:
-    - "Add Kimi K2.6 NVFP4 B200 Dynamo vLLM disaggregated configuration."
+    - "Add Kimi K2.5 NVFP4 B200 Dynamo vLLM disaggregated configuration."
     - "Add seven checked-in srt-slurm recipes selected one-to-one by the 8K/1K prefill/decode topology and concurrency entries."
     - "Image: vllm/vllm-openai:v0.25.1"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2360
@@ -5355,3 +5355,9 @@
     - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path"
     - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213
+
+- config-keys:
+    - kimik2.5-fp4-b200-vllm
+  description:
+    - "Add Kimi K2.5 NVFP4 B200 vLLM: v0.26.0 image, TP8/TP4 conc to 512, DEP4 arm with --prefill-schedule-interval 4"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2443