[AgentX]: B200 Kimi K3 DSpark with corrected harness

benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Dynamo serializes the draft config without Kimi K3's config-class alias.
+# Prepare a local metadata view that exposes the equivalent parallel-draft token
+# without changing the downloaded checkpoint or vLLM source.
+python3 - <<'PY'
+import json
+import os
+from pathlib import Path
+
+from huggingface_hub import snapshot_download
+
+repo_id = "Inferact/Kimi-K3-DSpark"
+target = Path("/tmp/Kimi-K3-DSpark")
+snapshot = Path(snapshot_download(repo_id=repo_id))
+target.mkdir(parents=True, exist_ok=True)
+
+for source in snapshot.iterdir():
+    if source.name == "config.json":
+        continue
+    destination = target / source.name
+    if destination.is_symlink():
+        if destination.resolve() == source.resolve():
+            continue
+        destination.unlink()
+    elif destination.exists():
+        raise RuntimeError(f"Refusing to replace non-symlink path: {destination}")
+    destination.symlink_to(source)
+
+config = json.loads((snapshot / "config.json").read_text())
+mask_token_id = config.get("mask_token_id")
+if not isinstance(mask_token_id, int):
+    raise RuntimeError(f"{repo_id} config is missing integer mask_token_id")
+
+pard_token = config.get("pard_token")
+if pard_token not in (None, mask_token_id):
+    raise RuntimeError(
+        f"{repo_id} pard_token={pard_token} disagrees with mask_token_id={mask_token_id}"
+    )
+config["pard_token"] = mask_token_id
+
+temporary = target / "config.json.tmp"
+temporary.write_text(json.dumps(config, indent=2) + "\n")
+os.replace(temporary, target / "config.json")
+print(f"Prepared {repo_id} compatibility view at {target}")
+PY

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml

@@ -0,0 +1,114 @@
+name: "kimik3-vllm-agg-b200-tp16-latency-dspark-agentic"
+
+# Official latency-oriented multi_node_tp profile for two 8xB200 nodes.
+# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless.
+# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp
+model:
+  path: "kimik3"
+  container: "vllm/vllm-openai:kimi-k3"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+  container:
+    image: "vllm/vllm-openai:kimi-k3"
+
+dynamo:
+  install: false
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  interval_seconds: 10
+  max_attempts: 720
+
+resources:
+  gpu_type: "b200"
+  gpus_per_node: 8
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 16
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: vllm
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    # K3's fused tail uses torch symmetric memory, which cannot exchange file
+    # descriptors across this two-node TP group.
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0"
+    # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires.
+    VLLM_ALLREDUCE_USE_FLASHINFER: "0"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 16
+      trust-remote-code: true
+      load-format: fastsafetensors
+      moe-backend: auto
+      # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can
+      # OOM at the official 0.95 setting. Preserve the observed-safe headroom.
+      gpu-memory-utilization: 0.90
+      no-enable-flashinfer-autotune: true
+      # The generic custom communicator also probes MNNVL and stalls before
+      # model load on this pool. Use PyNCCL for the cross-node TP group.
+      disable-custom-all-reduce: true
+      # The default allreduce/RMS fusion independently allocates a FlashInfer
+      # MNNVL workspace during profile_run, bypassing the communicator flags.
+      compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}'
+      max-model-len: 1048576
+      kv-cache-dtype: fp8
+      attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}'
+      enable-prefix-caching: true
+      enable-auto-tool-choice: true
+      tool-call-parser: kimi_k3
+      reasoning-parser: kimi_k3
+      language-model-only: true
+      max-num-seqs: 8
+      max-num-batched-tokens: 8192
+      # Performance runs use the committed K7 golden synthetic AL. The official
+      # recipe's real-verification form uses rejection_sample_method=block.
+      speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}'
+
+sbatch_directives:
+  segment: "1"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
+    AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml

@@ -0,0 +1,115 @@
+name: "kimik3-vllm-agg-b200-tp16-simple-offload-dspark-agentic"
+
+# Official latency-oriented multi_node_tp profile for two 8xB200 nodes.
+# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless.
+# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp
+model:
+  path: "kimik3"
+  container: "vllm/vllm-openai:kimi-k3"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+  container:
+    image: "vllm/vllm-openai:kimi-k3"
+
+dynamo:
+  install: false
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  interval_seconds: 10
+  max_attempts: 720
+
+resources:
+  gpu_type: "b200"
+  gpus_per_node: 8
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 16
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: vllm
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    # K3's fused tail uses torch symmetric memory, which cannot exchange file
+    # descriptors across this two-node TP group.
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0"
+    # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires.
+    VLLM_ALLREDUCE_USE_FLASHINFER: "0"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 16
+      trust-remote-code: true
+      load-format: fastsafetensors
+      moe-backend: auto
+      # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can
+      # OOM at the official 0.95 setting. Preserve the observed-safe headroom.
+      gpu-memory-utilization: 0.90
+      no-enable-flashinfer-autotune: true
+      # The generic custom communicator also probes MNNVL and stalls before
+      # model load on this pool. Use PyNCCL for the cross-node TP group.
+      disable-custom-all-reduce: true
+      # The default allreduce/RMS fusion independently allocates a FlashInfer
+      # MNNVL workspace during profile_run, bypassing the communicator flags.
+      compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}'
+      max-model-len: 1048576
+      kv-cache-dtype: fp8
+      attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}'
+      enable-prefix-caching: true
+      enable-auto-tool-choice: true
+      tool-call-parser: kimi_k3
+      reasoning-parser: kimi_k3
+      language-model-only: true
+      max-num-seqs: 8
+      max-num-batched-tokens: 8192
+      # Performance runs use the committed K7 golden synthetic AL. The official
+      # recipe's real-verification form uses rejection_sample_method=block.
+      speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}'
+      kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}'
+
+sbatch_directives:
+  segment: "1"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
+    AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

configs/nvidia-master.yaml

@@ -9253,6 +9253,87 @@ kimik3-fp4-b200-dynamo-vllm-agentic:
           ep: 1
           dp-attn: false
 
+# DSpark's K3DSparkModel does not implement pipeline parallelism, so Kimi K3
+# DSpark must use the official PP1 TP16 layout across all 16 B200s. Keep plain
+# TP because the latency and capacity objectives do not require expert
+# parallelism. Split the same serving topology into latency, balanced,
+# throughput, and CPU KV-offloaded capacity tiers.
+kimik3-fp4-b200-dynamo-vllm-agentic-dspark:
+  image: vllm/vllm-openai:kimi-k3
+  model: moonshotai/Kimi-K3
+  model-prefix: kimik3
+  runner: cluster:b200-dgxc
+  precision: fp4
+  framework: dynamo-vllm
+  multinode: true
+  disagg: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.61
+      search-space:
+      # Low-latency and high-interactivity points.
+      - spec-decoding: mtp
+        conc-list: [1, 2, 4]
+        prefill:
+          num-worker: 1
+          tp: 16
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 16
+          ep: 1
+          dp-attn: false
+      # Balanced medium-concurrency points.
+      - spec-decoding: mtp
+        conc-list: [8, 12, 16]
+        prefill:
+          num-worker: 1
+          tp: 16
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 16
+          ep: 1
+          dp-attn: false
+      # GPU-resident throughput points around the prior c16-c32 KV cliff.
+      - spec-decoding: mtp
+        conc-list: [20, 24, 28, 32]
+        prefill:
+          num-worker: 1
+          tp: 16
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 16
+          ep: 1
+          dp-attn: false
+      # CPU KV-offloaded capacity points beyond the GPU-resident cliff.
+      - spec-decoding: mtp
+        kv-offloading: dram
+        kv-offload-backend: { name: vllm-simple, version: "13c59a3" }
+        conc-list: [32, 48, 64]
+        prefill:
+          num-worker: 1
+          tp: 16
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 16
+          ep: 1
+          dp-attn: false
+
 qwen3.5-fp8-gb300-dynamo-sglang-mtp:
   image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3
   model: Qwen/Qwen3.5-397B-A17B-FP8

perf-changelog.yaml

@@ -5374,3 +5374,10 @@
     - "Image: lmsysorg/sglang:v0.5.16-cu130"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420
 
+- config-keys:
+    - kimik3-fp4-b200-dynamo-vllm-agentic-dspark
+  description:
+    - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the PP-free TP16 DSpark topology"
+    - "Add a TP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank"
+    - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475