From b4fd077b70279324617f6c180dc11393f529ba06 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:34:29 -0400 Subject: [PATCH 01/25] feat: add Kimi-K3 MXFP4 B200 aggregated TP8xPP2 Dynamo-vLLM agentic recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (no expert parallelism) for the agentic-coding trace replay. Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1, fastsafetensors load format, kimi_k3 tool-call/reasoning parsers. Model pre-staged at /lustre/fsw/models/Kimi-K3; launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay, and the agentic cache default_mounts used by the GB200/GB300 agentic paths. 中文：新增 Kimi-K3 MXFP4 B200 聚合式 TP8xPP2 Dynamo-vLLM 智能体编码基准测试配方 （2 节点 / 16 GPU，纯 TP，不启用专家并行(EP)）。使用专用 bring-up 镜像 vllm/vllm-openai:kimi-k3（VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1、 fastsafetensors 加载格式、kimi_k3 工具调用/推理解析器）。模型已预置于 /lustre/fsw/models/Kimi-K3；启动器 launch_b200-dgxc.sh 增加 kimik3/fp4 模型路径映射、智能体配方覆盖及智能体缓存挂载。 Co-Authored-By: Claude Fable 5 --- .../agentic/agg-b200-tp8pp2-agentic.yaml | 109 ++++++++++++++++++ configs/nvidia-master.yaml | 43 +++++++ perf-changelog.yaml | 11 ++ runners/launch_b200-dgxc.sh | 28 +++++ 4 files changed, 191 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml new file mode 100644 index 0000000000..2ded17c2fd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -0,0 +1,109 @@ +name: "kimik3-vllm-agg-b200-tp8pp2-agentic" + +# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). +# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not +# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the +# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is +# deliberately off, so the 896 routed experts are TP-sharded inside each +# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. +# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in +# the kimi-k3 bring-up image. +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0.dev20260618" + +dynamo: + install: true + wheel: "1.3.0.dev20260618" + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_SERVER_DEV_MODE: "1" + # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window + # generous, and let one long AgentX request hold a PP stage beyond vLLM's + # 300-second model-execution default. + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.95 + no-enable-flashinfer-autotune: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + # Full native 1M context: the AgentX trace carries multi-hundred-k-token + # requests, and K3's KDA layers keep per-token KV small (only the 24 + # gated-MLA layers hold cache). Cap prefill chunks so a single long + # request cannot OOM a pipeline stage; let vLLM pick max-num-seqs. + max-model-len: 1048576 + max-num-batched-tokens: 8192 + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + aiperf_server_metrics: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep the aggregate worker in the multinode result schema so ingestion + # uses the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 36ae816809..bf1cb716d5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8325,3 +8325,46 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: tp: 16 ep: 16 dp-attn: true + +# Kimi-K3 MXFP4 B200 aggregated vLLM via Dynamo (TP8 x PP2, 2 nodes / 16 +# GPUs), agentic bring-up. The native MXFP4 checkpoint (2.8T total params, +# ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense +# and PP2 splits layers. Plain TP (NOT TEP): ep 1, no expert parallelism — +# the 896 routed experts are TP-sharded within each pipeline stage. Node +# count = tp*pp/gpus_per_node = 8*2/8 = 2. Aggregated (prefill num-worker 1 + +# decode num-worker 0, RECIPES.md section 5) — the single worker serves both +# phases, so no P/D KV transfer. Dedicated kimi-k3 vLLM bring-up image with +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 and the kimi_k3 tool-call/reasoning +# parsers. +# Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +kimik3-fp4-b200-dynamo-vllm-agentic: + image: vllm/vllm-openai:kimi-k3 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b200-dgxc + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [8, 16, 32] + prefill: + num-worker: 1 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts the 16 GPUs only once. + decode: + num-worker: 0 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0524d0c0c9..0a4e40426b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5116,3 +5116,14 @@ description: - "Bump image from lmsysorg/sglang:v0.5.14-rocm720-mi35x to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2349 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic + description: + - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" + - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code" + - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 8/16/32" + - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay into the srt-slurm-nv clone, and the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" + - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index a276644575..e166dbb595 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -72,6 +72,10 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then # NVFP4 checkpoint, pre-staged on the b200-dgxc scratch tree. export MODEL_PATH="/scratch/fsw/models/MiniMax-M3-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m3-nvfp4" +elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then + # Native MXFP4 checkpoint, pre-staged on the SRE-managed Lustre tree. + export MODEL_PATH="/lustre/fsw/models/Kimi-K3" + export SRT_SLURM_MODEL_PREFIX="kimik3" else echo "Unsupported model prefix/precision: $MODEL_PREFIX/$PRECISION" echo "Available models under /lustre/fsw/models:" @@ -107,6 +111,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 + # Overlay InferenceX-staged agentic recipes onto the clone (cp -rT so + # an upstream stub directory is merged rather than nested). + if [[ $MODEL_PREFIX == "kimik3" ]]; then + mkdir -p recipes/vllm/kimi-k3/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic + fi elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 @@ -207,6 +218,22 @@ if [[ "$IS_MULTINODE" == "true" ]]; then export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" + # Agentic runs bind-mount two persistent caches into every worker + # container (Lustre, shared across nodes): aiperf's content-addressed + # dataset mmap cache and the HF hub cache holding the trace dataset + # download. The container-side paths are referenced by the agentic + # recipes' benchmark.env (AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache, + # HF_HUB_CACHE=/hf_hub_cache). + DEFAULT_MOUNTS_BLOCK="" + if [[ "$IS_AGENTIC" == "1" ]]; then + HF_HUB_CACHE_HOST_PATH="/lustre/fsw/gharunners/hf-hub-cache" + mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" + chmod 777 "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true + DEFAULT_MOUNTS_BLOCK="default_mounts: + ${AIPERF_MMAP_CACHE_HOST_PATH}: /aiperf_mmap_cache + ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache" + fi + # Create srtslurm.yaml for srtctl (used by both frameworks) SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." @@ -234,6 +261,7 @@ containers: "${IMAGE}": "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" use_exclusive_sbatch_directive: true +${DEFAULT_MOUNTS_BLOCK} EOF echo "Generated srtslurm.yaml:" From 4dbbdc8a5908a6d6177674be46a226b721be80f3 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:35:35 -0400 Subject: [PATCH 02/25] docs: link PR #2355 in changelog entry and MODELS rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：在更新日志条目与 MODELS 表格行中补充 PR #2355 链接。 Co-Authored-By: Claude Fable 5 --- MODELS.md | 2 +- MODELS_zh.md | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODELS.md b/MODELS.md index 3fc822420e..31cb9df252 100644 --- a/MODELS.md +++ b/MODELS.md @@ -18,7 +18,7 @@ This document tracks every model benchmarked by InferenceX-e2e: when it was adde | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | TBD | Agentic coding | | -| Kimi-K3 | `kimik3` | 2026-07-27 | Agentic coding | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | Agentic coding | | | GLM-5.2 | `glm5.2` | 2026-07-18 ([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | Agentic coding | | | MiniMax-M3 | `minimaxm3` | 2026-06-12 ([#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24 ([#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | diff --git a/MODELS_zh.md b/MODELS_zh.md index 177a4b4ba3..e71067c13b 100644 --- a/MODELS_zh.md +++ b/MODELS_zh.md @@ -18,7 +18,7 @@ | 模型架构类别 | 前缀 | 加入日期 | 启用场景 | 已弃用场景 | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | 待定 | 智能体编码 | | -| Kimi-K3 | `kimik3` | 2026-07-27 | 智能体编码 | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | 智能体编码 | | | GLM-5.2 | `glm5.2` | 2026-07-18（[#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)） | 智能体编码 | | | MiniMax-M3 | `minimaxm3` | 2026-06-12（[#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24（[#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0a4e40426b..5a7800bd48 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5126,4 +5126,4 @@ - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 8/16/32" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay into the srt-slurm-nv clone, and the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 From e8d42a73cdc7126fae0a3c1a30b8e54132dff1f1 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:48:01 -0400 Subject: [PATCH 03/25] fix: pin agentic srt-slurm to NVIDIA v1.0.36, dynamo 1.2.1, conc-8 smoke test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cquil11/srt-slurm-nv cam/sa-submission-q2-2026 fork rejected the recipe (benchmark.aiperf_server_metrics: Unknown field). Switch the b200-dgxc agentic clone to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341), drop the aiperf_server_metrics field, pin dynamo wheel/router to 1.2.1 (the combination validated with v1.0.36), and reduce the bring-up to a single conc-8 smoke test. 中文：cquil11/srt-slurm-nv 分支的 srtctl 校验拒绝了配方字段 benchmark.aiperf_server_metrics（Unknown field）。将 b200-dgxc 智能体路径改用 上游 NVIDIA/srt-slurm v1.0.36（已在 #2302/#2341 验证），移除该字段，dynamo wheel/router 固定为 1.2.1，并将 bring-up 缩减为单并发（conc 8）冒烟测试。 Co-Authored-By: Claude Fable 5 --- .../kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 5 ++--- configs/nvidia-master.yaml | 6 ++++-- perf-changelog.yaml | 5 +++-- runners/launch_b200-dgxc.sh | 13 ++++++++++--- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 2ded17c2fd..cbee00a2b5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -20,11 +20,11 @@ identity: container: image: "vllm/vllm-openai:kimi-k3" frameworks: - dynamo: "1.3.0.dev20260618" + dynamo: "1.2.1" dynamo: + wheel: "1.2.1" install: true - wheel: "1.3.0.dev20260618" slurm: time_limit: "8:00:00" @@ -94,7 +94,6 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index bf1cb716d5..82c69638c8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8344,14 +8344,16 @@ kimik3-fp4-b200-dynamo-vllm-agentic: runner: cluster:b200-dgxc precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260618" } + router: { name: dynamo-router, version: "1.2.1" } multinode: true disagg: false scenarios: agentic-coding: - search-space: + # Single-concurrency smoke test for the bring-up; widen the conc curve + # once the topology is proven green. - spec-decoding: none - conc-list: [8, 16, 32] + conc-list: [8] prefill: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5a7800bd48..b8b11fa508 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5123,7 +5123,8 @@ - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code" - - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 8/16/32" - - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay into the srt-slurm-nv clone, and the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" + - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" + - "Dynamo 1.2.1 (ai-dynamo wheel + dynamo-router), the version validated with srt-slurm v1.0.36 in #2341's GB300 aggregate AgentX recipes" + - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index e166dbb595..1dd4e19d14 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -109,14 +109,21 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # TODO(CJQ): make first class upon srt-slurm upstream refactor if [[ "$IS_AGENTIC" == "1" ]]; then - git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" + # Agentic recipes use NVIDIA/srt-slurm v1.0.36, the upstream version + # validated in InferenceX PR #2302/#2341 for the vLLM agentic path + # (BenchmarkType.CUSTOM + benchmark.command/env, DynamoConfig.wheel, + # srun_options propagation, per-node DP, matching Dynamo health + # counts). Keep it pinned so sweeps are reproducible. Note the older + # cquil11/srt-slurm-nv cam/sa-submission-q2-2026 fork previously + # cloned here rejects newer recipe schema fields. + git clone --branch v1.0.36 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 # Overlay InferenceX-staged agentic recipes onto the clone (cp -rT so # an upstream stub directory is merged rather than nested). if [[ $MODEL_PREFIX == "kimik3" ]]; then - mkdir -p recipes/vllm/kimi-k3/agentic + mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic + recipes/vllm/kimi-k3/agentic || exit 1 fi elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" From c1e2a568101d37556da96f9fd80c262f30d2dabb Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:54:05 -0400 Subject: [PATCH 04/25] fix: drop OpenAI-frontend tool-choice flags from dynamo-vllm worker args MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dynamo-vllm worker entrypoint rejected --enable-auto-tool-choice --tool-call-parser kimi_k3 (unrecognized arguments; different arg parser than vllm serve). Chat parsing happens at the dynamo frontend — same convention as the DSv4 GB300 agentic recipes. Keep --reasoning-parser kimi_k3 (accepted by the worker). Also drop the explicit max-model-len and let vLLM derive the native 1M window from the model config, mirroring the agentic recipe convention. 中文：dynamo-vllm worker 入口不接受 --enable-auto-tool-choice 与 --tool-call-parser kimi_k3（unrecognized arguments，与 vllm serve 的参数解析器 不同），聊天解析由 dynamo 前端处理，与 DSv4 GB300 智能体配方约定一致；保留 worker 可接受的 --reasoning-parser kimi_k3。同时移除显式 max-model-len， 由 vLLM 从模型配置推导原生 1M 上下文窗口。 Co-Authored-By: Claude Fable 5 --- .../agentic/agg-b200-tp8pp2-agentic.yaml | 19 ++++++++++++------- perf-changelog.yaml | 4 ++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index cbee00a2b5..dcb1ec7afa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -76,14 +76,19 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 + # Note: --enable-auto-tool-choice / --tool-call-parser are NOT set on + # the worker. The dynamo-vllm worker entrypoint doesn't accept them + # (different arg parser than `vllm serve`; first sweep attempt died with + # "unrecognized arguments"). Chat parsing happens at the dynamo + # frontend, not at the worker — same convention as the DSv4 GB300 + # agentic recipes. --reasoning-parser IS accepted by the worker. reasoning-parser: kimi_k3 - # Full native 1M context: the AgentX trace carries multi-hundred-k-token - # requests, and K3's KDA layers keep per-token KV small (only the 24 - # gated-MLA layers hold cache). Cap prefill chunks so a single long - # request cannot OOM a pipeline stage; let vLLM pick max-num-seqs. - max-model-len: 1048576 + # No explicit max-model-len: let vLLM derive the native 1M window from + # the model config (agentic trajectories blow past any small cap, and + # K3's KDA layers keep per-token KV small — only the 24 gated-MLA + # layers hold cache). Prefix caching stays on (default) for trajectory + # reuse. Cap prefill chunks so a single long request cannot OOM a + # pipeline stage; let vLLM pick max-num-seqs. max-num-batched-tokens: 8192 sbatch_directives: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b8b11fa508..61b86f3db3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,8 +5122,8 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code" - - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --reasoning-parser kimi_k3, --trust-remote-code. --enable-auto-tool-choice/--tool-call-parser are NOT set: the dynamo-vllm worker entrypoint rejects them (unrecognized arguments; different arg parser than vllm serve) — chat parsing happens at the dynamo frontend, same convention as the DSv4 GB300 agentic recipes" + - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "Dynamo 1.2.1 (ai-dynamo wheel + dynamo-router), the version validated with srt-slurm v1.0.36 in #2341's GB300 aggregate AgentX recipes" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" From ef35fd1f030fd821fa31436d9a26f425e77aaeec Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:20:42 -0400 Subject: [PATCH 05/25] fix: try dynamo wheel 1.2.0.dev20260426 for Kimi-K3 frontend tokenizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third sweep attempt: the engine loaded and served (TP8xPP2 healthy in ~14 min), but dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type 'kimi_k3' (supported: kimi, kimi_k2, kimi_k25, deepseek_v3), so the model never registered and all chat completions returned 404, aborting the AgentX warmup. Switch to the 1.2.0.dev20260426 wheel used by the DSv4 GB300/B200 Dynamo-vLLM recipes. Upstream published v1.4.0-kimi-k3-dev.1 (2026-07-27) as the day-zero K3 build if this wheel also lacks support. 中文：第三次扫描中引擎已成功加载并提供服务（TP8xPP2 约 14 分钟就绪），但 dynamo 1.2.1 的 rust 前端分词器不支持 Kimi-K3 的 tiktoken model_type 'kimi_k3'，模型未能注册，所有请求返回 404，AgentX 预热中止。改用 DSv4 GB300/B200 Dynamo-vLLM 配方所用的 1.2.0.dev20260426 wheel；如仍不支持， 上游已于 2026-07-27 发布 day-zero 构建 v1.4.0-kimi-k3-dev.1。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index dcb1ec7afa..d36158073f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -20,11 +20,11 @@ identity: container: image: "vllm/vllm-openai:kimi-k3" frameworks: - dynamo: "1.2.1" + dynamo: "1.2.0.dev20260426" dynamo: - wheel: "1.2.1" install: true + wheel: "1.2.0.dev20260426" slurm: time_limit: "8:00:00" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 82c69638c8..3dd5261b27 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8344,7 +8344,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic: runner: cluster:b200-dgxc precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.1" } + router: { name: dynamo-router, version: "1.2.0.dev20260426" } multinode: true disagg: false scenarios: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 61b86f3db3..2001751894 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5124,7 +5124,7 @@ - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --reasoning-parser kimi_k3, --trust-remote-code. --enable-auto-tool-choice/--tool-call-parser are NOT set: the dynamo-vllm worker entrypoint rejects them (unrecognized arguments; different arg parser than vllm serve) — chat parsing happens at the dynamo frontend, same convention as the DSv4 GB300 agentic recipes" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - - "Dynamo 1.2.1 (ai-dynamo wheel + dynamo-router), the version validated with srt-slurm v1.0.36 in #2341's GB300 aggregate AgentX recipes" + - "Dynamo 1.2.0.dev20260426 (ai-dynamo wheel + dynamo-router), the wheel used by the DSv4 GB300/B200 Dynamo-vLLM recipes. Note: dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registers and all chat completions 404 (third sweep attempt); upstream published v1.4.0-kimi-k3-dev.1 on 2026-07-27 as the day-zero K3 build if this wheel also lacks kimi_k3 tiktoken support" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 From be6c56e6a1a6717d3fc018570e21471f496c7138 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:26:38 -0400 Subject: [PATCH 06/25] fix: pin dynamo to day-zero Kimi-K3 commit, restore kimi_k3 parser flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin dynamo to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ("feat: Added support for Kimi-K3", tag v1.4.0-kimi-k3-dev.1) via srt-slurm's hash-cached source install: it adds the kimi_k3 tiktoken tokenizer to the rust frontend (dynamo <=1.2.1 404s every request because the model never registers) and accepts the kimi_k3 tool-call/reasoning parser worker args, so restore --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3. 中文：将 dynamo 固定到 day-zero Kimi-K3 提交 ba83080（"feat: Added support for Kimi-K3"，标签 v1.4.0-kimi-k3-dev.1），通过 srt-slurm 的哈希缓存源码 安装：该提交为 rust 前端新增 kimi_k3 tiktoken 分词器（dynamo <=1.2.1 因模型 无法注册而全部返回 404），worker 亦支持 kimi_k3 解析器参数，故恢复 --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3。 Co-Authored-By: Claude Fable 5 --- .../agentic/agg-b200-tp8pp2-agentic.yaml | 21 ++++++++++++------- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 4 ++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index d36158073f..22362d730c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -20,11 +20,16 @@ identity: container: image: "vllm/vllm-openai:kimi-k3" frameworks: - dynamo: "1.2.0.dev20260426" + dynamo: "1.4.0-kimi-k3-dev.1" dynamo: install: true - wheel: "1.2.0.dev20260426" + # Day-zero Kimi-K3 dynamo ("feat: Added support for Kimi-K3", tag + # v1.4.0-kimi-k3-dev.1 == this commit): adds the kimi_k3 tiktoken tokenizer + # to the rust frontend (dynamo <=1.2.1 only knows kimi/kimi_k2/kimi_k25/ + # deepseek_v3, so the model never registers and every request 404s) and the + # kimi_k3 tool-call/reasoning parser worker args. + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" slurm: time_limit: "8:00:00" @@ -76,12 +81,12 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - # Note: --enable-auto-tool-choice / --tool-call-parser are NOT set on - # the worker. The dynamo-vllm worker entrypoint doesn't accept them - # (different arg parser than `vllm serve`; first sweep attempt died with - # "unrecognized arguments"). Chat parsing happens at the dynamo - # frontend, not at the worker — same convention as the DSv4 GB300 - # agentic recipes. --reasoning-parser IS accepted by the worker. + # kimi_k3 parsers: the day-zero K3 dynamo build pinned above accepts + # these on the worker. Dynamo <=1.2.1 rejected --enable-auto-tool-choice + # / --tool-call-parser as unrecognized arguments (different arg parser + # than `vllm serve`). + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 # No explicit max-model-len: let vLLM derive the native 1M window from # the model config (agentic trajectories blow past any small cap, and diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3dd5261b27..89bf2fad2b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8344,7 +8344,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic: runner: cluster:b200-dgxc precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.0.dev20260426" } + router: { name: dynamo-router, version: "1.4.0-kimi-k3-dev.1" } multinode: true disagg: false scenarios: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2001751894..c687f6da00 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,9 +5122,9 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --reasoning-parser kimi_k3, --trust-remote-code. --enable-auto-tool-choice/--tool-call-parser are NOT set: the dynamo-vllm worker entrypoint rejects them (unrecognized arguments; different arg parser than vllm serve) — chat parsing happens at the dynamo frontend, same convention as the DSv4 GB300 agentic recipes" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code (the kimi_k3 parser worker args require the day-zero K3 dynamo below; dynamo <=1.2.1 rejects the tool-choice flags as unrecognized arguments)" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - - "Dynamo 1.2.0.dev20260426 (ai-dynamo wheel + dynamo-router), the wheel used by the DSv4 GB300/B200 Dynamo-vLLM recipes. Note: dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registers and all chat completions 404 (third sweep attempt); upstream published v1.4.0-kimi-k3-dev.1 on 2026-07-27 as the day-zero K3 build if this wheel also lacks kimi_k3 tiktoken support" + - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 From c6917e640fb34327cd684189f5741ae475ada231 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:32:17 -0400 Subject: [PATCH 07/25] fix: use dynamo namespaced --dyn-* kimi_k3 parser args on the worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the vLLM OpenAI-frontend spellings (--enable-auto-tool-choice / --tool-call-parser) with dynamo's namespaced worker args: --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3. 中文：将 vLLM OpenAI 前端风格参数（--enable-auto-tool-choice / --tool-call-parser）替换为 dynamo 命名空间的 worker 参数： --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3。 Co-Authored-By: Claude Fable 5 --- .../kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 13 +++++++------ perf-changelog.yaml | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 22362d730c..ab4c5314b0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -81,13 +81,14 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - # kimi_k3 parsers: the day-zero K3 dynamo build pinned above accepts - # these on the worker. Dynamo <=1.2.1 rejected --enable-auto-tool-choice - # / --tool-call-parser as unrecognized arguments (different arg parser - # than `vllm serve`). - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 + # kimi_k3 parsers via dynamo's namespaced worker args (--dyn-*), which + # the day-zero K3 dynamo build pinned above provides. The vLLM + # OpenAI-frontend spellings (--enable-auto-tool-choice / + # --tool-call-parser) are rejected by the dynamo worker entrypoint as + # unrecognized arguments (different arg parser than `vllm serve`). + dyn-tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 # No explicit max-model-len: let vLLM derive the native 1M window from # the model config (agentic trajectories blow past any small cap, and # K3's KDA layers keep per-token KV small — only the 24 gated-MLA diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c687f6da00..43fd4df391 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,7 +5122,7 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code (the kimi_k3 parser worker args require the day-zero K3 dynamo below; dynamo <=1.2.1 rejects the tool-choice flags as unrecognized arguments)" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --trust-remote-code, and kimi_k3 parsers via dynamo's namespaced worker args --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3 (the vLLM OpenAI-frontend spellings --enable-auto-tool-choice/--tool-call-parser are rejected by the dynamo worker entrypoint as unrecognized arguments)" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" From a26853aa844da823eaddb96fd2138038dc7cc666 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:59:49 -0400 Subject: [PATCH 08/25] fix: disable aiperf conv-aware routing (session_control 400-rejected) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifth sweep attempt: the day-zero dynamo registered the kimi_k3 tiktoken tokenizer and the engine served, but all warmup requests got 400 — aiperf's conv-aware routing emits nvext.session_control, a removed POC field this dynamo build rejects (schema moved to router/routing_constraints/ agent_hints). Opt out via AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, matching the GB300 aggregate AgentX recipes; a single aggregate worker has no P/D routing to bind anyway. 中文：第五次扫描中 day-zero dynamo 已成功注册 kimi_k3 tiktoken 分词器并正常 服务，但全部预热请求返回 400——aiperf 的会话感知路由会发送 nvext.session_control（已被移除的 POC 字段，schema 已迁移至 router/routing_constraints/agent_hints）。通过 AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0 关闭，与 GB300 聚合式 AgentX 配方 一致；单聚合 worker 本无需 P/D 路由绑定。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 7 ++++++- perf-changelog.yaml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index ab4c5314b0..ba6da59153 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -113,7 +113,12 @@ benchmark: # Keep the aggregate worker in the multinode result schema so ingestion # uses the zero decode-worker count instead of duplicating TP into P and D. IS_MULTINODE: "true" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + # aiperf's conv-aware routing emits nvext.session_control, a removed POC + # field this dynamo build 400-rejects at warmup (schema moved to + # router/routing_constraints/agent_hints). Same opt-out as the GB300 + # aggregate AgentX recipes — and with a single aggregate worker there is + # no P/D routing to bind anyway. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 43fd4df391..b94f744c83 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5126,5 +5126,6 @@ - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" + - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 From f61eafbe4bf13c25828715f2c6488c549708f77a Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:52:22 -0400 Subject: [PATCH 09/25] fix: patch kimi-k3 image mamba_hybrid index_fill_ dtype via setup_script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sixth sweep attempt (both A and C variants): warmup requests 500 then the model 503s — the image's first decode step crashes in the KDA hybrid-state postprocess (mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index; torch requires an int64 index but the runner passes the int32 idx_mapping). Ship an in-container patch through srt-slurm's setup_script hook (same pattern as configs/patches/ vllm_numa_bind_hash_fix.py): coerce the index with .long(), idempotent, refuses to run if the image layout changed. 中文：第六次扫描（A、C 两个变体一致）：预热请求先 500、随后模型 503——镜像 首个解码步在 KDA 混合状态后处理中崩溃（mamba_hybrid.py postprocess_state， IndexError: index_fill_() 需要 int64 索引，但 runner 传入 int32 idx_mapping）。 通过 srt-slurm 的 setup_script 钩子在容器内打补丁：将索引用 .long() 转换， 幂等，且镜像布局变化时拒绝执行。 Co-Authored-By: Claude Fable 5 --- .../configs/kimi-k3-container-deps.sh | 39 +++++++++++++++++++ .../agentic/agg-b200-tp8pp2-agentic.yaml | 5 +++ perf-changelog.yaml | 1 + runners/launch_b200-dgxc.sh | 4 ++ 4 files changed, 49 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh new file mode 100644 index 0000000000..abddf4d9c3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Setup script for the Kimi-K3 vLLM bring-up image (vllm/vllm-openai:kimi-k3). +# srt-slurm runs this in every worker container before dynamo install and +# worker startup (recipe field: setup_script). + +set -euo pipefail + +# The image's first decode step crashes in the KDA hybrid-state postprocess: +# vllm/v1/worker/gpu/model_states/mamba_hybrid.py, postprocess_state: +# IndexError: index_fill_(): Expected dtype int64 for index. +# torch's index_fill_ requires an int64 index tensor, but the runner passes +# the int32 idx_mapping (hit by moonshotai/Kimi-K3 agentic bring-up, first +# decode step, engine v0.1.dev19262+gb6bbf29dd). Coerce the index to int64. +# Idempotent: exits 0 if the patch is already applied. +python3 - <<'PY' +import pathlib +import re + +import vllm.v1.worker.gpu.model_states.mamba_hybrid as mh + +path = pathlib.Path(mh.__file__) +src = path.read_text() +if "idx_mapping.long()" in src: + print(f"mamba_hybrid index_fill_ patch already applied: {path}") + raise SystemExit(0) + +new, n = re.subn( + r"index_fill_\(\s*0,\s*idx_mapping,", + "index_fill_(0, idx_mapping.long(),", + src, +) +if n != 1: + raise SystemExit( + f"expected exactly one index_fill_(0, idx_mapping, ...) call in " + f"{path}, found {n} — image layout changed, refusing to patch" + ) +path.write_text(new) +print(f"Patched mamba_hybrid index_fill_ index dtype: {path}") +PY diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index ba6da59153..6570cc1fd9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -31,6 +31,11 @@ dynamo: # kimi_k3 tool-call/reasoning parser worker args. hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +# Patches the image's mamba_hybrid postprocess_state: torch index_fill_ +# requires an int64 index but the runner passes the int32 idx_mapping, +# crashing the first decode step (IndexError: Expected dtype int64 for index). +setup_script: kimi-k3-container-deps.sh + slurm: time_limit: "8:00:00" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b94f744c83..6c39e4be50 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5127,5 +5127,6 @@ - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" + - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 1dd4e19d14..bff9b48af0 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -124,6 +124,10 @@ if [[ "$IS_MULTINODE" == "true" ]]; then mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ recipes/vllm/kimi-k3/agentic || exit 1 + # In-container vLLM patch for the kimi-k3 image, referenced by the + # recipes' setup_script field (srt-slurm mounts configs/ at /configs). + cp "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh" \ + configs/kimi-k3-container-deps.sh || exit 1 fi elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" From 8fd319c06802dcf344080f517ee5fc457d71e570 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:07:41 -0400 Subject: [PATCH 10/25] =?UTF-8?q?feat:=20agentic=20experiment=20D=20?= =?UTF-8?q?=E2=80=94=20direct=20vllm=20serve=20via=20srt-slurm=20PR=20#278?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serve Kimi-K3 directly with vllm serve (srt-slurm PR #278 frontend.type: vllm, branch kylliang/direct-aggregate-vllm): no dynamo frontend/worker/ router, which removes the dynamo tokenizer/schema gaps entirely, and the OpenAI-frontend flags --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 become legitimate. PR #278 validates single-node only, so ship patches/srt-slurm-pr278-direct-vllm-multinode.patch extending it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology. Keeps the mamba_hybrid index-dtype container patch (engine bug is frontend-agnostic). 中文：智能体实验变体 D——通过 srt-slurm PR #278（frontend.type: vllm）直接以 vllm serve 提供服务：去除 dynamo 前端/worker/router，从根本上规避 dynamo 的 分词器与 schema 兼容问题，OpenAI 前端参数 --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 因此可用。PR #278 仅 支持单节点，故新增补丁将其扩展为 vLLM 原生多节点 serve（--master-addr/ --nnodes/--node-rank，非主节点 headless），以运行 2 节点 TP8xPP2 拓扑。保留 mamba_hybrid 索引类型容器补丁（引擎缺陷与前端无关）。 Co-Authored-By: Claude Fable 5 --- ...rt-slurm-pr278-direct-vllm-multinode.patch | 62 +++++++++++++++++++ .../agentic/agg-b200-tp8pp2-agentic.yaml | 32 +++++----- configs/nvidia-master.yaml | 4 +- perf-changelog.yaml | 4 +- runners/launch_b200-dgxc.sh | 20 +++--- 5 files changed, 91 insertions(+), 31 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/patches/srt-slurm-pr278-direct-vllm-multinode.patch diff --git a/benchmarks/multi_node/srt-slurm-recipes/patches/srt-slurm-pr278-direct-vllm-multinode.patch b/benchmarks/multi_node/srt-slurm-recipes/patches/srt-slurm-pr278-direct-vllm-multinode.patch new file mode 100644 index 0000000000..9408bc7764 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/patches/srt-slurm-pr278-direct-vllm-multinode.patch @@ -0,0 +1,62 @@ +diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py +index 74f673b..377606a 100644 +--- a/src/srtctl/backends/vllm.py ++++ b/src/srtctl/backends/vllm.py +@@ -716,25 +716,31 @@ class VLLMProtocol: + if frontend_type == "vllm": + if mode != "agg": + raise ValueError("frontend.type: vllm supports aggregate vLLM jobs only") +- if is_multi_node: +- raise ValueError("frontend.type: vllm currently supports single-node aggregate jobs only") + + config.pop("host", None) + config.pop("port", None) + config.pop("connector", None) + config.setdefault("served-model-name", served_model_name) + +- cmd.extend( +- [ +- "vllm", +- "serve", +- model_arg, +- "--host", +- "0.0.0.0", +- "--port", +- str(runtime.frontend_port), +- ] +- ) ++ node_rank = endpoint_nodes.index(process.node) ++ cmd.extend(["vllm", "serve", model_arg]) ++ if node_rank == 0: ++ cmd.extend(["--host", "0.0.0.0", "--port", str(runtime.frontend_port)]) ++ if is_multi_node: ++ # vLLM-native multi-node serve (torchrun-style): the leader owns ++ # the OpenAI server; other node ranks run headless engine workers. ++ cmd.extend( ++ [ ++ "--master-addr", ++ leader_ip, ++ "--nnodes", ++ str(len(endpoint_nodes)), ++ "--node-rank", ++ str(node_rank), ++ ] ++ ) ++ if node_rank > 0: ++ cmd.append("--headless") + if not self.set_cuda_visible_devices: + device_ids = ",".join(str(i) for i in sorted(process.gpu_indices)) + if device_ids: +diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py +index 1263ddc..0ef7ae4 100644 +--- a/src/srtctl/core/schema.py ++++ b/src/srtctl/core/schema.py +@@ -1587,8 +1587,6 @@ class SrtConfig: + raise ValidationError("frontend.type: vllm supports aggregate jobs only, not disaggregated layouts") + if self.resources.num_agg < 1: + raise ValidationError("frontend.type: vllm requires resources.agg_workers >= 1") +- if (self.resources.agg_nodes or 1) != 1: +- raise ValidationError("frontend.type: vllm currently supports single-node aggregate jobs only") + + def _validate_het_jobs(self): + """When ``resources.het_jobs`` is set to True, enforce supported shape. diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 6570cc1fd9..2e313acf70 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -19,17 +19,12 @@ identity: repo: "moonshotai/Kimi-K3" container: image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.4.0-kimi-k3-dev.1" +# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the +# InferenceX multinode patch): `vllm serve` owns the OpenAI port itself, so +# no Dynamo frontend/worker is involved and no dynamo install is needed. dynamo: - install: true - # Day-zero Kimi-K3 dynamo ("feat: Added support for Kimi-K3", tag - # v1.4.0-kimi-k3-dev.1 == this commit): adds the kimi_k3 tiktoken tokenizer - # to the rust frontend (dynamo <=1.2.1 only knows kimi/kimi_k2/kimi_k25/ - # deepseek_v3, so the model never registers and every request 404s) and the - # kimi_k3 tool-call/reasoning parser worker args. - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: false # Patches the image's mamba_hybrid postprocess_state: torch index_fill_ # requires an int64 index but the runner passes the int32 idx_mapping, @@ -55,7 +50,11 @@ infra: nats_max_payload_mb: 32 frontend: - type: dynamo + # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns + # the public port; rank-1 runs a headless engine worker (vLLM-native + # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, enabled by + # patches/srt-slurm-pr278-direct-vllm-multinode.patch). + type: vllm enable_multiple_frontends: false backend: @@ -86,14 +85,13 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - # kimi_k3 parsers via dynamo's namespaced worker args (--dyn-*), which - # the day-zero K3 dynamo build pinned above provides. The vLLM - # OpenAI-frontend spellings (--enable-auto-tool-choice / - # --tool-call-parser) are rejected by the dynamo worker entrypoint as - # unrecognized arguments (different arg parser than `vllm serve`). - dyn-tool-call-parser: kimi_k3 + # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — + # legitimate here because this recipe serves directly with vllm serve + # (frontend.type: vllm), not through the dynamo worker entrypoint that + # rejects them. + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 # No explicit max-model-len: let vLLM derive the native 1M window from # the model config (agentic trajectories blow past any small cap, and # K3's KDA layers keep per-token KV small — only the 24 gated-MLA diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 89bf2fad2b..014afa7314 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8343,8 +8343,10 @@ kimik3-fp4-b200-dynamo-vllm-agentic: model-prefix: kimik3 runner: cluster:b200-dgxc precision: fp4 + # framework stays dynamo-vllm for launcher routing, but this variant serves + # DIRECTLY with vllm serve (srt-slurm PR #278 frontend.type: vllm + the + # InferenceX multinode patch) — no dynamo frontend/worker/router involved. framework: dynamo-vllm - router: { name: dynamo-router, version: "1.4.0-kimi-k3-dev.1" } multinode: true disagg: false scenarios: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6c39e4be50..03d735f567 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,9 +5122,9 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --trust-remote-code, and kimi_k3 parsers via dynamo's namespaced worker args --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3 (the vLLM OpenAI-frontend spellings --enable-auto-tool-choice/--tool-call-parser are rejected by the dynamo worker entrypoint as unrecognized arguments)" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --trust-remote-code, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3 (agentic experiment Variant D: the OpenAI-frontend parser flags are legitimate here because serving is DIRECT vllm serve, not the dynamo worker entrypoint that rejects them)" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" + - "DIRECT vLLM serving via srt-slurm PR #278 (kylliang/direct-aggregate-vllm, frontend.type: vllm): vllm serve owns the OpenAI port itself, removing the dynamo frontend/worker/router entirely (dynamo install: false) and with it the kimi_k3 tiktoken tokenizer gap that 404'd every request on dynamo <=1.2.1. PR #278 validates single-node only, so patches/srt-slurm-pr278-direct-vllm-multinode.patch extends it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index bff9b48af0..fe8e518126 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -108,18 +108,16 @@ if [[ "$IS_MULTINODE" == "true" ]]; then fi # TODO(CJQ): make first class upon srt-slurm upstream refactor - if [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic recipes use NVIDIA/srt-slurm v1.0.36, the upstream version - # validated in InferenceX PR #2302/#2341 for the vLLM agentic path - # (BenchmarkType.CUSTOM + benchmark.command/env, DynamoConfig.wheel, - # srun_options propagation, per-node DP, matching Dynamo health - # counts). Keep it pinned so sweeps are reproducible. Note the older - # cquil11/srt-slurm-nv cam/sa-submission-q2-2026 fork previously - # cloned here rejects newer recipe schema fields. - git clone --branch v1.0.36 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then + # Direct-vLLM agentic experiment (Variant D): srt-slurm PR #278 + # (kylliang/direct-aggregate-vllm) adds frontend.type: vllm — `vllm + # serve` owns the OpenAI port itself, no Dynamo layer. The InferenceX + # patch below extends it from single-node to vLLM-native multi-node + # serve (--master-addr/--nnodes/--node-rank + headless non-leader + # ranks) so the 2-node TP8xPP2 topology can run. + git clone --branch kylliang/direct-aggregate-vllm --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 - # Overlay InferenceX-staged agentic recipes onto the clone (cp -rT so - # an upstream stub directory is merged rather than nested). + git apply "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/patches/srt-slurm-pr278-direct-vllm-multinode.patch" || exit 1 if [[ $MODEL_PREFIX == "kimik3" ]]; then mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ From c0ace4d01c6911ea3f341b5c8c9a7e39ea1b9b47 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:08:31 -0400 Subject: [PATCH 11/25] docs: point changelog and MODELS rows at experiment PR #2359 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将更新日志条目与 MODELS 表格行链接指向实验 PR #2359。 Co-Authored-By: Claude Fable 5 --- MODELS.md | 2 +- MODELS_zh.md | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODELS.md b/MODELS.md index 31cb9df252..b68b49759a 100644 --- a/MODELS.md +++ b/MODELS.md @@ -18,7 +18,7 @@ This document tracks every model benchmarked by InferenceX-e2e: when it was adde | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | TBD | Agentic coding | | -| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | Agentic coding | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2359](https://github.com/SemiAnalysisAI/InferenceX/pull/2359)) | Agentic coding | | | GLM-5.2 | `glm5.2` | 2026-07-18 ([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | Agentic coding | | | MiniMax-M3 | `minimaxm3` | 2026-06-12 ([#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24 ([#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | diff --git a/MODELS_zh.md b/MODELS_zh.md index e71067c13b..bcbbde0017 100644 --- a/MODELS_zh.md +++ b/MODELS_zh.md @@ -18,7 +18,7 @@ | 模型架构类别 | 前缀 | 加入日期 | 启用场景 | 已弃用场景 | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | 待定 | 智能体编码 | | -| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | 智能体编码 | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2359](https://github.com/SemiAnalysisAI/InferenceX/pull/2359)) | 智能体编码 | | | GLM-5.2 | `glm5.2` | 2026-07-18（[#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)） | 智能体编码 | | | MiniMax-M3 | `minimaxm3` | 2026-06-12（[#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24（[#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 03d735f567..c13907e0c6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5129,4 +5129,4 @@ - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2359 From 862024dc12f36dc71b31e966c38851e06594b249 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:21:28 -0400 Subject: [PATCH 12/25] fix: drop gpu-memory-utilization to 0.90 (flashinfer MoE workspace OOM) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same engine-level OOM as the dynamo-frontend variants: the flashinfer trtllm MXFP4 MoE kernel allocates a ~1.6 GiB runtime workspace outside vLLM's memory pool on the first forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free. 中文：与 dynamo 前端变体相同的引擎级 OOM：flashinfer trtllm MXFP4 MoE 内核在 首个前向时于 vLLM 显存池外分配约 1.6 GiB 工作区，0.95 下仅剩约 1.35 GiB。 改为 0.90。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 7 ++++++- perf-changelog.yaml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 2e313acf70..65b123d85f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -83,7 +83,12 @@ backend: trust-remote-code: true load-format: fastsafetensors moe-backend: auto - gpu-memory-utilization: 0.95 + # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a + # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first + # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first + # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 + # matches the GB200/GB300 agentic recipes. + gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — # legitimate here because this recipe serves directly with vllm serve diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c13907e0c6..36133bbe5e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,7 +5122,7 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --trust-remote-code, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3 (agentic experiment Variant D: the OpenAI-frontend parser flags are legitimate here because serving is DIRECT vllm serve, not the dynamo worker entrypoint that rejects them)" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.90 (0.95 OOMs: the flashinfer trtllm MXFP4 MoE kernel allocates a ~1.6 GiB runtime workspace outside vLLM's pool on the first forward), --no-enable-flashinfer-autotune, --trust-remote-code, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3 (agentic experiment Variant D: the OpenAI-frontend parser flags are legitimate here because serving is DIRECT vllm serve, not the dynamo worker entrypoint that rejects them)" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "DIRECT vLLM serving via srt-slurm PR #278 (kylliang/direct-aggregate-vllm, frontend.type: vllm): vllm serve owns the OpenAI port itself, removing the dynamo frontend/worker/router entirely (dynamo install: false) and with it the kimi_k3 tiktoken tokenizer gap that 404'd every request on dynamo <=1.2.1. PR #278 validates single-node only, so patches/srt-slurm-pr278-direct-vllm-multinode.patch extends it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" From 4370988a8e5dbdf8e4ceba9323697598a4783399 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:03:53 -0400 Subject: [PATCH 13/25] fix: expandable_segments allocator, drop NCCL_CUMEM_ENABLE (prefill OOM) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inherited from the closed dynamo-frontend variants (#2355/#2358): at gpu-mem-util 0.90 the first long-context MLA prefill OOM'd on a 2.92 GiB transient while 3.39 GiB sat reserved-but-unallocated (fragmentation). Set PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True and drop NCCL_CUMEM_ENABLE. 中文：继承自已关闭的 dynamo 前端变体（#2355/#2358）：0.90 显存利用率下首个 长上下文 MLA 预填充因 2.92 GiB 瞬时分配 OOM，而 3.39 GiB 处于已保留未分配 状态（碎片化）。设置 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True 并 移除 NCCL_CUMEM_ENABLE。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 9 ++++++++- perf-changelog.yaml | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 65b123d85f..b00c9dfeb1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -68,7 +68,14 @@ backend: # 300-second model-execution default. VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - NCCL_CUMEM_ENABLE: "1" + # expandable_segments: the dynamo-frontend variants OOM'd on a 2.92 GiB + # MLA long-context prefill transient (kv_b_proj in + # _compute_prefill_context) while 3.39 GiB sat reserved-but-unallocated — + # allocator fragmentation, exactly the case this mode fixes (and what the + # torch OOM message recommends; the DSv4 recipes set it too). + # NCCL_CUMEM_ENABLE dropped at the same time to trim NCCL's share of + # non-PyTorch device memory. + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 36133bbe5e..d21d3f153a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5126,6 +5126,7 @@ - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "DIRECT vLLM serving via srt-slurm PR #278 (kylliang/direct-aggregate-vllm, frontend.type: vllm): vllm serve owns the OpenAI port itself, removing the dynamo frontend/worker/router entirely (dynamo install: false) and with it the kimi_k3 tiktoken tokenizer gap that 404'd every request on dynamo <=1.2.1. PR #278 validates single-node only, so patches/srt-slurm-pr278-direct-vllm-multinode.patch extends it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" + - "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True and NCCL_CUMEM_ENABLE dropped: the dynamo-frontend variants (#2355/#2358, closed in favor of this PR) OOM'd at gpu-mem-util 0.90 on a 2.92 GiB MLA long-context prefill transient (kv_b_proj in _compute_prefill_context) while 3.39 GiB sat reserved-but-unallocated — allocator fragmentation, fixed by expandable_segments (recommended by the torch OOM message; the DSv4 recipes set it); dropping NCCL_CUMEM_ENABLE trims NCCL's share of the ~8 GiB non-PyTorch device memory" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" From 0c5fe1140ac82b59d4b5d2338ba0fb677803ada8 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:11:45 -0400 Subject: [PATCH 14/25] Revert "fix: expandable_segments allocator, drop NCCL_CUMEM_ENABLE (prefill OOM)" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 4370988a8. The superseded direct-vllm run served the agentic benchmark for 24 minutes on the original env (NCCL_CUMEM_ENABLE=1, no expandable_segments) without any OOM — the allocator change was precautionary carryover from the closed dynamo-frontend variants and was never justified by evidence from this serving path. Restore the env that was demonstrably running. 中文：回滚 4370988a8。被中断的 direct-vllm 运行在原始环境 （NCCL_CUMEM_ENABLE=1、未设 expandable_segments）下已稳定运行智能体基准测试 24 分钟且无 OOM——该分配器改动只是从已关闭的 dynamo 前端变体沿袭的预防性 措施，并无本服务路径上的证据支持。恢复已被验证可运行的环境。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 9 +-------- perf-changelog.yaml | 1 - 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index b00c9dfeb1..65b123d85f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -68,14 +68,7 @@ backend: # 300-second model-execution default. VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - # expandable_segments: the dynamo-frontend variants OOM'd on a 2.92 GiB - # MLA long-context prefill transient (kv_b_proj in - # _compute_prefill_context) while 3.39 GiB sat reserved-but-unallocated — - # allocator fragmentation, exactly the case this mode fixes (and what the - # torch OOM message recommends; the DSv4 recipes set it too). - # NCCL_CUMEM_ENABLE dropped at the same time to trim NCCL's share of - # non-PyTorch device memory. - PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d21d3f153a..36133bbe5e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5126,7 +5126,6 @@ - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "DIRECT vLLM serving via srt-slurm PR #278 (kylliang/direct-aggregate-vllm, frontend.type: vllm): vllm serve owns the OpenAI port itself, removing the dynamo frontend/worker/router entirely (dynamo install: false) and with it the kimi_k3 tiktoken tokenizer gap that 404'd every request on dynamo <=1.2.1. PR #278 validates single-node only, so patches/srt-slurm-pr278-direct-vllm-multinode.patch extends it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - - "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True and NCCL_CUMEM_ENABLE dropped: the dynamo-frontend variants (#2355/#2358, closed in favor of this PR) OOM'd at gpu-mem-util 0.90 on a 2.92 GiB MLA long-context prefill transient (kv_b_proj in _compute_prefill_context) while 3.39 GiB sat reserved-but-unallocated — allocator fragmentation, fixed by expandable_segments (recommended by the torch OOM message; the DSv4 recipes set it); dropping NCCL_CUMEM_ENABLE trims NCCL's share of the ~8 GiB non-PyTorch device memory" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" From e675dd239d9f1897e23f602486c9efb02a2b329e Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:59:49 -0400 Subject: [PATCH 15/25] feat: add VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep prefix-cache blocks alive across agentic turn gaps, matching the GB200/GB300 AgentX recipes. 中文：新增 VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768，使前缀缓存块在智能体 回合间隔内保持留存，与 GB200/GB300 AgentX 配方一致。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 65b123d85f..7e9941dddb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -68,6 +68,10 @@ backend: # 300-second model-execution default. VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + # Keep prefix-cache blocks alive across agentic turn gaps (same value as + # the GB200/GB300 AgentX recipes) — trajectory reuse is the point of the + # agentic scenario. + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" From 4b0c3a44ea8a9894ee1a5a6dc86cc61f54513461 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:07:43 -0400 Subject: [PATCH 16/25] feat: widen agentic conc list to 1/8/16/32 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将智能体并发列表从单点 8 扩展为 1/8/16/32。 Co-Authored-By: Claude Fable 5 --- configs/nvidia-master.yaml | 4 +--- perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 014afa7314..0aab2ff0fb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8352,10 +8352,8 @@ kimik3-fp4-b200-dynamo-vllm-agentic: scenarios: agentic-coding: - search-space: - # Single-concurrency smoke test for the bring-up; widen the conc curve - # once the topology is proven green. - spec-decoding: none - conc-list: [8] + conc-list: [1, 8, 16, 32] prefill: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 36133bbe5e..5d2966a962 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5123,7 +5123,7 @@ - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.90 (0.95 OOMs: the flashinfer trtllm MXFP4 MoE kernel allocates a ~1.6 GiB runtime workspace outside vLLM's pool on the first forward), --no-enable-flashinfer-autotune, --trust-remote-code, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3 (agentic experiment Variant D: the OpenAI-frontend parser flags are legitimate here because serving is DIRECT vllm serve, not the dynamo worker entrypoint that rejects them)" - - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" + - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 1/8/16/32" - "DIRECT vLLM serving via srt-slurm PR #278 (kylliang/direct-aggregate-vllm, frontend.type: vllm): vllm serve owns the OpenAI port itself, removing the dynamo frontend/worker/router entirely (dynamo install: false) and with it the kimi_k3 tiktoken tokenizer gap that 404'd every request on dynamo <=1.2.1. PR #278 validates single-node only, so patches/srt-slurm-pr278-direct-vllm-multinode.patch extends it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" From 479b74bc237a0fd8ea74fe0ed9ef9cb6137df68a Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:41:24 -0400 Subject: [PATCH 17/25] fix: remove VLLM_PREFIX_CACHE_RETENTION_INTERVAL (K3 scheduler block) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine init hard-fails on Kimi-K3 with the GB200/GB300 AgentX value: "VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative and a multiple of scheduler_block_size (3145728)" — the KDA hybrid architecture gives K3 a 3.1M-token scheduler block. Default retention served fine in the earlier runs, so drop the override. 中文：移除 VLLM_PREFIX_CACHE_RETENTION_INTERVAL——Kimi-K3 的 KDA 混合架构使 scheduler_block_size 达 3145728，GB200/GB300 AgentX 的 32768 取值导致引擎 初始化直接失败（必须为其整数倍）。此前运行证明默认留存策略可正常服务， 故不再覆盖。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 7e9941dddb..8eed2ec13b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -68,10 +68,12 @@ backend: # 300-second model-execution default. VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - # Keep prefix-cache blocks alive across agentic turn gaps (same value as - # the GB200/GB300 AgentX recipes) — trajectory reuse is the point of the - # agentic scenario. - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value + # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a + # scheduler_block_size of 3145728 and the interval must be a multiple of + # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative + # and a multiple of scheduler_block_size (3145728)"). Default retention + # served fine in earlier runs. NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" From 245e36ec9fef9a29e3de18ee09079fec86c7cc57 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:24:09 -0400 Subject: [PATCH 18/25] =?UTF-8?q?feat:=20agentic=20experiment=20I=20?= =?UTF-8?q?=E2=80=94=20variant=20D=20+=20DSpark=20spec=20decoding,=20golde?= =?UTF-8?q?n=20AL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Variant D (#2359) plus DSpark speculative decoding with the Inferact/Kimi-K3-DSpark draft head: num_speculative_tokens 7, FLASHINFER_MLA attention backend, probabilistic draft sampling + block rejection sampling (the #2366 acceptance-rate optimization), and synthetic_acceptance_length 3.84 — the committed golden AL for exactly this sampling config at k=7 (#2368, measured in run 30316471205). Conc curve widened to 1/2/4/8/16/32; spec-decoding labeled draft_model. 中文：智能体实验变体 I——变体 D（#2359）加 DSpark 投机解码 （Inferact/Kimi-K3-DSpark 草稿头）：num_speculative_tokens 7、 FLASHINFER_MLA 注意力后端、probabilistic 草稿采样 + block 拒绝采样 （#2366 的接受率优化），synthetic_acceptance_length 3.84 即该采样配置在 k=7 下的黄金 AL（#2368，run 30316471205 测得）。并发曲线扩展为 1/2/4/8/16/32；spec-decoding 标注为 draft_model。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 9 ++++++++- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 3 ++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 8eed2ec13b..976d6ea4d1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -1,4 +1,4 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-agentic" +name: "kimik3-vllm-agg-b200-tp8pp2-dspark-agentic" # Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). # The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not @@ -103,6 +103,13 @@ backend: enable-auto-tool-choice: true tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 + # DSpark speculative decoding (variant I) with the Inferact draft head, + # probabilistic draft sampling + block rejection sampling (the #2366 + # acceptance-rate optimization), k=7, and synthetic_acceptance_length + # 3.84 = the committed golden AL for exactly this sampling config at + # k=7 (golden_al_distribution/kimik3_dspark_probabilistic_sample_method_ + # block_rejection_sample_method.yaml, measured in run 30316471205). + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark", "num_speculative_tokens":7, "method": "dspark", "attention_backend": "FLASHINFER_MLA", "draft_sample_method": "probabilistic", "rejection_sample_method": "block", "synthetic_acceptance_length": 3.84}' # No explicit max-model-len: let vLLM derive the native 1M window from # the model config (agentic trajectories blow past any small cap, and # K3's KDA layers keep per-token KV small — only the 24 gated-MLA diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0aab2ff0fb..ea12f4f01a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8352,8 +8352,8 @@ kimik3-fp4-b200-dynamo-vllm-agentic: scenarios: agentic-coding: - search-space: - - spec-decoding: none - conc-list: [1, 8, 16, 32] + - spec-decoding: draft_model + conc-list: [1, 2, 4, 8, 16, 32] prefill: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5d2966a962..1b16d24112 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5128,5 +5128,6 @@ - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" + - "Agentic experiment Variant I (of the #2359 direct-vllm Variant D): DSpark speculative decoding with the Inferact/Kimi-K3-DSpark draft head — num_speculative_tokens 7, FLASHINFER_MLA attention backend, probabilistic draft sampling + block rejection sampling (the #2366 acceptance-rate optimization), synthetic_acceptance_length 3.84 = the committed golden AL for this exact sampling config at k=7 (#2368's golden_al_distribution curve, measured in run 30316471205). Conc curve widened to 1/2/4/8/16/32; spec-decoding labeled draft_model" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2359 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 6dc9b4a6e349d2dd5e7790fc877949ad6d8aefef Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:25:09 -0400 Subject: [PATCH 19/25] docs: point changelog and MODELS rows at experiment PR #2376 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将更新日志条目与 MODELS 表格行链接指向实验 PR #2376。 Co-Authored-By: Claude Fable 5 --- MODELS.md | 2 +- MODELS_zh.md | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODELS.md b/MODELS.md index 4d9fa6ff7c..9e7453f142 100644 --- a/MODELS.md +++ b/MODELS.md @@ -22,7 +22,7 @@ This document tracks every model benchmarked by InferenceX-e2e: when it was adde | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | TBD | Agentic coding | | -| Kimi-K3 | `kimik3` | 2026-07-27 ([#2359](https://github.com/SemiAnalysisAI/InferenceX/pull/2359)) | Agentic coding | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2376](https://github.com/SemiAnalysisAI/InferenceX/pull/2376)) | Agentic coding | | | GLM-5.2 | `glm5.2` | 2026-07-18 ([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | Agentic coding | | | MiniMax-M3 | `minimaxm3` | 2026-06-12 ([#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24 ([#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | diff --git a/MODELS_zh.md b/MODELS_zh.md index c259c96de2..ea7d2706f8 100644 --- a/MODELS_zh.md +++ b/MODELS_zh.md @@ -22,7 +22,7 @@ | 模型架构类别 | 前缀 | 加入日期 | 启用场景 | 已弃用场景 | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | 待定 | 智能体编码 | | -| Kimi-K3 | `kimik3` | 2026-07-27 ([#2359](https://github.com/SemiAnalysisAI/InferenceX/pull/2359)) | 智能体编码 | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2376](https://github.com/SemiAnalysisAI/InferenceX/pull/2376)) | 智能体编码 | | | GLM-5.2 | `glm5.2` | 2026-07-18（[#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)） | 智能体编码 | | | MiniMax-M3 | `minimaxm3` | 2026-06-12（[#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24（[#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c5aff6e386..fb21245407 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5143,4 +5143,4 @@ - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - "Agentic experiment Variant I (of the #2359 direct-vllm Variant D): DSpark speculative decoding with the Inferact/Kimi-K3-DSpark draft head — num_speculative_tokens 7, FLASHINFER_MLA attention backend, probabilistic draft sampling + block rejection sampling (the #2366 acceptance-rate optimization), synthetic_acceptance_length 3.84 = the committed golden AL for this exact sampling config at k=7 (#2368's golden_al_distribution curve, measured in run 30316471205). Conc curve widened to 1/2/4/8/16/32; spec-decoding labeled draft_model" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/0 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2376 From b3bde33e6aae5c63ca118b11c8b46aeb08bbc63e Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:26:45 -0400 Subject: [PATCH 20/25] feat: label variant I spec-decoding as dspark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the spec-decoding schema Literal with "dspark" (alongside mtp/draft_model/none) and label the variant I entry dspark instead of draft_model. Downstream consumers (workflow env, result filenames, process_result) treat the value as an opaque string, so results carry spec-dspark. 中文：为 spec-decoding 的 schema Literal 新增 "dspark"（与 mtp/draft_model/none 并列），并将变体 I 的条目由 draft_model 改标为 dspark。下游（工作流环境变量、结果文件名、process_result）将该值视为 不透明字符串，结果将带有 spec-dspark 标注。 Co-Authored-By: Claude Fable 5 --- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 2 +- utils/matrix_logic/validation.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index ea12f4f01a..bc05c91414 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8352,7 +8352,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic: scenarios: agentic-coding: - search-space: - - spec-decoding: draft_model + - spec-decoding: dspark conc-list: [1, 2, 4, 8, 16, 32] prefill: num-worker: 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fb21245407..9903575ca2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5141,6 +5141,6 @@ - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - - "Agentic experiment Variant I (of the #2359 direct-vllm Variant D): DSpark speculative decoding with the Inferact/Kimi-K3-DSpark draft head — num_speculative_tokens 7, FLASHINFER_MLA attention backend, probabilistic draft sampling + block rejection sampling (the #2366 acceptance-rate optimization), synthetic_acceptance_length 3.84 = the committed golden AL for this exact sampling config at k=7 (#2368's golden_al_distribution curve, measured in run 30316471205). Conc curve widened to 1/2/4/8/16/32; spec-decoding labeled draft_model" + - "Agentic experiment Variant I (of the #2359 direct-vllm Variant D): DSpark speculative decoding with the Inferact/Kimi-K3-DSpark draft head — num_speculative_tokens 7, FLASHINFER_MLA attention backend, probabilistic draft sampling + block rejection sampling (the #2366 acceptance-rate optimization), synthetic_acceptance_length 3.84 = the committed golden AL for this exact sampling config at k=7 (#2368's golden_al_distribution curve, measured in run 30316471205). Conc curve widened to 1/2/4/8/16/32; spec-decoding labeled dspark (new schema value alongside mtp/draft_model)" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2376 diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 9f19572f08..531390de08 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -156,7 +156,7 @@ class SingleNodeMatrixEntry(BaseModel): model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value) precision: str framework: str - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "draft_model", "dspark", "none"] = Field( alias=Fields.SPEC_DECODING.value ) runner: str @@ -223,7 +223,7 @@ class MultiNodeMatrixEntry(BaseModel): model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value) precision: str framework: str - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "draft_model", "dspark", "none"] = Field( alias=Fields.SPEC_DECODING.value ) runner: str @@ -276,7 +276,7 @@ class SingleNodeAgenticMatrixEntry(BaseModel): pcp_size: int = Field(alias=Fields.PCP_SIZE.value, gt=0, strict=True) ep: int dp_attn: bool = Field(alias=Fields.DP_ATTN.value) - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "draft_model", "dspark", "none"] = Field( default="none", alias=Fields.SPEC_DECODING.value ) conc: int @@ -314,7 +314,7 @@ class MultiNodeAgenticMatrixEntry(BaseModel): model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value) precision: str framework: str - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "draft_model", "dspark", "none"] = Field( alias=Fields.SPEC_DECODING.value ) runner: str @@ -483,7 +483,7 @@ class SingleNodeSearchSpaceEntry(BaseModel): pcp_size: int = Field( default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) ep: Optional[int] = None - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "draft_model", "dspark", "none"] = Field( default="none", alias=Fields.SPEC_DECODING.value) dp_attn: Optional[bool] = Field( default=None, alias=Fields.DP_ATTN.value) @@ -508,7 +508,7 @@ class MultiNodeSearchSpaceEntry(BaseModel): """Multinode search space configuration.""" model_config = ConfigDict(extra='forbid', populate_by_name=True) - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "draft_model", "dspark", "none"] = Field( default="none", alias=Fields.SPEC_DECODING.value) prefill: WorkerConfig decode: WorkerConfig @@ -564,7 +564,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel): default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) ep: Optional[int] = None dp_attn: Optional[bool] = Field(default=None, alias=Fields.DP_ATTN.value) - spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + spec_decoding: Literal["mtp", "draft_model", "dspark", "none"] = Field( default="none", alias=Fields.SPEC_DECODING.value) prefill: Optional[WorkerConfig] = None decode: Optional[WorkerConfig] = None From 08217d920cf0d2b0787c02cbcba2291fb1b3e152 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:28:01 -0400 Subject: [PATCH 21/25] refactor: name the variant I recipe file by its spec method (dspark) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename agg-b200-tp8pp2-agentic.yaml to agg-b200-tp8pp2-dspark-agentic.yaml on this branch, matching the --agentic naming of the DSv4 MTP recipes, and update the CONFIG_FILE routing and changelog. Also add method context: DSpark is DeepSeek's confidence-scheduled semi-autoregressive speculative decoding (arXiv 2607.05147), reported ~27-31% higher accepted length than EAGLE3 and 60-85% faster per-user generation than MTP-1 at matched throughput. 中文：将变体 I 的配方文件更名为 agg-b200-tp8pp2-dspark-agentic.yaml，与 DSv4 MTP 配方的「拓扑-投机方法-agentic」命名一致，并同步更新 CONFIG_FILE 路由与更新日志。补充方法背景：DSpark 为 DeepSeek 的置信度调度半自回归投机 解码框架（arXiv 2607.05147），报告显示接受长度较 EAGLE3 高约 27-31%，同等 吞吐下单用户生成速度较 MTP-1 快 60-85%。 Co-Authored-By: Claude Fable 5 --- ...pp2-agentic.yaml => agg-b200-tp8pp2-dspark-agentic.yaml} | 6 +++++- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/{agg-b200-tp8pp2-agentic.yaml => agg-b200-tp8pp2-dspark-agentic.yaml} (94%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml similarity index 94% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml index 976d6ea4d1..d828acb278 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml @@ -103,7 +103,11 @@ backend: enable-auto-tool-choice: true tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 - # DSpark speculative decoding (variant I) with the Inferact draft head, + # DSpark (DeepSeek's confidence-scheduled semi-autoregressive + # speculative decoding, arXiv 2607.05147): parallel backbone drafting + # plus a lightweight sequential head — reported ~27-31% higher accepted + # length than EAGLE3 and 60-85% faster per-user generation than MTP-1 + # at matched throughput. Variant I uses the Inferact draft head, # probabilistic draft sampling + block rejection sampling (the #2366 # acceptance-rate optimization), k=7, and synthetic_acceptance_length # 3.84 = the committed golden AL for exactly this sampling config at diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index bc05c91414..a2aa233db5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8361,7 +8361,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" # The aggregate worker also performs decode; keep the decode worker # count at zero so result aggregation counts the 16 GPUs only once. decode: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9903575ca2..dca6048666 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5142,5 +5142,5 @@ - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - "Agentic experiment Variant I (of the #2359 direct-vllm Variant D): DSpark speculative decoding with the Inferact/Kimi-K3-DSpark draft head — num_speculative_tokens 7, FLASHINFER_MLA attention backend, probabilistic draft sampling + block rejection sampling (the #2366 acceptance-rate optimization), synthetic_acceptance_length 3.84 = the committed golden AL for this exact sampling config at k=7 (#2368's golden_al_distribution curve, measured in run 30316471205). Conc curve widened to 1/2/4/8/16/32; spec-decoding labeled dspark (new schema value alongside mtp/draft_model)" - - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" + - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2376 From 7176883426eb80e12483758337983d8418234a8f Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Tue, 28 Jul 2026 02:16:17 -0400 Subject: [PATCH 22/25] fix: DSpark synthetic rejection sampling at the golden AL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace draft_sample_method=probabilistic + rejection_sample_method=block with rejection_sample_method=synthetic. The golden AL (3.84 at k=7) was MEASURED under probabilistic+block (#2366/#2368); the benchmark injects it via synthetic rejection sampling per the golden-AL convention (golden_al_distribution/README.md), the same pattern as the DSv4 MTP recipes (rejection_sample_method synthetic + synthetic_acceptance_length). 中文：将 draft_sample_method=probabilistic + rejection_sample_method=block 替换为 rejection_sample_method=synthetic。黄金 AL（k=7 时 3.84）是在 probabilistic+block 下测得（#2366/#2368）；基准测试按黄金 AL 约定通过合成 拒绝采样注入该值（golden_al_distribution/README.md），与 DSv4 MTP 配方的 模式一致。 Co-Authored-By: Claude Fable 5 --- .../agentic/agg-b200-tp8pp2-dspark-agentic.yaml | 16 +++++++++------- perf-changelog.yaml | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml index d828acb278..90ce1964b6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml @@ -107,13 +107,15 @@ backend: # speculative decoding, arXiv 2607.05147): parallel backbone drafting # plus a lightweight sequential head — reported ~27-31% higher accepted # length than EAGLE3 and 60-85% faster per-user generation than MTP-1 - # at matched throughput. Variant I uses the Inferact draft head, - # probabilistic draft sampling + block rejection sampling (the #2366 - # acceptance-rate optimization), k=7, and synthetic_acceptance_length - # 3.84 = the committed golden AL for exactly this sampling config at - # k=7 (golden_al_distribution/kimik3_dspark_probabilistic_sample_method_ - # block_rejection_sample_method.yaml, measured in run 30316471205). - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark", "num_speculative_tokens":7, "method": "dspark", "attention_backend": "FLASHINFER_MLA", "draft_sample_method": "probabilistic", "rejection_sample_method": "block", "synthetic_acceptance_length": 3.84}' + # at matched throughput. This variant uses the Inferact draft head at + # k=7 with SYNTHETIC rejection sampling, injecting + # synthetic_acceptance_length 3.84 — the committed golden AL measured + # under probabilistic draft + block rejection sampling (the #2366 + # acceptance-rate optimization; golden_al_distribution/ + # kimik3_dspark_probabilistic_sample_method_block_rejection_sample_ + # method.yaml, run 30316471205) — per the golden-AL synthetic- + # acceptance convention (golden_al_distribution/README.md). + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark", "num_speculative_tokens":7, "method": "dspark", "attention_backend": "FLASHINFER_MLA", "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 3.84}' # No explicit max-model-len: let vLLM derive the native 1M window from # the model config (agentic trajectories blow past any small cap, and # K3's KDA layers keep per-token KV small — only the 24 gated-MLA diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 94bd6a699a..8afaa412be 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5150,6 +5150,6 @@ - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - - "Agentic experiment Variant I (of the #2359 direct-vllm Variant D): DSpark speculative decoding with the Inferact/Kimi-K3-DSpark draft head — num_speculative_tokens 7, FLASHINFER_MLA attention backend, probabilistic draft sampling + block rejection sampling (the #2366 acceptance-rate optimization), synthetic_acceptance_length 3.84 = the committed golden AL for this exact sampling config at k=7 (#2368's golden_al_distribution curve, measured in run 30316471205). Conc curve widened to 1/2/4/8/16/32; spec-decoding labeled dspark (new schema value alongside mtp/draft_model)" + - "Agentic experiment Variant I (of the #2359 direct-vllm Variant D): DSpark speculative decoding with the Inferact/Kimi-K3-DSpark draft head — num_speculative_tokens 7, FLASHINFER_MLA attention backend, SYNTHETIC rejection sampling with synthetic_acceptance_length 3.84 = the committed golden AL measured under probabilistic draft + block rejection sampling (the #2366 acceptance-rate optimization) at k=7 (#2368's golden_al_distribution curve, measured in run 30316471205). Conc curve widened to 1/2/4/8/16/32; spec-decoding labeled dspark (new schema value alongside mtp/draft_model)" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2376 From 41dd5a90dd515d59486fc244b2ad3ab554f59eb7 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Tue, 28 Jul 2026 02:58:37 -0400 Subject: [PATCH 23/25] fix: verify DSpark draft against a pp=1 parallel-config view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine init died with "NotImplementedError: Pipeline parallelism is not supported for this model" — SpeculativeConfig verifies the draft model against a parallel config that unconditionally inherits the target's pipeline_parallel_size (2), and the Inferact/Kimi-K3-DSpark draft head does not implement SupportsPP. At runtime V1 drafters load ONLY on the final pipeline stage (vllm-project/vllm#16568), effectively draft_pipeline_parallel_size=1, and upstream has no config knob for it — so extend the in-container patch script to verify the draft against a pp=1 view of the parallel config. Idempotent; refuses to patch if the call-site shape changed. 中文：引擎初始化报 "Pipeline parallelism is not supported for this model"—— SpeculativeConfig 用无条件继承目标 pipeline_parallel_size(2) 的并行配置校验 草稿模型，而 Inferact/Kimi-K3-DSpark 草稿头未实现 SupportsPP。V1 运行时草稿 模型仅加载在最后一个流水线阶段（vllm#16568），等效 draft_pipeline_parallel_size=1，且上游无相应配置项——故扩展容器内补丁脚本， 以 pp=1 视图校验草稿模型。补丁幂等，调用点形态变化时拒绝执行。 Co-Authored-By: Claude Fable 5 --- .../configs/kimi-k3-container-deps.sh | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh index abddf4d9c3..66edcf96a8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh @@ -37,3 +37,60 @@ if n != 1: path.write_text(new) print(f"Patched mamba_hybrid index_fill_ index dtype: {path}") PY + +# The DSpark draft head (Inferact/Kimi-K3-DSpark) does not implement +# SupportsPP, and SpeculativeConfig verifies the draft model against a +# parallel config that unconditionally inherits the target's +# pipeline_parallel_size — so TP8xPP2 dies at engine init with +# "NotImplementedError: Pipeline parallelism is not supported for this +# model". At runtime, however, V1 drafters are loaded ONLY on the final +# pipeline stage (vllm-project/vllm#16568: "drafters are only loaded in the +# last pp stage, which essentially means draft_pipeline_parallel_size=1"), +# so the correct check is against a pp=1 view of the parallel config. +# Idempotent; refuses to patch if the call-site shape changed. +python3 - <<'PY' +import pathlib +import re + +import vllm.config.speculative as sp + +path = pathlib.Path(sp.__file__) +src = path.read_text() +if "_infmax_pp1_draft_view" in src: + print(f"draft-PP verification patch already applied: {path}") + raise SystemExit(0) + +pat = re.compile( + r"self\.draft_model_config\.verify_with_parallel_config\(\s*" + r"(self\.(?:draft|target)_parallel_config)\s*,?\s*\)", + re.S, +) +matches = list(pat.finditer(src)) +if len(matches) != 1: + raise SystemExit( + f"expected exactly one draft verify_with_parallel_config call in " + f"{path}, found {len(matches)} — image layout changed, refusing to patch" + ) +src = pat.sub( + lambda m: ( + "self.draft_model_config.verify_with_parallel_config(" + f"_infmax_pp1_draft_view({m.group(1)}))" + ), + src, + count=1, +) +src += ''' + +def _infmax_pp1_draft_view(parallel_config): + """InferenceX: V1 drafters load whole on the final pipeline stage, so the + draft model is verified against a pp=1 view of the parallel config (the + DSpark draft head does not implement SupportsPP).""" + import copy + + pc = copy.deepcopy(parallel_config) + pc.pipeline_parallel_size = 1 + return pc +''' +path.write_text(src) +print(f"Patched draft-model PP verification: {path}") +PY From 18d66d537d67679a96144e85c950a753d223782f Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Tue, 28 Jul 2026 03:28:28 -0400 Subject: [PATCH 24/25] feat: transport DSpark aux hidden states across PP stages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The K3 DSpark head consumes aux hidden states from five target layers (target_layer_ids [2, 23, 47, 71, 89]); the target model captures them per PP stage, so under TP8xPP2 the layers-2/23 captures live on stage 0 and never reach the last-stage drafter, whose combine_hidden_states projection expects hidden_size x 5. This build's V2 runner already broadcasts sampled counts across PP ranks (PPHandler), so the remaining gap is the aux transport — the same gap vllm-ascend PR #12507 closes for EAGLE3. Extend the container patch script with three coordinated edits to kimi_k3/nvidia/model.py: preallocate aux_hidden_ receive buffers for upstream-owned aux layers, attach/pass-through captured aux states on non-last ranks via IntermediateTensors, and merge received + local aux in global layer order on the last rank. Anchors verified against the public vllm kimi-k3 branch source; idempotent; refuses to patch on layout drift. 中文：K3 DSpark 草稿头消费五个目标层（target_layer_ids [2, 23, 47, 71, 89]） 的辅助隐状态；目标模型按流水线阶段各自捕获，TP8xPP2 下第 2/23 层的捕获位于 阶段 0，无法到达最后阶段的草稿模型（其 combine_hidden_states 期望 hidden_size x 5）。该镜像的 V2 runner 已通过 PPHandler 跨阶段广播采样计数， 唯一缺口即辅助隐状态传输（与 vllm-ascend PR #12507 为 EAGLE3 修复的相同）。 容器补丁脚本新增三处协同修改：接收侧预分配上游辅助层缓冲、非最后阶段经 IntermediateTensors 附带/透传捕获的辅助隐状态、最后阶段按全局层序合并。 锚点已对照公开 kimi-k3 分支源码验证；幂等；源码布局变化时拒绝执行。 Co-Authored-By: Claude Fable 5 --- .../configs/kimi-k3-container-deps.sh | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh index 66edcf96a8..661e189284 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh @@ -94,3 +94,114 @@ def _infmax_pp1_draft_view(parallel_config): path.write_text(src) print(f"Patched draft-model PP verification: {path}") PY + +# DSpark under pipeline parallelism, part 2: transport aux hidden states +# across PP stages. The K3 DSpark head consumes aux hidden states from five +# target layers (target_layer_ids [2, 23, 47, 71, 89]); the target model +# captures them per PP stage, so with TP8xPP2 the layers-2/23 captures live +# on stage 0 and never reach the last-stage drafter (whose +# combine_hidden_states expects hidden_size x 5). This build's V2 runner +# already broadcasts sampled counts across PP ranks (PPHandler), so the only +# missing piece is the aux transport — the same gap vllm-ascend PR #12507 +# fixes for EAGLE3. Three coordinated edits to kimi_k3/nvidia/model.py: +# (a) preallocate aux_hidden_ intermediate-tensor buffers on +# receiving ranks for aux layers owned by earlier stages; +# (b) non-last ranks attach their captured aux states (and pass through +# upstream ones) to the outgoing IntermediateTensors; +# (c) the last rank merges received + local aux states in global layer +# order before returning them to the drafter. +# Idempotent; hard-asserts on exact anchors and refuses to patch otherwise. +python3 - <<'PY' +import pathlib + +import vllm.models.kimi_k3.nvidia.model as m + +path = pathlib.Path(m.__file__) +src = path.read_text() +if "_infmax_aux_pp" in src: + print(f"aux-PP transport patch already applied: {path}") + raise SystemExit(0) + +def replace_once(src: str, old: str, new: str, tag: str) -> str: + n = src.count(old) + if n != 1: + raise SystemExit( + f"aux-PP patch anchor {tag!r} matched {n} times in {path} — " + "image layout changed, refusing to patch" + ) + return src.replace(old, new) + +# (a) preallocate upstream-aux receive buffers +old_a = """ return IntermediateTensors( + { + "hidden_states": torch.zeros( + (batch_size, self.config.hidden_size), dtype=dtype, device=device + ), + "residual": torch.zeros(residual_shape, dtype=dtype, device=device), + } + )""" +new_a = """ _infmax_aux_pp = { + f"aux_hidden_{_l}": torch.zeros( + (batch_size, self.config.hidden_size), dtype=dtype, device=device + ) + for _l in sorted(getattr(self, "aux_hidden_state_layers", ()) or ()) + if _l < self.start_layer + } + return IntermediateTensors( + { + "hidden_states": torch.zeros( + (batch_size, self.config.hidden_size), dtype=dtype, device=device + ), + "residual": torch.zeros(residual_shape, dtype=dtype, device=device), + **_infmax_aux_pp, + } + )""" +src = replace_once(src, old_a, new_a, "prealloc") + +# (b) non-last ranks: send local aux states, pass through upstream ones +old_b = """ if prefix_sum is not None: + hidden_states = hidden_states + prefix_sum + return IntermediateTensors( + {"hidden_states": hidden_states, "residual": residual} + )""" +new_b = """ if prefix_sum is not None: + hidden_states = hidden_states + prefix_sum + _infmax_out = {"hidden_states": hidden_states, "residual": residual} + if intermediate_tensors is not None: + for _k, _v in intermediate_tensors.tensors.items(): + if _k.startswith("aux_hidden_"): + _infmax_out.setdefault(_k, _v) + _infmax_local = sorted( + _l + for _l in (getattr(self, "aux_hidden_state_layers", ()) or ()) + if self.start_layer <= _l <= self.end_layer + ) + for _l, _t in zip(_infmax_local, aux_hidden_states): + _infmax_out[f"aux_hidden_{_l}"] = _t + return IntermediateTensors(_infmax_out)""" +src = replace_once(src, old_b, new_b, "send") + +# (c) last rank: merge received + local aux in global layer order +old_c = """ if aux_hidden_states: + return hidden_states, aux_hidden_states + return hidden_states""" +new_c = """ if aux_hidden_states: + _infmax_merged = {} + if intermediate_tensors is not None: + for _k, _v in intermediate_tensors.tensors.items(): + if _k.startswith("aux_hidden_"): + _infmax_merged[int(_k.rsplit("_", 1)[1])] = _v + _infmax_local = sorted( + _l + for _l in (getattr(self, "aux_hidden_state_layers", ()) or ()) + if self.start_layer <= _l <= self.end_layer + ) + _infmax_merged.update(zip(_infmax_local, aux_hidden_states)) + aux_hidden_states = [_infmax_merged[_k] for _k in sorted(_infmax_merged)] + return hidden_states, aux_hidden_states + return hidden_states""" +src = replace_once(src, old_c, new_c, "merge") + +path.write_text(src) +print(f"Patched aux-hidden-state PP transport ({path})") +PY From 83fc62234d709b9020095bf97eef260ac9c2812d Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:40:37 -0400 Subject: [PATCH 25/25] fix: lift V2 runner's dspark-under-PP guard (aux transport in place) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner hard-raises "dspark with pipeline parallel is not supported" for aux-hidden-state spec methods — the guard exists precisely because aux captures were stage-local, which the aux-transport patch now fixes (and PPHandler already broadcasts sampled counts across ranks in this build). Downgrade the ValueError to a warning via the container patch script. Anchor verified against the public kimi-k3 branch source. 中文：V2 runner 对依赖辅助隐状态的投机方法在流水线并行下硬性报错 （"dspark with pipeline parallel is not supported"）——该防护正是因辅助隐 状态仅限本阶段捕获而设，前一补丁已实现跨阶段传输（且该构建的 PPHandler 已 跨阶段广播采样计数）。通过容器补丁脚本将 ValueError 降级为警告。锚点已对照 公开 kimi-k3 分支源码验证。 Co-Authored-By: Claude Fable 5 --- .../configs/kimi-k3-container-deps.sh | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh index 661e189284..c7e716aa0b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh @@ -205,3 +205,45 @@ src = replace_once(src, old_c, new_c, "merge") path.write_text(src) print(f"Patched aux-hidden-state PP transport ({path})") PY + +# DSpark under pipeline parallelism, part 3: lift the runner's blanket guard. +# The V2 runner refuses aux-hidden-state spec methods (eagle3/dflash/dspark) +# under PP because the aux captures were stage-local — exactly the gap the +# aux-transport patch above closes. With transport in place (and PPHandler +# already broadcasting sampled counts across ranks), downgrade the hard +# ValueError to a warning. Idempotent; refuses to patch on layout drift. +python3 - <<'PY' +import pathlib + +import vllm.v1.worker.gpu.model_runner as mr + +path = pathlib.Path(mr.__file__) +src = path.read_text() +if "_infmax_pp_aux_enabled" in src: + print(f"PP spec-method guard patch already applied: {path}") + raise SystemExit(0) + +old = """ self.use_aux_hidden_state_outputs = True + if self.use_pp: + raise ValueError( + f"{self.speculative_config.method} with pipeline parallel " + "is not supported." + )""" +new = """ self.use_aux_hidden_state_outputs = True + if self.use_pp: + # _infmax_pp_aux_enabled: aux hidden states are shipped + # across PP stages by the InferenceX container patch. + logger.warning( + "InferenceX: %s under pipeline parallelism enabled " + "via patched aux-hidden-state transport.", + self.speculative_config.method, + )""" +n = src.count(old) +if n != 1: + raise SystemExit( + f"expected exactly one PP spec-method guard in {path}, found {n} — " + "image layout changed, refusing to patch" + ) +path.write_text(src.replace(old, new)) +print(f"Lifted PP spec-method guard: {path}") +PY