[KimiK3][AgentX]: H200 KimiK3 Day 0 support

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml

@@ -0,0 +1,94 @@
+name: "kimik3-vllm-agg-h200-tp16dp2ep32-latency-agentic"
+
+model:
+  path: "kimik3"
+  container: "vllm/vllm-openai:kimi-k3"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+    revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569"
+  container:
+    image: "vllm/vllm-openai:kimi-k3"
+  frameworks:
+    vllm: "0.1.dev19262+gb6bbf29dd.d20260727"
+
+dynamo:
+  install: false
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "h200"
+  gpus_per_node: 8
+  agg_nodes: 4
+  agg_workers: 1
+  gpus_per_agg: 32
+
+frontend:
+  type: vllm
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    CUDA_LAUNCH_BLOCKING: "1"
+    GLOO_SOCKET_IFNAME: "eth0"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
+    NCCL_SOCKET_IFNAME: "eth0"
+    NCCL_CUMEM_ENABLE: "1"
+    PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"
+    PYTHONNOUSERSITE: "1"
+    TRANSFORMERS_CACHE: "/hf_hub_cache"
+    VLLM_ENGINE_READY_TIMEOUT_S: "7200"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 16
+      data-parallel-size: 2
+      enable-expert-parallel: true
+      trust-remote-code: true
+      load-format: fastsafetensors
+      moe-backend: marlin
+      attention-backend: FLASHMLA
+      gpu-memory-utilization: 0.975
+      max-num-seqs: 5
+      max-num-batched-tokens: 4096
+      speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}'
+      enforce-eager: true
+      compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}'
+      enable-prefix-caching: true
+      enable-prompt-tokens-details: true
+      no-enable-flashinfer-autotune: true
+      disable-custom-all-reduce: true
+      enable-auto-tool-choice: true
+      tool-call-parser: kimi_k3
+      reasoning-parser: kimi_k3
+      language-model-only: true
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml

@@ -0,0 +1,95 @@
+name: "kimik3-vllm-agg-h200-tp8dp4ep32-balanced-agentic"
+
+model:
+  path: "kimik3"
+  container: "vllm/vllm-openai:kimi-k3"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+    revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569"
+  container:
+    image: "vllm/vllm-openai:kimi-k3"
+  frameworks:
+    vllm: "0.1.dev19262+gb6bbf29dd.d20260727"
+
+dynamo:
+  install: false
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "h200"
+  gpus_per_node: 8
+  agg_nodes: 4
+  agg_workers: 1
+  gpus_per_agg: 32
+
+frontend:
+  type: vllm
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    CUDA_LAUNCH_BLOCKING: "1"
+    GLOO_SOCKET_IFNAME: "eth0"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
+    NCCL_SOCKET_IFNAME: "eth0"
+    NCCL_CUMEM_ENABLE: "1"
+    PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"
+    PYTHONNOUSERSITE: "1"
+    TRANSFORMERS_CACHE: "/hf_hub_cache"
+    VLLM_ENGINE_READY_TIMEOUT_S: "7200"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 8
+      data-parallel-size: 4
+      enable-expert-parallel: true
+      trust-remote-code: true
+      load-format: fastsafetensors
+      moe-backend: marlin
+      attention-backend: FLASHMLA
+      gpu-memory-utilization: 0.95
+      max-num-seqs: 8
+      max-num-batched-tokens: 4096
+      speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}'
+      enforce-eager: true
+      no-async-scheduling: true
+      compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}'
+      enable-prefix-caching: true
+      enable-prompt-tokens-details: true
+      no-enable-flashinfer-autotune: true
+      disable-custom-all-reduce: true
+      enable-auto-tool-choice: true
+      tool-call-parser: kimi_k3
+      reasoning-parser: kimi_k3
+      language-model-only: true
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml

@@ -0,0 +1,96 @@
+name: "kimik3-vllm-agg-h200-tp8dp4ep32-vllm-simple-agentic"
+
+model:
+  path: "kimik3"
+  container: "vllm/vllm-openai:kimi-k3"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+    revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569"
+  container:
+    image: "vllm/vllm-openai:kimi-k3"
+  frameworks:
+    vllm: "0.1.dev19262+gb6bbf29dd.d20260727"
+
+dynamo:
+  install: false
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "h200"
+  gpus_per_node: 8
+  agg_nodes: 4
+  agg_workers: 1
+  gpus_per_agg: 32
+
+frontend:
+  type: vllm
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    CUDA_LAUNCH_BLOCKING: "1"
+    GLOO_SOCKET_IFNAME: "eth0"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
+    NCCL_SOCKET_IFNAME: "eth0"
+    NCCL_CUMEM_ENABLE: "1"
+    PYTHONHASHSEED: "42"
+    PYTHONNOUSERSITE: "1"
+    TRANSFORMERS_CACHE: "/hf_hub_cache"
+    VLLM_ENGINE_READY_TIMEOUT_S: "7200"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 8
+      data-parallel-size: 4
+      enable-expert-parallel: true
+      trust-remote-code: true
+      load-format: fastsafetensors
+      moe-backend: marlin
+      attention-backend: FLASHMLA
+      gpu-memory-utilization: 0.95
+      max-num-seqs: 16
+      max-num-batched-tokens: 4096
+      speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}'
+      enforce-eager: true
+      no-async-scheduling: true
+      compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}'
+      enable-prefix-caching: true
+      enable-prompt-tokens-details: true
+      kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":154250000000,"lazy_offload":false}}'
+      no-enable-flashinfer-autotune: true
+      disable-custom-all-reduce: true
+      enable-auto-tool-choice: true
+      tool-call-parser: kimi_k3
+      reasoning-parser: kimi_k3
+      language-model-only: true
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

configs/nvidia-master.yaml

@@ -1411,6 +1411,80 @@ kimik2.5-int4-h200-vllm:
       search-space:
       - { tp: 8, conc-start: 4, conc-end: 64 }
 
+# Kimi K3 is a 2.8T MXFP4 MoE served on H200 nodes.
+# These are the three aggregated strategy classes from the official H200
+# recipe (https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=h200):
+# Wider TP is latency-oriented, while progressively wider DP is balanced or
+# throughput-oriented. EP32 keeps the model shards within HBM and preserves
+# the native 1M context on four H200 nodes. All arms use probabilistic DSpark K2
+# with synthetic acceptance pinned to the committed golden AL of 2.51.
+# The existing `mtp` matrix label covers speculative-decoding methods; the
+# checked-in recipes select DSpark explicitly. The H200 runner supplies
+# cluster paths while srt-slurm owns allocation, direct vLLM startup,
+# readiness, and log capture. P/D disaggregation is intentionally deferred.
+kimik3-fp4-h200-vllm-agentic:
+  image: vllm/vllm-openai:kimi-k3
+  model: moonshotai/Kimi-K3
+  model-prefix: kimik3
+  runner: cluster:h200-dgxc
+  precision: fp4
+  framework: vllm
+  multinode: true
+  disagg: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      # Latency-oriented: TP16 x DP2 with EP32 across four H200 nodes.
+      - spec-decoding: mtp
+        kv-offloading: none
+        conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12]
+        prefill:
+          num-worker: 2
+          tp: 16
+          ep: 32
+          dp-attn: true
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 16
+          ep: 32
+          dp-attn: true
+      # Balanced: TP8 x DP4 with EP32 across four H200 nodes.
+      - spec-decoding: mtp
+        kv-offloading: none
+        conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16]
+        prefill:
+          num-worker: 4
+          tp: 8
+          ep: 32
+          dp-attn: true
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 8
+          ep: 32
+          dp-attn: true
+      # Throughput-oriented hybrid TP/DEP/EP with vLLM's host-DRAM KV tier.
+      - spec-decoding: mtp
+        kv-offloading: dram
+        kv-offload-backend: { name: vllm-simple }
+        conc-list: [8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32]
+        prefill:
+          num-worker: 4
+          tp: 8
+          ep: 32
+          dp-attn: true
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 8
+          ep: 32
+          dp-attn: true
+
 # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html
 # does not have a B300-specific recipe, so this config reuses the existing
 # Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available.

perf-changelog.yaml

@@ -5527,3 +5527,8 @@
     - "Record the recipes' active HiCache host-DRAM tier and, for disaggregated points, Dynamo router commit 5a638087 in nvidia-master metadata so generated artifacts no longer report kv_offloading=none, allocated_cpu_dram_gb=0, or a null router."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2477
 
+- config-keys:
+    - kimik3-fp4-h200-vllm-agentic
+  description:
+    - "Add Day-0 Kimi K3 FP4 AgentX H200 performance recipes from the official vLLM guidance, using fit-safe TP/DP/EP topologies, optional SimpleCPU KV offload, and a five-minute reconstructed trace idle-gap cap"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2353