[DNM, Experimental] GLM-5.3-Flash MI325X FP8 HiCache and concurrency / [DNM, Experimental] GLM-5.3-Flash MI325X FP8 HiCache与并发优化

.github/workflows/README.md

@@ -162,7 +162,7 @@ test-config --config-keys dsr1-fp8-h200-sglang --evals-only --all-evals --config
 
 ## PR Sweep Labels
 
-`run-sweep.yml` sweeps only same-repository PRs that change
+`run-sweep.yml` sweeps only same-repository PRs, including stacked PRs, that change
 `inferencex-e2e/perf-changelog.yaml`, using the appended entries as the matrix.
 Apply exactly one primary label, normally `full-sweep-fail-fast`.
 Fork PRs use the [trusted dispatch](#trusted-external-fork-sweep-dispatch-poc).

.github/workflows/speedbench-al.yml

@@ -19,13 +19,19 @@ on: # zizmor: ignore[concurrency-limits]
         type: string
         default: 'dsv4'
       framework:
-        description: "Serving framework; selects the <model-prefix>_fp4_b300_<framework>[_<collector-variant>].sh collector"
+        description: "Serving framework; selects the <model-prefix>_<precision>_b300_<framework>[_<collector-variant>].sh collector"
         required: false
         type: choice
         options: [vllm, sglang]
         default: 'vllm'
+      precision:
+        description: "Checkpoint precision; selects the collector and queue metadata"
+        required: false
+        type: choice
+        options: [fp4, fp8, bf16]
+        default: 'fp4'
       collector-variant:
-        description: "Collector variant suffix; non-empty selects <model-prefix>_fp4_b300_<framework>_<collector-variant>.sh (e.g. probabilistic_sample_method_block_rejection_sample_method)"
+        description: "Collector variant suffix; non-empty selects <model-prefix>_<precision>_b300_<framework>_<collector-variant>.sh"
         required: false
         type: string
         default: ''
@@ -97,7 +103,7 @@ env:
   HF_HUB_CACHE: '/mnt/hf_hub_cache/'
   MODEL: ${{ inputs.model }}
   MODEL_PREFIX: ${{ inputs.model-prefix }}
-  PRECISION: fp4
+  PRECISION: ${{ inputs.precision }}
   FRAMEWORK: ${{ inputs.framework }}
   EXP_NAME: ${{ inputs.model-prefix }}_speedbench
   IMAGE: ${{ inputs.image }}
@@ -155,11 +161,12 @@ jobs:
             --arg runner "$RUNNER" \
             --arg model_prefix "$MODEL_PREFIX" \
             --arg framework "$FRAMEWORK" \
+            --arg precision "$PRECISION" \
             '[{
               runner: $runner,
               framework: $framework,
               "model-prefix": $model_prefix,
-              precision: "fp4",
+              precision: $precision,
               "spec-decoding": "mtp"
             }]')
           scored=$(printf '%s' "$entry" |
@@ -250,7 +257,7 @@ jobs:
         run: |
           if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
           collector_dir=benchmarks/single_node/speedbench
-          collector="${MODEL_PREFIX}_fp4_b300_${FRAMEWORK}"
+          collector="${MODEL_PREFIX}_${PRECISION}_b300_${FRAMEWORK}"
           if [[ -n "$COLLECTOR_VARIANT" ]]; then
             collector+="_${COLLECTOR_VARIANT}"
           fi

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/stage-hf-checkpoint.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+source /infmax-workspace/benchmarks/check_env.sh
+check_env_vars HF_HUB_CACHE HF_CHECKPOINT_REPO HF_CHECKPOINT_REVISION HF_CHECKPOINT_BUDGET_BYTES
+
+if [[ ! "$HF_CHECKPOINT_REVISION" =~ ^[0-9a-f]{40}$ ]] ||
+   [[ ! "$HF_CHECKPOINT_BUDGET_BYTES" =~ ^[1-9][0-9]*$ ]]; then
+    echo "Checkpoint staging requires a commit SHA and a positive byte budget" >&2
+    exit 1
+fi
+mkdir -p "$HF_HUB_CACHE"
+cache_name="models--${HF_CHECKPOINT_REPO//\//--}"
+exec 9>"$HF_HUB_CACHE/.${cache_name}.download.lock"
+flock -x 9
+# Only staging goes online; the parent worker's serving environment is unchanged.
+HF_HUB_OFFLINE=0 TRANSFORMERS_OFFLINE=0 python3 - <<'PY'
+import os
+import shutil
+from pathlib import Path
+
+from huggingface_hub import HfApi, snapshot_download, try_to_load_from_cache
+
+repo = os.environ["HF_CHECKPOINT_REPO"]
+revision = os.environ["HF_CHECKPOINT_REVISION"]
+cache = os.environ["HF_HUB_CACHE"]
+budget = int(os.environ["HF_CHECKPOINT_BUDGET_BYTES"])
+info = HfApi().model_info(repo, revision=revision, files_metadata=True)
+files = info.siblings
+if info.sha != revision or not files or any(
+    type(file.size) is not int or file.size < 0 for file in files
+):
+    raise SystemExit("Checkpoint metadata must identify the pinned revision and every file size")
+
+
+def cached(file):
+    path = try_to_load_from_cache(repo, file.rfilename, revision=revision, cache_dir=cache)
+    return isinstance(path, str) and Path(path).is_file() and Path(path).stat().st_size == file.size
+
+
+total = sum(file.size for file in files)
+missing = [file for file in files if not cached(file)]
+remaining = sum(file.size for file in missing)
+# Do not credit old revisions or incomplete blobs. A complete snapshot needs no reserve.
+required = remaining + max(0, budget - total) if missing else 0
+available = shutil.disk_usage(cache).free
+print(f"Checkpoint disk: available={available} remaining={remaining} required={required}", flush=True)
+if available < required:
+    raise SystemExit("Insufficient shared-cache disk for the pinned checkpoint")
+if missing:
+    snapshot_download(repo, revision=revision, cache_dir=cache)
+    if any(not cached(file) for file in files):
+        raise SystemExit("Pinned checkpoint is incomplete after download")
+PY

inferencex-e2e/benchmarks/single_node/speedbench/glm53flash_fp8_b300_vllm.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -eo pipefail
+repo_root="$(cd "$(dirname "$0")/../../.." && pwd)"
+source "$repo_root/benchmarks/check_env.sh"
+check_env_vars \
+    CATEGORY CHAT_TEMPLATE_KWARGS_ON IMAGE MODEL MODEL_PATH MTP_LIST \
+    OUT_YAML PORT SPEEDBENCH_OUTPUT_LEN THINKING_MODES TP
+
+export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}"
+export VLLM_ENGINE_READY_TIMEOUT_S=3600
+export VLLM_USE_RUST_FRONTEND=0
+exec python3 -m infx.bench speedbench-al \
+    --concurrency 1 \
+    --temperature 1.0 \
+    --top-p 0.95 \
+    -- \
+    --tensor-parallel-size "$TP" --data-parallel-size 1 \
+    --trust-remote-code \
+    --gpu-memory-utilization 0.80 \
+    --max-model-len 32768 \
+    --max-num-batched-tokens 8192 \
+    --max-num-seqs 32 \
+    --kv-cache-dtype fp8 \
+    --tokenizer-mode auto \
+    --reasoning-parser glm47 \
+    --chat-template-content-format string \
+    --no-enable-prefix-caching

inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/agg-variants.yaml

@@ -0,0 +1,129 @@
+schema: 2
+
+base:
+  identity:
+    model:
+      revision: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c
+    container: {}
+    frameworks:
+      sglang: v0.5.21
+  resources:
+    gpu_type: mi325x
+    gpus_per_node: 8
+  setup_script: stage-hf-checkpoint.sh
+  environment:
+    HF_CHECKPOINT_REPO: zai-org/GLM-5.3-Flash
+    HF_CHECKPOINT_REVISION: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c
+    # The 328 GB checkpoint needs download and metadata headroom.
+    HF_CHECKPOINT_BUDGET_BYTES: '400000000000'
+  frontend:
+    type: sglang
+    enable_multiple_frontends: false
+  observability:
+    enabled: false
+    tachometer:
+      enabled: false
+  engine: sglang
+  roles:
+    agg:
+      nodes: 1
+      workers: 1
+      gpus: 4
+      env:
+        HF_HUB_OFFLINE: '1'
+        TRANSFORMERS_OFFLINE: '1'
+        PYTHONNOUSERSITE: '1'
+        SGLANG_TIMEOUT_KEEP_ALIVE: '900'
+        SGLANG_USE_AITER: '1'
+        AITER_ONLINE_TUNE: '0'
+        ROCM_QUICK_REDUCE_QUANTIZATION: 'NONE'
+        SGLANG_DSA_FUSE_TOPK: 'false'
+        SGLANG_OPT_USE_TOPK_V2: 'false'
+        SGLANG_OPT_FUSE_MHC_POST_PRE: '0'
+        SGLANG_GLM_NEXTN_MOE_PTPC: '0'
+        SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '0'
+      args:
+        served-model-name: zai-org/GLM-5.3-Flash
+        revision: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c
+        tokenizer-path: /mnt/hf_hub_cache/models--zai-org--GLM-5.3-Flash/snapshots/46a050bec13002ad29ee795c16e61cf4fa3c4a9c
+        trust-remote-code: true
+        tensor-parallel-size: 4
+        expert-parallel-size: 4
+        quantization: fp8
+        context-length: 1048576
+        page-size: 64
+        mem-fraction-static: 0.82
+        chunked-prefill-size: 8192
+        max-prefill-tokens: 8192
+        attention-backend: dsa
+        dsa-prefill-backend: tilelang
+        dsa-decode-backend: tilelang
+        dsa-topk-backend: torch
+        linear-attn-backend: triton
+        linear-attn-verify-backend: triton
+        kv-cache-dtype: bfloat16
+        moe-runner-backend: aiter
+        cuda-graph-backend-prefill: disabled
+        speculative-algorithm: EAGLE
+        speculative-num-steps: 5
+        speculative-eagle-topk: 1
+        speculative-num-draft-tokens: 6
+        speculative-attention-mode: prefill
+        speculative-moe-a2a-backend: none
+        reasoning-parser: glm45
+        tool-call-parser: glm47
+        watchdog-timeout: 1800
+        dist-timeout: 600
+        enable-metrics: true
+  health_check:
+    max_attempts: 1440
+    interval_seconds: 10
+  benchmark:
+    env:
+      HF_HUB_OFFLINE: ''
+      TRANSFORMERS_OFFLINE: ''
+      AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
+      AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
+      AIPERF_APPLY_CHAT_TEMPLATE: 'true'
+
+override_tp4_ep4_c1:
+  roles:
+    agg:
+      args:
+        max-running-requests: 2
+        cuda-graph-max-bs-decode: 2
+
+override_tp4_ep4_c2:
+  roles:
+    agg:
+      args:
+        max-running-requests: 4
+        cuda-graph-max-bs-decode: 4
+
+override_tp4_ep4_c4:
+  roles:
+    agg:
+      args:
+        max-running-requests: 8
+        cuda-graph-max-bs-decode: 8
+
+override_tp4_ep4_c8:
+  roles:
+    agg:
+      args:
+        max-running-requests: 16
+        cuda-graph-max-bs-decode: 16
+
+override_tp4_ep4_c12:
+  roles:
+    agg:
+      args:
+        max-running-requests: 24
+        cuda-graph-max-bs-decode: 24
+
+override_tp4_ep4_c16:
+  roles:
+    agg:
+      args:
+        max-running-requests: 32
+        cuda-graph-max-bs-decode: 32

inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/mi325x-fp8/agentx/hicache.yaml

@@ -0,0 +1,111 @@
+schema: 2
+
+base:
+  identity:
+    model:
+      revision: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c
+    container: {}
+    frameworks:
+      sglang: v0.5.22
+  resources:
+    gpu_type: mi325x
+    gpus_per_node: 8
+  setup_script: stage-hf-checkpoint.sh
+  environment:
+    HF_CHECKPOINT_REPO: zai-org/GLM-5.3-Flash
+    HF_CHECKPOINT_REVISION: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c
+    HF_CHECKPOINT_BUDGET_BYTES: '400000000000'
+  frontend:
+    type: sglang
+    enable_multiple_frontends: false
+  observability:
+    enabled: false
+    tachometer:
+      enabled: false
+  engine: sglang
+  roles:
+    agg:
+      nodes: 1
+      workers: 1
+      gpus: 4
+      env:
+        HF_HUB_OFFLINE: '1'
+        TRANSFORMERS_OFFLINE: '1'
+        PYTHONNOUSERSITE: '1'
+        SGLANG_TIMEOUT_KEEP_ALIVE: '900'
+        SGLANG_USE_AITER: '1'
+        AITER_ONLINE_TUNE: '0'
+        ROCM_QUICK_REDUCE_QUANTIZATION: 'NONE'
+        SGLANG_DSA_FUSE_TOPK: 'false'
+        SGLANG_OPT_USE_TOPK_V2: 'false'
+        SGLANG_OPT_FUSE_MHC_POST_PRE: '0'
+        SGLANG_GLM_NEXTN_MOE_PTPC: '0'
+        SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '0'
+      args:
+        served-model-name: zai-org/GLM-5.3-Flash
+        revision: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c
+        tokenizer-path: /mnt/hf_hub_cache/models--zai-org--GLM-5.3-Flash/snapshots/46a050bec13002ad29ee795c16e61cf4fa3c4a9c
+        trust-remote-code: true
+        tensor-parallel-size: 4
+        expert-parallel-size: 4
+        quantization: fp8
+        context-length: 1048576
+        page-size: 64
+        mem-fraction-static: 0.82
+        chunked-prefill-size: 8192
+        max-prefill-tokens: 8192
+        attention-backend: dsa
+        dsa-prefill-backend: tilelang
+        dsa-decode-backend: tilelang
+        dsa-topk-backend: torch
+        linear-attn-backend: triton
+        linear-attn-verify-backend: triton
+        kv-cache-dtype: bfloat16
+        moe-runner-backend: aiter
+        cuda-graph-backend-prefill: disabled
+        speculative-algorithm: EAGLE
+        speculative-num-steps: 5
+        speculative-eagle-topk: 1
+        speculative-num-draft-tokens: 6
+        speculative-attention-mode: prefill
+        speculative-moe-a2a-backend: none
+        reasoning-parser: glm45
+        tool-call-parser: glm47
+        watchdog-timeout: 1800
+        dist-timeout: 600
+        enable-metrics: true
+  health_check:
+    max_attempts: 1440
+    interval_seconds: 10
+  benchmark:
+    env:
+      HF_HUB_OFFLINE: ''
+      TRANSFORMERS_OFFLINE: ''
+      AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
+      AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
+      AIPERF_APPLY_CHAT_TEMPLATE: 'true'
+
+zip_override_off:
+  roles:
+    agg:
+      args:
+        max-running-requests: [24, 32, 48, 64]
+        cuda-graph-max-bs-decode: [24, 32, 48, 64]
+  benchmark:
+    env:
+      CONC: ['12', '16', '24', '32']
+
+zip_override_on:
+  roles:
+    agg:
+      args:
+        max-running-requests: [24, 32, 48, 64]
+        cuda-graph-max-bs-decode: [24, 32, 48, 64]
+        enable-hierarchical-cache: true
+        # Keep size and ratio unset so the combined cgroup-aware guard stays active.
+        hicache-host-memory-fraction: 0.60
+        hicache-io-backend: direct
+        hicache-mem-layout: page_first_direct
+  benchmark:
+    env:
+      CONC: ['12', '16', '24', '32']