[TileRT] GLM-5.3 MI355X AgentX parallel KV inject experiments, reverted pending tile-ai/TileRT#66 / GLM-5.3 MI355X AgentX 并行 KV 注入实验（已回退，待 tile-ai/TileRT#66）

MODELS.md

@@ -155,6 +155,7 @@ Other offloading tiers, including NVMe KV cache offloading, are outside the init
 | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios |
 |---|---|---|---|---|
 | DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | Agentic coding (vLLM: DSpark, Engram UVA offload; SGLang: DSpark arms added per SKU from 2026-09-17; GPU validation pending) | |
+| GLM-5.3 | `glm5.3` | 2026-09-22 ([#3366](https://github.com/SemiAnalysisAI/InferenceX/pull/3366)) | Agentic coding (MTP only, per the Deprecation Notice) | Single-turn 1k1k (deprecated for all models before this model was added; never run) |
 | Qwen3.8-Flash-Next | `qwen3.8next` | 2026-08-26 ([#2742](https://github.com/SemiAnalysisAI/InferenceX/pull/2742)) | Agentic coding | |
 | Kimi-K3 | `kimik3` | 2026-07-27 ([#2391](https://github.com/SemiAnalysisAI/InferenceX/pull/2391)) | Agentic coding (DSpark may be disabled for better Pareto points) | Standalone non-DSpark A/B baseline (not required from day 0) |
 | GLM-5.2 | `glm5.2` | 2026-07-18 ([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | Agentic coding (non-MTP points remain eligible under the Pareto policy; see Deprecation Notice) | |

MODELS_zh.md

@@ -155,6 +155,7 @@ InferenceX 支持 SGLang 和 vLLM 双方的维护者，并响应 AI 实验室和
 | 模型架构类别 | 前缀 | 加入日期 | 启用场景 | 已弃用场景 |
 |---|---|---|---|---|
 | DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | 智能体编码（DSpark、Engram UVA 卸载；GPU 待验证） | |
+| GLM-5.3 | `glm5.3` | 2026-09-22（[#3366](https://github.com/SemiAnalysisAI/InferenceX/pull/3366)） | 智能体编码（仅 MTP，见弃用公告） | 单轮 1k1k（在本模型加入前已对所有模型弃用，从未运行） |
 | Qwen3.8-Flash-Next | `qwen3.8next` | 2026-08-26（[#2742](https://github.com/SemiAnalysisAI/InferenceX/pull/2742)） | 智能体编码 | |
 | Kimi-K3 | `kimik3` | 2026-07-27 ([#2391](https://github.com/SemiAnalysisAI/InferenceX/pull/2391)) | 智能体编码（可关闭 DSpark 以获得更优帕累托点） | 独立非 DSpark A/B 基线（自第 0 天起即不要求） |
 | GLM-5.2 | `glm5.2` | 2026-07-18（[#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)） | 智能体编码（非 MTP 数据点仍可按帕累托策略参与发布；见弃用公告） | |

benchmarks/benchmark_lib.sh

@@ -3162,7 +3162,7 @@ resolve_trace_source() {
     # corpus; 1M-context families take the unfiltered variant, others 256k.
     local default_loader
     case "${MODEL_PREFIX:-}" in
-        dsv4*|glm5.2*|minimaxm3*|kimik3*)
+        dsv4*|glm5.2*|glm5.3*|minimaxm3*|kimik3*)
             default_loader="semianalysis_cc_traces_weka_062126"
             ;;
         *)

benchmarks/multi_node/agentic/glm5.3_fp8_mi355x_tilert.sh

@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../../benchmark_lib.sh"
+
+check_env_vars \
+    CONC_LIST \
+    ISL \
+    OSL \
+    IMAGE \
+    SPEC_DECODING \
+    MODEL_PATH \
+    MODEL_NAME \
+    PREFILL_NUM_WORKERS \
+    PREFILL_TP \
+    PREFILL_EP \
+    PREFILL_DP_ATTN \
+    DECODE_NUM_WORKERS \
+    DECODE_TP \
+    DECODE_EP \
+    DECODE_DP_ATTN \
+    PREFILL_NODES \
+    DECODE_NODES \
+    RANDOM_RANGE_RATIO \
+    DURATION \
+    MODEL_PREFIX \
+    PRECISION \
+    RESULT_FILENAME \
+    KV_OFFLOADING \
+    IS_AGENTIC \
+    FRAMEWORK \
+    PREFILL_IMAGE
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+  echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+set -x
+
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1
+
+export TIME_LIMIT=08:00:00
+export MODEL_PATH=$MODEL_PATH
+export MODEL_NAME=$MODEL_NAME
+export CONTAINER_IMAGE=$IMAGE
+export PREFILL_IMAGE
+
+export RESULT_FILENAME
+
+if [[ "$PREFILL_NODES" -ne 1 || "$DECODE_NODES" -ne 1 || \
+      "$PREFILL_NUM_WORKERS" -ne 1 || "$DECODE_NUM_WORKERS" -ne 1 ]]; then
+    echo "Error: tilert supports exactly 1 prefill node/worker + 1 decode node/worker" \
+         "(got PREFILL_NODES=$PREFILL_NODES x$PREFILL_NUM_WORKERS, DECODE_NODES=$DECODE_NODES x$DECODE_NUM_WORKERS)" >&2
+    exit 1
+fi
+
+if [[ "$KV_OFFLOADING" != "none" ]]; then
+    echo "Error: tilert has no KV offload backend; kv-offloading must be 'none' (got '$KV_OFFLOADING')" >&2
+    exit 1
+fi
+
+# TileRT configuration. Every value is explicit here: server_tilert.sh
+# validates each one with check_env_vars and supplies no defaults of its own.
+export TILERT_VERSION=0.1.6.post1
+export TILERT_PROFILE=glm5_2          # decode_server --model (TileRT model profile)
+export TILERT_MODEL_TYPE=glm-5        # weight_converter --model_type (fallback converter)
+export TILERT_MODEL_PKG=glm_5_2_rocm  # per-model converter package, preferred when importable
+export SERVED_MODEL_NAME=glm5_2
+# GLM-5.3's full context window (config.json max_position_embeddings), as every
+# in-tree GLM-5.2 recipe serves. (202752 was GLM-5.1's, inherited from the B200
+# TileRT recipe this mirrors.)
+#
+# Memory at this context, per rank, bf16 wire layout (verified against the
+# tilert 0.1.6 and vLLM 0.24.0 sources and the MI355X logs, 287.98 GiB cards;
+# the undivided PD buffer sizes are what 0.1.6 allocated on one card):
+#   decode : weights 90.72 GiB + engine cache window 93.25 GiB
+#            + PD receive buffer 99.06 GiB (receive_server.py, dense in max_seq_len)
+#   prefill: weights 90.45 GiB + profiling/non-torch 40.3 GiB + vLLM KV 91.71 GiB
+#            + PD staging buffer 99.06 GiB (prefill_connector.py, TP rank 0,
+#              allocated OUTSIDE vLLM's gpu-memory-utilization budget)
+# Undivided, neither side starts: the decode rank is node-marginal (~283 of
+# 288 GiB) and the prefill rank cannot fit at any utilization (~321 GiB).
+# tilert 0.1.6.post1 keeps both buffers on the GPU but shards them by layer
+# across the eight devices (layer lid on device lid % 8, TILERT_PD_SHARDS,
+# default on), so each card holds 12.54 GiB instead of 99.06 GiB on one.
+# convert() dequantises each layer on the device that received it, which spreads
+# its transients too (108.6 KiB/token, 82.9 GiB at 800k tokens) instead of
+# leaving them on cuda:0. Measured on 2x8 MI350X at this context with bf16 KV:
+# decode peaks at 202.1 GiB per card, prefill at 269.2 GiB of 287.69 GiB, and
+# the KV path stays device-to-device at 108 GB/s (81 GB in 751 ms, 54% of the
+# 4x400 GbE line rate). No host hop and no patch: the wheel runs as shipped.
+export TILERT_MAX_MODEL_LEN=1048576
+export TILERT_TRANSPORT=mooncake
+export TILERT_PARSER=none
+export TILERT_RDMA_STRICT=0
+export TILERT_CONVERT_LOCK_WAIT=21600
+export TILERT_SIMULATE_ACC_METHOD=match-expected
+export TILERT_WEIGHTS_DIR="/models/${MODEL_NAME}-tilert-tp${DECODE_TP}"
+# bf16 MLA KV on both roles. This is the only layout TileRT 0.1.6 can consume
+# from vLLM on ROCm: MlaNsaProfile.classify_layers infers the layout from the
+# cache tensor stride and accepts exactly 1152 B/token (bf16) or 656 B/token
+# (fp8_ds_mla). vLLM's ROCM_AITER_MLA_SPARSE backend has no fp8_ds_mla; its
+# plain "fp8" writes a flat 576 B/token row, which the connector rejects at
+# register_kv_caches. Explicit bfloat16 rather than auto so the stride does not
+# depend on the model dtype. Never float16: it passes the 1152 B check and is
+# then read as bf16.
+export PREFILL_KV_DTYPE=bfloat16
+# The ROCm backend supports block sizes [1, 64] and vLLM picks 1, which makes
+# the connector's KI plane copy fail and MLA address the wrong rows.
+export PREFILL_BLOCK_SIZE=64
+export DECODE_KV_DTYPE=bf16
+# The PD staging shard sits outside vLLM's budget, so vLLM needs 90.45 (weights)
+# + 40.3 (profiling) + 91.71 GiB (KV for one 1048576-token request) = 222.5 GiB
+# inside it: 0.85 x 287.98 = 244.8 GiB leaves 22 GiB of KV margin and 43 GiB
+# outside the budget for the 12.54 GiB staging shard plus the ~6.3 GiB non-torch
+# baseline measured on the decode OOM node (287.98 - 95.94 free - 184.17 - 1.58
+# reserved). 0.75 (216 GiB) refuses with "91.71 GiB KV cache is needed ...
+# available 85.25 GiB".
+export GPU_MEM_UTIL=0.85
+export SKIP_CONTAINER_BARRIER=0
+# Two images, one per rank, ~32 GB each. On a node that has neither cached the
+# pull alone outlasts the SGLang path's 300s default and the 1800s this script
+# used to hardcode, and the rank that comes up first waits out the whole
+# timeout while its peer is still pulling.
+export CONTAINER_BARRIER_TIMEOUT=5400
+export ROUTER_PORT=30000
+export PREFILL_PORT=8000
+export DECODE_CTRL_PORT=5556
+export DECODE_HTTP_PORT=5557
+export DECODE_WAIT=7200               # prefill waits for the decode ctrl port
+export PREFILL_WAIT=3600              # prefill waits for its own vLLM port
+export ROUTER_WAIT=10800              # decode waits for the router port to open
+
+if [[ "$SPEC_DECODING" == "mtp" ]]; then
+    # TileRT decode drafts at depth 3 (the only depth the ROCm GLM profile
+    # builds) and the golden acceptance curve is keyed on it. The vLLM prefill
+    # rank only has to materialise the MTP layer's KV, so it runs at 1.
+    export DECODE_MTP_SIZE=3
+    export PREFILL_SPEC_TOKENS=1
+else
+    export DECODE_MTP_SIZE=0
+    export PREFILL_SPEC_TOKENS=0
+fi
+export TILERT_QUEUE_TIMEOUT=1800  # requests wait on the bs=1 decode engine
+export THINKING_MODE=thinking_on
+
+if [[ "$PREFILL_EP" -ne 1 || "$DECODE_EP" -ne 1 || \
+      "$PREFILL_DP_ATTN" == "true" || "$DECODE_DP_ATTN" == "true" ]]; then
+    echo "Error: tilert runs pure TP8 on both roles; ep must be 1 and dp-attn false" >&2
+    exit 1
+fi
+export PREFILL_ENABLE_EP=false
+export PREFILL_ENABLE_DP=false
+export DECODE_ENABLE_EP=false
+export DECODE_ENABLE_DP=false
+
+JOB_ID=$(bash ./submit.sh $PREFILL_NODES \
+    $PREFILL_NUM_WORKERS \
+    $DECODE_NODES \
+    $DECODE_NUM_WORKERS \
+    $ISL $OSL "${CONC_LIST// /x}" inf \
+    ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \
+    ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \
+    ${PREFILL_TP} ${DECODE_TP} \
+    ${RANDOM_RANGE_RATIO})
+
+if [[ $? -ne 0 ]]; then
+    echo "Failed to submit job" >&2
+    exit 1
+fi
+
+echo "$JOB_ID"

benchmarks/multi_node/amd_utils/env.sh

@@ -1,10 +1,18 @@
 #!/bin/bash
 
 source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only
-check_env_vars \
-    MORI_IO_SQ_BACKOFF_TIMEOUT_US MORI_IO_QP_MAX_SEND_WR MORI_IO_QP_MAX_CQE MORI_IO_QP_MAX_SGE MORI_IO_TC_DISABLE \
-    UCX_IB_GID_INDEX MORI_APP_LOG_LEVEL SGLANG_ROUTER_STDOUT_LOGS TORCH_NCCL_BLOCKING_WAIT NCCL_BLOCKING_WAIT \
-    SGLANG_OPT_USE_AITER_INDEXER
+check_env_vars ENGINE
+# MoRI-IO queue-pair tuning, the UCX RoCE GID index, SGLang router logging and the
+# SGLang decode cuda-graph NCCL workaround. Only the SGLang and vLLM MoRI KV paths
+# below read these. ENGINE=tilert moves KV over mooncake and starts no SGLang
+# router, so it is neither given nor reads them: validating them there would force
+# the recipe to invent MoRI tuning for a transport it never uses.
+if [[ "$ENGINE" != "tilert" ]]; then
+    check_env_vars \
+        MORI_IO_SQ_BACKOFF_TIMEOUT_US MORI_IO_QP_MAX_SEND_WR MORI_IO_QP_MAX_CQE MORI_IO_QP_MAX_SGE MORI_IO_TC_DISABLE \
+        UCX_IB_GID_INDEX MORI_APP_LOG_LEVEL SGLANG_ROUTER_STDOUT_LOGS TORCH_NCCL_BLOCKING_WAIT NCCL_BLOCKING_WAIT \
+        SGLANG_OPT_USE_AITER_INDEXER
+fi
 # Dual-engine environment setup for multi-node disaggregated serving.
 #
 # ENGINE=sglang-disagg or vllm-disagg selects the engine-specific block.
@@ -180,6 +188,9 @@ $1 == "DSCP" && $2 == ":" && $NF == p {
     set +x
     echo "[INFO] IBDEVICES=$IBDEVICES  UCX_NET_DEVICES=$UCX_NET_DEVICES  NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME  UCX_IB_GID_INDEX=$UCX_IB_GID_INDEX  UCX_IB_TRAFFIC_CLASS=${UCX_IB_TRAFFIC_CLASS:-unset}"
 
+elif [[ "$ENGINE" == "tilert" ]]; then
+    echo "[INFO] tilert: IBDEVICES=$IBDEVICES  NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME  NCCL_IB_HCA=$NCCL_IB_HCA"
+
 else
 
     export SGLANG_USE_AITER=1

benchmarks/multi_node/amd_utils/job.slurm

@@ -38,6 +38,8 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then
     MODELS_YAML="$(pwd)/models_vllm.yaml"
 elif [[ "$ENGINE" == "atom-disagg" ]]; then
     MODELS_YAML="$(pwd)/models_atom.yaml"
+elif [[ "$ENGINE" == "tilert" ]]; then
+    MODELS_YAML="$(pwd)/models_tilert.yaml"
 else
     MODELS_YAML="$(pwd)/models.yaml"
 fi
@@ -52,6 +54,17 @@ if [[ -z "${DOCKER_IMAGE_NAME:-}" ]]; then
     exit 1
 fi
 
+if [[ "$ENGINE" == "tilert" && -z "${PREFILL_IMAGE:-}" ]]; then
+    echo "Error: ENGINE=tilert requires PREFILL_IMAGE (e.g. PREFILL_IMAGE=vllm/vllm-openai-rocm:nightly-<sha> in prefill.additional-settings)."
+    exit 1
+fi
+if [[ "$ENGINE" == "tilert" ]]; then
+    # server_tilert.sh takes the container-creation barrier timeout from the
+    # recipe (no 300s default as on the SGLang path); fail here, before sbatch
+    # work is done, rather than inside the container.
+    check_env_vars CONTAINER_BARRIER_TIMEOUT
+fi
+
 # Resolve the models.yaml entry the same way server_sglang.sh does: agentic runs
 # (IS_AGENTIC) use the '<model>-AgentX' recipe, non-agentic disaggregated runs use
 # '<model>-DI'. Fall back to the bare model name if the variant key is absent.
@@ -522,6 +535,41 @@ elif [[ "$ENGINE" == "atom-disagg" ]]; then
         -e EXTRA_SERVER_ARGS=\${EXTRA_SERVER_ARGS:-}
         -e IBDEVICES=${IBDEVICES:-}
     )
+elif [[ "$ENGINE" == "tilert" ]]; then
+    DOCKER_ENV_ENGINE=(
+        -e MODEL_PATH=$DOCKER_MODEL_PATH
+        -e PREFILL_IMAGE=${PREFILL_IMAGE}
+        -e TILERT_VERSION=${TILERT_VERSION}
+        -e TILERT_PROFILE=${TILERT_PROFILE}
+        -e TILERT_MODEL_TYPE=${TILERT_MODEL_TYPE}
+        -e TILERT_MODEL_PKG=${TILERT_MODEL_PKG}
+        -e TILERT_MAX_MODEL_LEN=${TILERT_MAX_MODEL_LEN}
+        -e TILERT_TRANSPORT=${TILERT_TRANSPORT}
+        -e TILERT_PARSER=${TILERT_PARSER}
+        -e TILERT_QUEUE_TIMEOUT=${TILERT_QUEUE_TIMEOUT}
+        -e TILERT_WEIGHTS_DIR=${TILERT_WEIGHTS_DIR}
+        -e TILERT_RDMA_STRICT=${TILERT_RDMA_STRICT}
+        -e TILERT_CONVERT_LOCK_WAIT=${TILERT_CONVERT_LOCK_WAIT}
+        -e TILERT_SIMULATE_ACC_METHOD=${TILERT_SIMULATE_ACC_METHOD}
+        -e \"TILERT_EXTRA_ENV=${TILERT_EXTRA_ENV:-}\"
+        -e SERVED_MODEL_NAME=${SERVED_MODEL_NAME}
+        -e PREFILL_KV_DTYPE=${PREFILL_KV_DTYPE}
+        -e PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE}
+        -e PREFILL_SPEC_TOKENS=${PREFILL_SPEC_TOKENS}
+        -e DECODE_KV_DTYPE=${DECODE_KV_DTYPE}
+        -e GPU_MEM_UTIL=${GPU_MEM_UTIL}
+        -e PREFILL_PORT=${PREFILL_PORT}
+        -e DECODE_CTRL_PORT=${DECODE_CTRL_PORT}
+        -e DECODE_HTTP_PORT=${DECODE_HTTP_PORT}
+        -e DECODE_WAIT=${DECODE_WAIT}
+        -e PREFILL_WAIT=${PREFILL_WAIT}
+        -e ROUTER_WAIT=${ROUTER_WAIT}
+        -e SKIP_CONTAINER_BARRIER=${SKIP_CONTAINER_BARRIER}
+        # Golden-acceptance selection on agentic MTP runs; unset elsewhere.
+        -e THINKING_MODE=${THINKING_MODE:-}
+        -e IBDEVICES=${IBDEVICES:-}
+        -e PYTHONPYCACHEPREFIX=/tmp/pycache
+    )
 else
     DOCKER_ENV_ENGINE=(
         -e SGLANG_WS_PATH=${WS_PATH}
@@ -704,6 +752,12 @@ if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \
     set +e
 fi
 
+RANK_IMAGE=
+if [[ \"$ENGINE\" == \"tilert\" && \"\$SLURM_PROCID\" -lt \"$xP\" ]]; then
+    RANK_IMAGE=\"$PREFILL_IMAGE\"
+    echo \"[tilert] rank \$SLURM_PROCID is a prefill rank; using PREFILL_IMAGE=\$RANK_IMAGE\"
+fi
+
 \$MAYBE_EXEC \$DOCKER_CMD run \
     --init \
     --stop-timeout 10 \
@@ -745,7 +799,7 @@ fi
     ${CLIENT_DOCKER_ENV} \
     --name \"$DOCKER_CONT_NAME\" \
     --entrypoint \"\" \
-    \"$DOCKER_IMAGE_NAME\" bash -lc '
+    \"\${RANK_IMAGE:-$DOCKER_IMAGE_NAME}\" bash -lc '
         set -o pipefail
         mkdir -p /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"'
         '"$RUN_FILE_FULL"' 2>&1 | tee /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"'/server_\$(hostname).log

benchmarks/multi_node/amd_utils/models_tilert.yaml

@@ -0,0 +1,6 @@
+# Model-owned engine settings for ENGINE=tilert. Everything the caller owns
+# (profile, topology, ports, dtypes, draft depth) is set by the recipe, not here.
+GLM-5.3:
+  prefill_env: "VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_ENGINE_READY_TIMEOUT_S=10800"
+  prefill_extra_flags: ""
+  decode_extra_flags: ""

benchmarks/multi_node/amd_utils/server.sh

@@ -6,6 +6,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only
 #   ENGINE=sglang-disagg (default) -> server_sglang.sh (SGLang + MoRI)
 #   ENGINE=vllm-disagg             -> server_vllm.sh  (vLLM + Nixl/MoRI-IO)
 #   ENGINE=atom-disagg             -> server_atom.sh  (ATOM + mooncake)
+#   ENGINE=tilert                  -> server_tilert.sh (vLLM prefill + TileRT decode)
 
 check_env_vars ENGINE WS_PATH
 if [[ -f /config/hicache_mc.env ]]; then
@@ -22,6 +23,8 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then
 elif [[ "$ENGINE" == "atom-disagg" ]]; then
     export ATOM_WS_PATH="$WS_PATH"
     source "$WS_PATH/server_atom.sh"
+elif [[ "$ENGINE" == "tilert" ]]; then
+    source "$WS_PATH/server_tilert.sh"
 else
     source "$WS_PATH/server_sglang.sh"
 fi