Selected recipe

Runtime: Container (community, ARM64, custom B12x). Run on every node after setting `HF_TOKEN`; replace the node-specific placeholders. Container image: eugr/spark-vllm-b12x:nightly-20261015

# Example two-Spark values (adjust to your network):
#   HEAD_IP="10.100.32.1"        # head node IP on the ETH subnet
#   ETH_IF="enp1s0f0np0"        # ConnectX baseboard ethernet (socket/control)
#   IB_IF="rocep1s0f0,roceP2p1s0f0"   # RoCE HCA names (data plane) - use both "twins"!
export HEAD_IP="<head-connectx-ip>"
export ETH_IF="<connectx-interface>"
export IB_IF="<roce-interface-1>,<roce-interface-2>"
export MASTER_PORT=29501
export IMAGE="eugr/spark-vllm-b12x:nightly-20261015"
export HF_CACHE_DIR="$HOME/.cache/huggingface"

mkdir -p "$HF_CACHE_DIR" "$HOME/.cache/vllm" "$HOME/.cache/flashinfer" "$HOME/.triton" "$HOME/.cache/b12x"

docker run --rm -it \
  --privileged \
  --gpus all \
  --ipc host \
  --network host \
  --shm-size=16g \
  --ulimit nofile=1048576:1048576 \
  --ulimit memlock=-1 \
  --ulimit stack=67108864 \
  -e HF_TOKEN=$HF_TOKEN \
  -e MN_IF_NAME="$ETH_IF" \
  -e UCX_NET_DEVICES="$ETH_IF" \
  -e NCCL_SOCKET_IFNAME="$ETH_IF" \
  -e NCCL_IB_HCA="$IB_IF" \
  -e NCCL_IB_DISABLE=0 \
  -e OMPI_MCA_btl_tcp_if_include="$ETH_IF" \
  -e GLOO_SOCKET_IFNAME="$ETH_IF" \
  -e TP_SOCKET_IFNAME="$ETH_IF" \
  -e NCCL_IGNORE_CPU_AFFINITY=1 \
  -e HEAD_IP="$HEAD_IP" \
  -e MASTER_PORT="$MASTER_PORT" \
  -v "$HF_CACHE_DIR:/root/.cache/huggingface" \
  -v "$HOME/.cache/vllm:/root/.cache/vllm" \
  -v "$HOME/.cache/flashinfer:/root/.cache/flashinfer" \
  -v "$HOME/.triton:/root/.triton" \
  -v $HOME/.cache/b12x:/root/.cache/b12x \
  --entrypoint /bin/bash \
  "$IMAGE"

export CUTE_DSL_ARCH="sm_121a"
export SAFETENSORS_FAST_GPU="1"
export VLLM_ENABLE_ROCE_ALLREDUCE="1"
export VLLM_ROCE_ALLREDUCE_MAX_SIZE="2MB"
export NCCL_BUFFSIZE=1048576
export NCCL_MAX_CTAS=8
export VLLM_WORKER_MULTIPROC_METHOD="spawn"
export VLLM_SSM_CONV_STATE_LAYOUT="DS"
export VLLM_USE_AOT_COMPILE="1"
export VLLM_USE_MEGA_AOT_ARTIFACT="1"
export VLLM_USE_V2_MODEL_RUNNER="1"
export VLLM_ENABLE_PCIE_ALLREDUCE="0"
export B12X_POLICY_MODE="auto"
export VLLM_B12X_MOE_FP4_FORCE_A16=1
export B12X_W4A16_A4_PREFILL=1

vllm serve nvidia/GLM-5.3-Flash-FP4-Local \
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 2 \
  --pipeline-parallel-size 1 \
  --decode-context-parallel-size 1 \
  --mamba-cache-mode align \
  --enable-prefix-caching \
  --enable-chunked-prefill \
  --dtype bfloat16 \
  --kv-cache-dtype fp8 \
  --quantization modelopt_mixed \
  --attention-backend B12X \
  --block-size 256 \
  --moe-backend b12x \
  --linear-backend b12x \
  --no-enable-flashinfer-autotune \
  --load-format b12x \
  --max-model-len 500000 \
  --max-num-seqs 4 \
  --max-num-batched-tokens 4096 \
  --speculative-config '{"method":"mtp","num_speculative_tokens":5,"moe_backend":"humming","attention_backend":"B12X"}' \
  --reasoning-parser glm45 \
  --tool-call-parser glm47 \
  --enable-auto-tool-choice \
  --gpu-memory-utilization 0.87 \
  --nnodes 2 \
  --node-rank 0 \
  --master-addr "$HEAD_IP" \
  --master-port "$MASTER_PORT"

export CUTE_DSL_ARCH="sm_121a"
export SAFETENSORS_FAST_GPU="1"
export VLLM_ENABLE_ROCE_ALLREDUCE="1"
export VLLM_ROCE_ALLREDUCE_MAX_SIZE="2MB"
export NCCL_BUFFSIZE=1048576
export NCCL_MAX_CTAS=8
export VLLM_WORKER_MULTIPROC_METHOD="spawn"
export VLLM_SSM_CONV_STATE_LAYOUT="DS"
export VLLM_USE_AOT_COMPILE="1"
export VLLM_USE_MEGA_AOT_ARTIFACT="1"
export VLLM_USE_V2_MODEL_RUNNER="1"
export VLLM_ENABLE_PCIE_ALLREDUCE="0"
export B12X_POLICY_MODE="auto"
export VLLM_B12X_MOE_FP4_FORCE_A16=1
export B12X_W4A16_A4_PREFILL=1

vllm serve nvidia/GLM-5.3-Flash-FP4-Local \
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 2 \
  --pipeline-parallel-size 1 \
  --decode-context-parallel-size 1 \
  --mamba-cache-mode align \
  --enable-prefix-caching \
  --enable-chunked-prefill \
  --dtype bfloat16 \
  --kv-cache-dtype fp8 \
  --quantization modelopt_mixed \
  --attention-backend B12X \
  --block-size 256 \
  --moe-backend b12x \
  --linear-backend b12x \
  --no-enable-flashinfer-autotune \
  --load-format b12x \
  --max-model-len 500000 \
  --max-num-seqs 4 \
  --max-num-batched-tokens 4096 \
  --speculative-config '{"method":"mtp","num_speculative_tokens":5,"moe_backend":"humming","attention_backend":"B12X"}' \
  --reasoning-parser glm45 \
  --tool-call-parser glm47 \
  --enable-auto-tool-choice \
  --gpu-memory-utilization 0.87 \
  --nnodes 2 \
  --node-rank 1 \
  --master-addr "$HEAD_IP" \
  --master-port "$MASTER_PORT" \
  --headless