runtime: trtllm
compile_backend: torch-cudagraph
max_seq_len: 2048
max_num_tokens: 2048
max_batch_size: 512
cuda_graph_batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
world_size: 8
enable_chunked_prefill: true
model_factory: AutoModelForCausalLM
kv_cache_config:
  enable_block_reuse: false
  free_gpu_memory_fraction: 0.95
  tokens_per_block: 64
model_kwargs:
  torch_dtype: bfloat16
transforms:
  export_to_gm:
    num_moe_experts_for_export: 2