System configuration example

In this section, we introduce the concept of the system schema, explain the meaning of each field, and describe how the fields should be used. The system schema is a TOML file that allows users to define a system’s configuration.

name = "example-cluster"
scheduler = "slurm"

install_path = "./install"
output_path = "./results"
default_partition = "partition_1"

mpi = "pmix"
gpus_per_node = 8
ntasks_per_node = 8

cache_docker_images_locally = true
docker_image_import_extra_sbatch_args = ["--time=00:20:00"]

[[partitions]]
name = "partition_1"

  [[partitions.groups]]
  name = "group_a"
  nodes = ["node-[001-025]"]

  [[partitions.groups]]
  name = "group_b"
  nodes = ["node-[026-050]"]

[[partitions]]
name = "partition_2"

[global_env_vars]
# NCCL Specific Configurations
NCCL_IB_GID_INDEX = "3"
NCCL_IB_TIMEOUT = "20"
NCCL_IB_QPS_PER_CONNECTION = "4"

# Device Visibility Configuration
MELLANOX_VISIBLE_DEVICES = "0,3,4,5,6,9,10,11"
CUDA_VISIBLE_DEVICES = "0,1,2,3,4,5,6,7"