topology#

Explicit distributed topology for vLLM runtime measurements.

Classes

RuntimeTopology

class RuntimeTopology#

Bases: object

__init__(tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, prefill_context_parallel_size=1, decode_context_parallel_size=1, enable_expert_parallel=False, distributed_executor_backend='mp', gpu_group_size=1)#
Parameters:

  • tensor_parallel_size (int)

  • pipeline_parallel_size (int)

  • data_parallel_size (int)

  • prefill_context_parallel_size (int)

  • decode_context_parallel_size (int)

  • enable_expert_parallel (bool)

  • distributed_executor_backend (str)

  • gpu_group_size (int)

Return type:

None

data_parallel_size: int = 1#
decode_context_parallel_size: int = 1#
distributed_executor_backend: str = 'mp'#
enable_expert_parallel: bool = False#
classmethod from_config(config)#
Parameters:

config (Any)

Return type:

RuntimeTopology

gpu_group_size: int = 1#
pipeline_parallel_size: int = 1#
prefill_context_parallel_size: int = 1#
tensor_parallel_size: int = 1#
to_dict()#
Return type:

dict[str, Any]

validate_model_dimensions(*, num_attention_heads, num_key_value_heads)#
Parameters:

  • num_attention_heads (int)

  • num_key_value_heads (int)

Return type:

None

property world_size: int#