Index

_

__init__() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) (nvidia_resiliency_ext.ptl_resiliency.StragglerDetectionCallback method)

__str__() (nvidia_resiliency_ext.straggler.straggler.CallableId method)

A

Abort (class in nvidia_resiliency_ext.inprocess.abort) AbortTorchDistributed (class in nvidia_resiliency_ext.inprocess.abort) add_finalize_fn() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest method) async_fn (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) (nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt.TorchAsyncCheckpoint attribute)

async_fn_args (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) async_fn_kwargs (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) async_save() (nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt.TorchAsyncCheckpoint method) AsyncCallsQueue (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) AsyncRequest (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) atomic() (nvidia_resiliency_ext.inprocess.CallWrapper method)

B

base_delay (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.SimulatedFaultParams attribute)

BaseCheckpointManager (class in nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager) BasicTensorAwareStateDict (class in nvidia_resiliency_ext.checkpointing.local.basic_state_dict)

C

calculate_and_set_timeouts() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) CallableId (class in nvidia_resiliency_ext.straggler.straggler) CallWrapper (class in nvidia_resiliency_ext.inprocess) CheckpointingException CliqueReplicationStrategy (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies) close() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method)

Compose (class in nvidia_resiliency_ext.inprocess) copy_tensors_to_cpu() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method) CudaHealthCheck (class in nvidia_resiliency_ext.inprocess.health_check) cupti_manager (nvidia_resiliency_ext.straggler.straggler.Detector attribute) custom_sections (nvidia_resiliency_ext.straggler.straggler.Detector attribute) CustomSection (class in nvidia_resiliency_ext.straggler.straggler)

D

detection_section() (nvidia_resiliency_ext.straggler.straggler.Detector class method)

Detector (class in nvidia_resiliency_ext.straggler.straggler) DistributedAsyncCaller (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core)

E

execute_sync() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest method)

F

fault_type (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.SimulatedFaultParams attribute) FaultCounter (class in nvidia_resiliency_ext.inprocess.health_check) FaultCounterExceeded FaultToleranceCallback (class in nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback) FaultToleranceConfig (class in nvidia_resiliency_ext.fault_tolerance.config) FillGaps (class in nvidia_resiliency_ext.inprocess.rank_assignment) FilterGroupedByKey (class in nvidia_resiliency_ext.inprocess.rank_assignment)

Finalize (class in nvidia_resiliency_ext.inprocess.finalize) finalize_async_save() (nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt.TorchAsyncCheckpoint method) finalize_fns (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) find_latest() (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager method) freeze() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest method) from_replication_params() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy class method) from_tensor_aware_state_dict() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO method)

G

gather_on_rank0 (nvidia_resiliency_ext.straggler.straggler.Detector attribute) generate_report() (nvidia_resiliency_ext.straggler.reporting.ReportGenerator method) (nvidia_resiliency_ext.straggler.straggler.Detector class method)

generate_report_if_interval_elapsed() (nvidia_resiliency_ext.straggler.straggler.Detector class method) get_num_unfinalized_calls() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method) get_partial_wrapper_constructor() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO class method)

H

HealthCheck (class in nvidia_resiliency_ext.inprocess.health_check) HealthCheckError

HeartbeatTimeouts (class in nvidia_resiliency_ext.fault_tolerance.rank_monitor_client) HierarchicalCheckpointIO (class in nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback)

I

identify_stragglers() (nvidia_resiliency_ext.straggler.reporting.Report method) init_tensors() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method) init_workload_monitoring() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) Initialize (class in nvidia_resiliency_ext.inprocess.initialize) initialize() (nvidia_resiliency_ext.straggler.straggler.Detector class method) initialized (nvidia_resiliency_ext.straggler.straggler.Detector attribute) insert_tensors() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method)

InternalError is_current_async_call_done() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.DistributedAsyncCaller method) is_frozen (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) is_hollow (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict property) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict property) is_interval_elapsed() (nvidia_resiliency_ext.straggler.straggler.Detector class method) iteration (nvidia_resiliency_ext.inprocess.CallWrapper property)

L

LazyCliqueReplicationStrategy (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies) LazyReplicationStrategyBuilder (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies) load() (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager method) load_checkpoint() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO method)

load_state_dict() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) local_ckpt_dir (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager.LocalCheckpointManager property) LocalCheckpointCallback (class in nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback) LocalCheckpointManager (class in nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager)

M

MaxActiveWorldSize (class in nvidia_resiliency_ext.inprocess.rank_filter) maybe_finalize_async_calls() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method) module nvidia_resiliency_ext.checkpointing.async_ckpt.core nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt nvidia_resiliency_ext.checkpointing.local.base_state_dict nvidia_resiliency_ext.checkpointing.local.basic_state_dict nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager nvidia_resiliency_ext.checkpointing.local.replication.strategies nvidia_resiliency_ext.fault_tolerance.rank_monitor_client nvidia_resiliency_ext.fault_tolerance.rank_monitor_server nvidia_resiliency_ext.inprocess, [1] nvidia_resiliency_ext.inprocess.abort nvidia_resiliency_ext.inprocess.exception nvidia_resiliency_ext.inprocess.finalize nvidia_resiliency_ext.inprocess.health_check nvidia_resiliency_ext.inprocess.initialize nvidia_resiliency_ext.inprocess.rank_assignment nvidia_resiliency_ext.inprocess.rank_filter nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback nvidia_resiliency_ext.straggler.reporting nvidia_resiliency_ext.straggler.statistics nvidia_resiliency_ext.straggler.straggler

N

name (nvidia_resiliency_ext.straggler.straggler.CallableId attribute) NoReplicasAvailableError nvidia_resiliency_ext.checkpointing.async_ckpt.core module nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt module nvidia_resiliency_ext.checkpointing.local.base_state_dict module nvidia_resiliency_ext.checkpointing.local.basic_state_dict module nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager module nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager module nvidia_resiliency_ext.checkpointing.local.replication.strategies module nvidia_resiliency_ext.fault_tolerance.rank_monitor_client module nvidia_resiliency_ext.fault_tolerance.rank_monitor_server module nvidia_resiliency_ext.inprocess module, [1] nvidia_resiliency_ext.inprocess.abort module

nvidia_resiliency_ext.inprocess.exception module nvidia_resiliency_ext.inprocess.finalize module nvidia_resiliency_ext.inprocess.health_check module nvidia_resiliency_ext.inprocess.initialize module nvidia_resiliency_ext.inprocess.rank_assignment module nvidia_resiliency_ext.inprocess.rank_filter module nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback module nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback module nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback module nvidia_resiliency_ext.straggler.reporting module nvidia_resiliency_ext.straggler.statistics module nvidia_resiliency_ext.straggler.straggler module

O

obj (nvidia_resiliency_ext.straggler.straggler.CallableId attribute) on_exception() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) on_load_checkpoint() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) on_save_checkpoint() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) on_train_batch_end() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) (nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback.StragglerDetectionCallback method) on_train_end() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method)

on_train_epoch_end() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.LocalCheckpointCallback method) on_train_start() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) on_validation_batch_end() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) on_validation_end() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.LocalCheckpointCallback method) on_validation_start() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) original_callables (nvidia_resiliency_ext.straggler.straggler.Detector attribute)

P

ping() (nvidia_resiliency_ext.inprocess.CallWrapper method) pop_tensors() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method)

profiling_interval (nvidia_resiliency_ext.straggler.straggler.Detector attribute)

R

rand_delay (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.SimulatedFaultParams attribute) rank (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager property) rank_to_fail (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.SimulatedFaultParams attribute) RankAssignment (class in nvidia_resiliency_ext.inprocess.rank_assignment) RankDiscarded RankFilter (class in nvidia_resiliency_ext.inprocess.rank_filter) RankMonitorClient (class in nvidia_resiliency_ext.fault_tolerance.rank_monitor_client) RankMonitorClientError RankMonitorServer (class in nvidia_resiliency_ext.fault_tolerance.rank_monitor_server) remove_checkpoint() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO method) replicate() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.ReplicationStrategy method) replication_strategy (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder property) ReplicationStrategy (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies) Report (class in nvidia_resiliency_ext.straggler.reporting)

report_interval_tracker (nvidia_resiliency_ext.straggler.straggler.Detector attribute) report_time_interval (nvidia_resiliency_ext.straggler.straggler.Detector attribute) reporter (nvidia_resiliency_ext.straggler.straggler.Detector attribute) ReportGenerator (class in nvidia_resiliency_ext.straggler.reporting) RestartAbort RestartError restore_original_callables() (nvidia_resiliency_ext.straggler.straggler.Detector class method) restore_tensor_device() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method) retrieve_execute() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.ReplicationStrategy method) retrieve_plan() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.ReplicationStrategy method) RetryController (class in nvidia_resiliency_ext.inprocess.initialize)

S

SameMachineReplicationException save() (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager method) save_checkpoint() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO method) schedule_async_call() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.DistributedAsyncCaller method) schedule_async_request() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method) scores_to_compute (nvidia_resiliency_ext.straggler.straggler.Detector attribute) send_heartbeat() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) setup() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) (nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback.StragglerDetectionCallback method)

ShiftRanks (class in nvidia_resiliency_ext.inprocess.rank_assignment) shutdown() (nvidia_resiliency_ext.straggler.straggler.Detector class method) shutdown_workload_monitoring() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) SimulatedFaultParams (class in nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback) state_dict() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) Statistic (class in nvidia_resiliency_ext.straggler.statistics) StragglerDetectionCallback (class in nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback) StragglerId (class in nvidia_resiliency_ext.straggler.reporting)

T

teardown() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) (nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback.StragglerDetectionCallback method) TensorAwareStateDict (class in nvidia_resiliency_ext.checkpointing.local.base_state_dict) tensors (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict property) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict property)

ThreadedFinalize (class in nvidia_resiliency_ext.inprocess.finalize) TimeoutError to_tensor_aware_state_dict() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO method) TorchAsyncCheckpoint (class in nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt)

V

values() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method)

W

WorldSizeDivisibleBy (class in nvidia_resiliency_ext.inprocess.rank_filter)

wrap_callables() (nvidia_resiliency_ext.straggler.straggler.Detector class method) Wrapper (class in nvidia_resiliency_ext.inprocess)