Index

_

__call__() (nvidia_resiliency_ext.inprocess.abort.Abort method) (nvidia_resiliency_ext.inprocess.finalize.Finalize method) (nvidia_resiliency_ext.inprocess.health_check.HealthCheck method) (nvidia_resiliency_ext.inprocess.initialize.Initialize method) (nvidia_resiliency_ext.inprocess.rank_assignment.RankAssignment method) (nvidia_resiliency_ext.inprocess.rank_assignment.RankFilter method)

__init__() (nvidia_resiliency_ext.ptl_resiliency.StragglerDetectionCallback method) __str__() (nvidia_resiliency_ext.attribution.straggler.straggler.CallableId method)

A

Abort (class in nvidia_resiliency_ext.inprocess.abort) abort_nvrx_checkpoint() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.core) AbortPersistentCheckpointProcesses (class in nvidia_resiliency_ext.inprocess.abort) AbortTorchDistributed (class in nvidia_resiliency_ext.inprocess.abort) AbortTransformerEngine (class in nvidia_resiliency_ext.inprocess.abort) add_finalize_fn() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest method) async_fn (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) (nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt.TorchAsyncCheckpoint attribute) async_fn_args (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute)

async_fn_kwargs (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) async_loop() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller static method) async_loop_for_daemon_worker() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller static method) async_process_target() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller static method) async_save() (nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt.TorchAsyncCheckpoint method) AsyncCaller (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) AsyncCallsQueue (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) AsyncRequest (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) atomic() (nvidia_resiliency_ext.inprocess.CallWrapper method)

B

BACKFILL (nvidia_resiliency_ext.inprocess.rank_assignment.LayerFlag attribute)

BaseCheckpointManager (class in nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager) BasicTensorAwareStateDict (class in nvidia_resiliency_ext.checkpointing.local.basic_state_dict)

C

calculate_and_set_hb_timeouts() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) calculate_and_set_section_timeouts() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) call_idx (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) CallableId (class in nvidia_resiliency_ext.attribution.straggler.straggler) CallWrapper (class in nvidia_resiliency_ext.inprocess) ChainedGPUHealthCheck (class in nvidia_resiliency_ext.inprocess.health_check) ChainedNicHealthCheck (class in nvidia_resiliency_ext.inprocess.health_check) ChainedNVLHealthCheck (class in nvidia_resiliency_ext.inprocess.health_check) checkpoint_id (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync property) CheckpointingException CheckpointMetadataCache (class in nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver) CKPT_CALL_IDX (in module nvidia_resiliency_ext.shared_utils.semconv) cleanup_tensor_caches() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync class method) cleanup_worker_data_cache() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller class method) CliqueReplicationStrategy (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies)

close() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.TemporalAsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt.TorchAsyncCheckpoint method) comp_q (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller attribute) ConsistentDataIdentifier (class in nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async) copy_tensors_to_cpu() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method) CudaHealthCheck (class in nvidia_resiliency_ext.inprocess.health_check) cupti_manager (nvidia_resiliency_ext.attribution.straggler.straggler.Detector attribute) cur_idx (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller attribute) cur_item (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller attribute) custom_sections (nvidia_resiliency_ext.attribution.straggler.straggler.Detector attribute) CustomSection (class in nvidia_resiliency_ext.attribution.straggler.straggler)

D

detection_section() (nvidia_resiliency_ext.attribution.straggler.straggler.Detector class method)

Detector (class in nvidia_resiliency_ext.attribution.straggler.straggler) disable_hang_protection() (nvidia_resiliency_ext.inprocess.CallWrapper method)

E

end_all_sections() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) end_section() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method)

execute_finalize_fns() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest method) execute_sync() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest method)

F

FaultCounter (class in nvidia_resiliency_ext.inprocess.health_check) FaultCounterExceeded FaultToleranceCallback (class in nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback) FaultToleranceConfig (class in nvidia_resiliency_ext.fault_tolerance.config) FileSystemWriterAsync (class in nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async) Finalize (class in nvidia_resiliency_ext.inprocess.finalize) finalize_async_save() (nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt.TorchAsyncCheckpoint method) finalize_fns (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute)

find_latest() (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager method) finish() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method) freeze() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest method) from_args() (nvidia_resiliency_ext.fault_tolerance.config.FaultToleranceConfig static method) from_kwargs() (nvidia_resiliency_ext.fault_tolerance.config.FaultToleranceConfig static method) from_replication_params() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy class method) from_tensor_aware_state_dict() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO method) from_yaml_file() (nvidia_resiliency_ext.fault_tolerance.config.FaultToleranceConfig static method)

G

gather_on_rank0 (nvidia_resiliency_ext.attribution.straggler.straggler.Detector attribute) generate_report() (nvidia_resiliency_ext.attribution.straggler.reporting.ReportGenerator method) (nvidia_resiliency_ext.attribution.straggler.straggler.Detector class method) generate_report_if_interval_elapsed() (nvidia_resiliency_ext.attribution.straggler.straggler.Detector class method) get_cache_metadata() (nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver.CheckpointMetadataCache method) get_infra_local_rank() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method) get_infra_rank() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method) get_instances() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.ObjectTracker method) get_log_file() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method) get_log_level() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method) get_log_to_stdout_cfg() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method)

get_max_file_size() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method) get_max_log_files() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method) get_node_id() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method) get_node_local_tmp_dir() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method) get_num_unfinalized_calls() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method) get_partial_wrapper_constructor() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO class method) get_process_name() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method) get_save_function_and_args() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method) get_workload_local_rank() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method) get_workload_rank() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method) get_write_results_queue() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async)

H

HealthCheck (class in nvidia_resiliency_ext.inprocess.health_check)

HealthCheckError HierarchicalCheckpointIO (class in nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback)

I

identify_stragglers() (nvidia_resiliency_ext.attribution.straggler.reporting.Report method) infra_local_rank (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property) infra_rank (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property) init_checkpoint_metadata_cache() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver) init_tensors() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method) init_workload_monitoring() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) Initialize (class in nvidia_resiliency_ext.inprocess.initialize) initialize() (nvidia_resiliency_ext.attribution.straggler.straggler.Detector class method) initialized (nvidia_resiliency_ext.attribution.straggler.straggler.Detector attribute) insert_tensors() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method)

InternalError is_current_async_call_done() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.TemporalAsyncCaller method) is_frozen (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) is_hollow (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict property) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict property) is_interval_elapsed() (nvidia_resiliency_ext.attribution.straggler.straggler.Detector class method) is_progress_tracking_enabled (nvidia_resiliency_ext.fault_tolerance.config.FaultToleranceConfig property) iteration (nvidia_resiliency_ext.inprocess.CallWrapper property)

L

Layer (class in nvidia_resiliency_ext.inprocess.rank_assignment) LayerFlag (class in nvidia_resiliency_ext.inprocess.rank_assignment) LazyCliqueReplicationStrategy (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies) LazyReplicationStrategyBuilder (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies) load() (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager method) load_checkpoint() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO method) load_state_dict() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method)

local_ckpt_dir (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager.LocalCheckpointManager property) LocalCheckpointCallback (class in nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback) LocalCheckpointManager (class in nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager) log_restarter_event() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_server.RankMonitorLogger method) LogConfig (class in nvidia_resiliency_ext.shared_utils.log_manager) logger (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property) LogManager (class in nvidia_resiliency_ext.shared_utils.log_manager)

M

main() (in module nvidia_resiliency_ext.shared_utils.log_aggregator) maybe_finalize_async_calls() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method) module nvidia_resiliency_ext.attribution.straggler.reporting nvidia_resiliency_ext.attribution.straggler.statistics nvidia_resiliency_ext.attribution.straggler.straggler nvidia_resiliency_ext.checkpointing.async_ckpt.core nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt nvidia_resiliency_ext.checkpointing.local.base_state_dict nvidia_resiliency_ext.checkpointing.local.basic_state_dict nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager nvidia_resiliency_ext.checkpointing.local.replication.strategies nvidia_resiliency_ext.fault_tolerance.config nvidia_resiliency_ext.fault_tolerance.rank_monitor_client nvidia_resiliency_ext.fault_tolerance.rank_monitor_server nvidia_resiliency_ext.inprocess nvidia_resiliency_ext.inprocess.abort nvidia_resiliency_ext.inprocess.exception nvidia_resiliency_ext.inprocess.finalize nvidia_resiliency_ext.inprocess.health_check nvidia_resiliency_ext.inprocess.initialize nvidia_resiliency_ext.inprocess.rank_assignment nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback nvidia_resiliency_ext.shared_utils.log_aggregator nvidia_resiliency_ext.shared_utils.log_manager nvidia_resiliency_ext.shared_utils.semconv

N

name (nvidia_resiliency_ext.attribution.straggler.straggler.CallableId attribute) (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig attribute) node_local_tmp_logging_enabled (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property) NodeLogAggregator (class in nvidia_resiliency_ext.shared_utils.log_node_local_tmp) NoReplicasAvailableError nvidia_resiliency_ext.attribution.straggler.reporting module nvidia_resiliency_ext.attribution.straggler.statistics module nvidia_resiliency_ext.attribution.straggler.straggler module nvidia_resiliency_ext.checkpointing.async_ckpt.core module nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async module nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver module nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt module nvidia_resiliency_ext.checkpointing.local.base_state_dict module nvidia_resiliency_ext.checkpointing.local.basic_state_dict module nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager module nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager module nvidia_resiliency_ext.checkpointing.local.replication.strategies module nvidia_resiliency_ext.fault_tolerance.config module

nvidia_resiliency_ext.fault_tolerance.rank_monitor_client module nvidia_resiliency_ext.fault_tolerance.rank_monitor_server module nvidia_resiliency_ext.inprocess module nvidia_resiliency_ext.inprocess.abort module nvidia_resiliency_ext.inprocess.exception module nvidia_resiliency_ext.inprocess.finalize module nvidia_resiliency_ext.inprocess.health_check module nvidia_resiliency_ext.inprocess.initialize module nvidia_resiliency_ext.inprocess.rank_assignment module nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback module nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback module nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback module nvidia_resiliency_ext.shared_utils.log_aggregator module nvidia_resiliency_ext.shared_utils.log_manager module nvidia_resiliency_ext.shared_utils.semconv module

O

obj (nvidia_resiliency_ext.attribution.straggler.straggler.CallableId attribute) ObjectTracker (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) on_exception() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) on_load_checkpoint() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) on_save_checkpoint() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) on_train_batch_end() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) (nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback.StragglerDetectionCallback method)

on_train_end() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) on_train_epoch_end() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.LocalCheckpointCallback method) on_train_start() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) on_validation_batch_end() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) on_validation_end() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.LocalCheckpointCallback method) on_validation_start() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) original_callables (nvidia_resiliency_ext.attribution.straggler.straggler.Detector attribute)

P

PersistentAsyncCaller (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) ping() (nvidia_resiliency_ext.inprocess.CallWrapper method) pop_tensors() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method) preload_fn (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) preload_q (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller attribute)

preload_tensors() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync static method) prepare_decentralized_global_plan() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method) prepare_local_plan() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method) prepare_save_state_dict_ret() (nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver.CheckpointMetadataCache method) prepare_write_data() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method) process (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller attribute) profiling_interval (nvidia_resiliency_ext.attribution.straggler.straggler.Detector attribute)

Q

queue (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller attribute)

R

rank (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller attribute) (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager property) RankAssignment (class in nvidia_resiliency_ext.inprocess.rank_assignment) RankAssignmentCtx (class in nvidia_resiliency_ext.inprocess.rank_assignment) RankDiscarded RankFilter (class in nvidia_resiliency_ext.inprocess.rank_assignment) RankMonitorClient (class in nvidia_resiliency_ext.fault_tolerance.rank_monitor_client) RankMonitorClientError RankMonitorLogger (class in nvidia_resiliency_ext.fault_tolerance.rank_monitor_server) RankMonitorServer (class in nvidia_resiliency_ext.fault_tolerance.rank_monitor_server) register_shm_drain_callback() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync class method) register_worker_restart_callback() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller class method) remove_checkpoint() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO method) replicate() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.ReplicationStrategy method) replication_strategy (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder property) ReplicationStrategy (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies)

Report (class in nvidia_resiliency_ext.attribution.straggler.reporting) report_interval_tracker (nvidia_resiliency_ext.attribution.straggler.straggler.Detector attribute) report_time_interval (nvidia_resiliency_ext.attribution.straggler.straggler.Detector attribute) reporter (nvidia_resiliency_ext.attribution.straggler.straggler.Detector attribute) ReportGenerator (class in nvidia_resiliency_ext.attribution.straggler.reporting) RESERVE (nvidia_resiliency_ext.inprocess.rank_assignment.LayerFlag attribute) RestartAbort RestartError restore_original_callables() (nvidia_resiliency_ext.attribution.straggler.straggler.Detector class method) restore_tensor_device() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method) retrieve_execute() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.ReplicationStrategy method) retrieve_plan() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.ReplicationStrategy method) retrieve_write_results() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method) RetryController (class in nvidia_resiliency_ext.inprocess.initialize)

S

SameMachineReplicationException save() (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager method) save_checkpoint() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO method) save_state_dict_async_finalize() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver) save_state_dict_async_plan() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver) schedule_async_call() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.TemporalAsyncCaller method) schedule_async_request() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method) scores_to_compute (nvidia_resiliency_ext.attribution.straggler.straggler.Detector attribute) send_heartbeat() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) send_workload_control_request() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) set_cache_metadata() (nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver.CheckpointMetadataCache method) set_cached_global_metadata() (nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver.CheckpointMetadataCache method) setup() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) (nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback.StragglerDetectionCallback method)

setup_logger() (in module nvidia_resiliency_ext.shared_utils.log_manager) shutdown() (nvidia_resiliency_ext.attribution.straggler.straggler.Detector class method) (nvidia_resiliency_ext.shared_utils.log_node_local_tmp.NodeLogAggregator method) shutdown_workload_monitoring() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) SPAN_GROUP_CKPT (in module nvidia_resiliency_ext.shared_utils.semconv) SPAN_GROUP_CKPT_PHASES (in module nvidia_resiliency_ext.shared_utils.semconv) SPAN_GROUP_FT (in module nvidia_resiliency_ext.shared_utils.semconv) SPAN_GROUP_STARTUP (in module nvidia_resiliency_ext.shared_utils.semconv) start_aggregator() (nvidia_resiliency_ext.shared_utils.log_node_local_tmp.NodeLogAggregator method) start_section() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) start_time (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller attribute) state_dict() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) Statistic (class in nvidia_resiliency_ext.attribution.straggler.statistics) StragglerDetectionCallback (class in nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback) StragglerId (class in nvidia_resiliency_ext.attribution.straggler.reporting) sync_all_async_calls() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCaller method)

T

teardown() (nvidia_resiliency_ext.ptl_resiliency.fault_tolerance_callback.FaultToleranceCallback method) (nvidia_resiliency_ext.ptl_resiliency.straggler_det_callback.StragglerDetectionCallback method) TemporalAsyncCaller (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) TensorAwareStateDict (class in nvidia_resiliency_ext.checkpointing.local.base_state_dict) tensors (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict property) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict property)

ThreadedFinalize (class in nvidia_resiliency_ext.inprocess.finalize) TimeoutError to_tensor_aware_state_dict() (nvidia_resiliency_ext.ptl_resiliency.local_checkpoint_callback.HierarchicalCheckpointIO method) to_yaml_file() (nvidia_resiliency_ext.fault_tolerance.config.FaultToleranceConfig method) TorchAsyncCheckpoint (class in nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt) Tree (class in nvidia_resiliency_ext.inprocess.rank_assignment)

V

validate_checkpoint_id() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync class method)

values() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) verify_global_md_reuse() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver)

W

warmup_persistent_caller() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue class method) workload_local_rank (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property) workload_rank (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property) wrap_callables() (nvidia_resiliency_ext.attribution.straggler.straggler.Detector class method) Wrapper (class in nvidia_resiliency_ext.inprocess)

write_data() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method) write_preloaded_data() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync static method) write_preloaded_data_multiproc() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync static method) write_preloaded_data_multithread() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync static method) write_preloaded_data_proc() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync static method)