Index

_

__call__() (nvidia_resiliency_ext.inprocess.finalize.Finalize method)

__init__() (nvidia_resiliency_ext.shared_utils.log_aggregator.NodeLogAggregator method) (nvidia_resiliency_ext.shared_utils.log_manager.LogManager method)

A

abort_nvrx_checkpoint() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.core) add_finalize_fn() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest method) async_fn (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) (nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt.TorchAsyncCheckpoint attribute) async_fn_args (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) async_fn_kwargs (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute)

async_loop() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller static method) async_loop_for_daemon_worker() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller static method) async_process_target() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller static method) async_save() (nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt.TorchAsyncCheckpoint method) AsyncCaller (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) AsyncCallsQueue (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) AsyncRequest (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core)

B

BaseCheckpointManager (class in nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager)

BasicTensorAwareStateDict (class in nvidia_resiliency_ext.checkpointing.local.basic_state_dict)

C

calculate_and_set_hb_timeouts() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) calculate_and_set_section_timeouts() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) call_idx (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) checkpoint_id (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync property) CheckpointingException CheckpointMetadataCache (class in nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver) CliqueReplicationStrategy (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies)

close() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.TemporalAsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt.TorchAsyncCheckpoint method) copy_tensors_to_cpu() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method)

E

end_all_sections() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) end_section() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method)

execute_finalize_fns() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest method) execute_sync() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest method)

F

FaultToleranceConfig (class in nvidia_resiliency_ext.fault_tolerance.config) FileSystemWriterAsync (class in nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async) Finalize (class in nvidia_resiliency_ext.inprocess.finalize) finalize_async_save() (nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt.TorchAsyncCheckpoint method) finalize_fns (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) find_latest() (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager method)

finish() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method) freeze() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest method) from_args() (nvidia_resiliency_ext.fault_tolerance.config.FaultToleranceConfig static method) from_kwargs() (nvidia_resiliency_ext.fault_tolerance.config.FaultToleranceConfig static method) from_replication_params() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy class method) from_yaml_file() (nvidia_resiliency_ext.fault_tolerance.config.FaultToleranceConfig static method)

G

get_cache_metadata() (nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver.CheckpointMetadataCache method) get_infra_local_rank() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1] get_infra_rank() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1] get_instances() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.ObjectTracker method) get_log_file() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1] get_log_level() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1] get_log_to_stdout_cfg() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1] get_max_file_size() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1] get_max_log_files() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1]

get_metadata_caching_status() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver) (nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver.CheckpointMetadataCache method) get_node_id() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1] get_node_local_tmp_dir() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1] get_num_unfinalized_calls() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method) get_process_name() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1] get_save_function_and_args() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method) get_workload_local_rank() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1] get_workload_rank() (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig class method), [1]

H

HealthCheckError

I

infra_local_rank (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property), [1] infra_rank (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property), [1] init_checkpoint_metadata_cache() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver) init_tensors() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method) init_workload_monitoring() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) insert_tensors() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method)

InternalError is_current_async_call_done() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.TemporalAsyncCaller method) is_frozen (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute) is_hollow (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict property) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict property) is_progress_tracking_enabled (nvidia_resiliency_ext.fault_tolerance.config.FaultToleranceConfig property)

L

LazyCliqueReplicationStrategy (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies) LazyReplicationStrategyBuilder (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies) load() (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager method) load_state_dict() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method)

local_ckpt_dir (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager.LocalCheckpointManager property) LocalCheckpointManager (class in nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager) LogConfig (class in nvidia_resiliency_ext.shared_utils.log_manager), [1] logger (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property), [1] LogManager (class in nvidia_resiliency_ext.shared_utils.log_manager), [1]

M

main() (in module nvidia_resiliency_ext.shared_utils.log_aggregator) maybe_finalize_async_calls() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method) module nvidia_resiliency_ext.checkpointing.async_ckpt.core nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt nvidia_resiliency_ext.checkpointing.local.base_state_dict nvidia_resiliency_ext.checkpointing.local.basic_state_dict nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager nvidia_resiliency_ext.checkpointing.local.replication.strategies nvidia_resiliency_ext.fault_tolerance.config nvidia_resiliency_ext.fault_tolerance.rank_monitor_client nvidia_resiliency_ext.inprocess.exception nvidia_resiliency_ext.inprocess.finalize nvidia_resiliency_ext.shared_utils.log_aggregator nvidia_resiliency_ext.shared_utils.log_manager

N

name (nvidia_resiliency_ext.shared_utils.log_manager.LogConfig attribute), [1] node_local_tmp_logging_enabled (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property), [1] NodeLogAggregator (class in nvidia_resiliency_ext.shared_utils.log_aggregator) NoReplicasAvailableError nvidia_resiliency_ext.checkpointing.async_ckpt.core module nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async module nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver module nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt module nvidia_resiliency_ext.checkpointing.local.base_state_dict module nvidia_resiliency_ext.checkpointing.local.basic_state_dict module nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager module

nvidia_resiliency_ext.checkpointing.local.ckpt_managers.local_manager module nvidia_resiliency_ext.checkpointing.local.replication.strategies module nvidia_resiliency_ext.fault_tolerance.config module nvidia_resiliency_ext.fault_tolerance.rank_monitor_client module nvidia_resiliency_ext.inprocess.exception module nvidia_resiliency_ext.inprocess.finalize module nvidia_resiliency_ext.shared_utils.log_aggregator module nvidia_resiliency_ext.shared_utils.log_manager module

O

ObjectTracker (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core)

P

PersistentAsyncCaller (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) pop_tensors() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method) preload_fn (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncRequest attribute)

preload_tensors() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync static method) prepare_decentralized_global_plan() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method) prepare_local_plan() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method) prepare_save_state_dict_ret() (nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver.CheckpointMetadataCache method) prepare_write_data() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method)

R

rank (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager property) RankMonitorClient (class in nvidia_resiliency_ext.fault_tolerance.rank_monitor_client) RankMonitorClientError replicate() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.ReplicationStrategy method) replication_strategy (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder property) ReplicationStrategy (class in nvidia_resiliency_ext.checkpointing.local.replication.strategies) RestartAbort

RestartError restore_tensor_device() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict method) retrieve_execute() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.ReplicationStrategy method) retrieve_plan() (nvidia_resiliency_ext.checkpointing.local.replication.strategies.CliqueReplicationStrategy method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.LazyReplicationStrategyBuilder method) (nvidia_resiliency_ext.checkpointing.local.replication.strategies.ReplicationStrategy method) retrieve_write_results() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method)

S

SameMachineReplicationException save() (nvidia_resiliency_ext.checkpointing.local.ckpt_managers.base_manager.BaseCheckpointManager method) save_state_dict_async_finalize() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver) save_state_dict_async_plan() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver) schedule_async_call() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.PersistentAsyncCaller method) (nvidia_resiliency_ext.checkpointing.async_ckpt.core.TemporalAsyncCaller method) schedule_async_request() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCallsQueue method) send_heartbeat() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method)

send_workload_control_request() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) set_cache_metadata() (nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver.CheckpointMetadataCache method) set_cached_global_metadata() (nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver.CheckpointMetadataCache method) setup_logger() (in module nvidia_resiliency_ext.shared_utils.log_manager), [1] shutdown() (nvidia_resiliency_ext.shared_utils.log_aggregator.NodeLogAggregator method) shutdown_workload_monitoring() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) start_aggregator() (nvidia_resiliency_ext.shared_utils.log_aggregator.NodeLogAggregator method) start_section() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) state_dict() (nvidia_resiliency_ext.fault_tolerance.rank_monitor_client.RankMonitorClient method) sync_all_async_calls() (nvidia_resiliency_ext.checkpointing.async_ckpt.core.AsyncCaller method)

T

TemporalAsyncCaller (class in nvidia_resiliency_ext.checkpointing.async_ckpt.core) TensorAwareStateDict (class in nvidia_resiliency_ext.checkpointing.local.base_state_dict) tensors (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict property) (nvidia_resiliency_ext.checkpointing.local.basic_state_dict.BasicTensorAwareStateDict property)

ThreadedFinalize (class in nvidia_resiliency_ext.inprocess.finalize) TimeoutError to_yaml_file() (nvidia_resiliency_ext.fault_tolerance.config.FaultToleranceConfig method) TorchAsyncCheckpoint (class in nvidia_resiliency_ext.checkpointing.async_ckpt.torch_ckpt)

V

validate_checkpoint_id() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync class method)

values() (nvidia_resiliency_ext.checkpointing.local.base_state_dict.TensorAwareStateDict method) verify_global_md_reuse() (in module nvidia_resiliency_ext.checkpointing.async_ckpt.state_dict_saver)

W

workload_local_rank (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property), [1] workload_rank (nvidia_resiliency_ext.shared_utils.log_manager.LogManager property), [1] write_data() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync method)

write_preloaded_data() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync static method) write_preloaded_data_multiproc() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync static method) write_preloaded_data_multithread_launcher() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync static method) write_preloaded_data_thread() (nvidia_resiliency_ext.checkpointing.async_ckpt.filesystem_async.FileSystemWriterAsync static method)