# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runtime helpers for dataloading, device placement, and parallelism setup."""
from __future__ import annotations
from collections.abc import Callable, Iterator, Mapping, Sequence
from contextlib import contextmanager
from typing import Any, TypeVar
import torch
from torch.utils.data import DataLoader
_ModelT = TypeVar("_ModelT", bound=torch.nn.Module)
__all__ = [
"configure_dataloader",
"configure_parallelism",
"eval_configured_models",
"evaluating",
"freeze_unconfigured_models",
"move_to_devices",
"rehome_optimizer_state",
"train_configured_models",
"unwrap_model",
]
@contextmanager
def train_configured_models(
models: dict[str, torch.nn.Module] | torch.nn.ModuleDict,
optimizer_configs: Mapping[str, object],
) -> Iterator[None]:
"""Temporarily put optimizer-configured models in training mode.
Parameters
----------
models : dict[str, torch.nn.Module] | torch.nn.ModuleDict
Named models participating in a training run.
optimizer_configs : Mapping[str, object]
Optimizer configuration keyed by model name. Models present in this
mapping are switched to training mode while the context is active.
Yields
------
None
Control while configured models are in training mode.
"""
state = {
key: model.training for key, model in models.items() if key in optimizer_configs
}
for key in state:
models[key].train()
try:
yield
finally:
for key, training in state.items():
models[key].train(training)
[docs]
@contextmanager
def evaluating(module: torch.nn.Module) -> Iterator[None]:
"""Temporarily put a module tree in evaluation mode.
Parameters
----------
module : torch.nn.Module
Module whose whole tree is switched to evaluation mode. Every
submodule's own ``training`` flag is restored on exit, so a child left
in training mode under an evaluation-mode root, or frozen on its own,
comes back exactly as it was.
Yields
------
None
Control while the module tree is in evaluation mode.
Notes
-----
:meth:`torch.nn.Module.train` is recursive, so restoring the root's flag
alone would overwrite every child's flag with the root's. This helper
records and restores each submodule individually.
"""
modes = {submodule: submodule.training for submodule in module.modules()}
module.eval()
try:
yield
finally:
for submodule, training in modes.items():
submodule.training = training
[docs]
def move_to_devices(
models: torch.nn.Module | dict[str, torch.nn.Module] | torch.nn.ModuleDict,
devices: Sequence[torch.device],
*,
non_blocking: bool = False,
) -> torch.nn.Module | dict[str, torch.nn.Module] | torch.nn.ModuleDict:
"""Move one model or named models to device(s), preserving input shape.
Parameters
----------
models : torch.nn.Module | dict[str, torch.nn.Module] | torch.nn.ModuleDict
Single module or named modules. Named modules are assigned devices in
insertion order.
devices : Sequence[torch.device]
One device broadcasts to all models; otherwise length must match the
number of models.
non_blocking : bool, optional
Forwarded to :meth:`torch.nn.Module.to`.
Returns
-------
torch.nn.Module | dict[str, torch.nn.Module] | torch.nn.ModuleDict
The same input shape after in-place ``.to(...)`` calls.
Raises
------
ValueError
If ``devices`` has length other than ``1`` or the number of models.
"""
if isinstance(models, (dict, torch.nn.ModuleDict)):
if len(devices) not in (1, len(models)):
raise ValueError(
f"devices must have length 1 or len(models)={len(models)}; "
f"got {len(devices)}."
)
expanded = list(devices) if len(devices) != 1 else list(devices) * len(models)
for model, device in zip(models.values(), expanded, strict=True):
model.to(device, non_blocking=non_blocking)
return models
if len(devices) != 1:
raise ValueError(
f"single-model device assignment requires exactly one device; "
f"got {len(devices)}."
)
return models.to(devices[0], non_blocking=non_blocking)
def _rehome_value(value: Any, device: torch.device) -> Any:
"""Return ``value`` with every tensor nested inside it placed on ``device``.
Dicts and lists are rewritten in place and tuples are rebuilt through their
own type (``_make`` for named tuples), so the container an optimizer chose
survives the walk. Anything that is neither a tensor nor one of those
containers is returned untouched.
"""
if isinstance(value, torch.Tensor):
return value if value.device == device else value.to(device)
if isinstance(value, dict):
for key, item in value.items():
value[key] = _rehome_value(item, device)
return value
if isinstance(value, list):
value[:] = [_rehome_value(item, device) for item in value]
return value
if isinstance(value, tuple):
moved = [_rehome_value(item, device) for item in value]
make = getattr(value, "_make", None)
return make(moved) if callable(make) else type(value)(moved)
return value
[docs]
def rehome_optimizer_state(optimizer: torch.optim.Optimizer) -> None:
"""Move an optimizer's per-parameter state onto its parameters' devices.
:meth:`torch.optim.Optimizer.load_state_dict` places state on the parameter
devices as they stand at load time and never revisits them, so any move
afterwards strands the state: resuming a checkpoint and then calling
:meth:`~nvalchemi.training.TrainingStrategy.run`, or letting a
:class:`~nvalchemi.training.hooks.DDPHook` re-pin a rank to its local GPU,
leaves ``exp_avg`` on the old device and the first step raises ``Expected
all tensors to be on the same device``. Call this after the parameters have
reached their final devices and before the first step.
A top-level ``step`` entry is left on the CPU when it is already there,
which is the placement PyTorch uses for optimizers that are neither
capturable nor fused; every other tensor, ``step`` included, follows its
parameter.
Each state entry is walked recursively, so a custom optimizer that keeps
its moments inside a dict, list, or tuple is rehomed as thoroughly as
:class:`~torch.optim.Adam`, and the containers it chose are preserved.
Parameters
----------
optimizer : torch.optim.Optimizer
Optimizer whose state is rehomed in place. Parameters without state
(never stepped) are skipped.
Returns
-------
None
Examples
--------
>>> from nvalchemi.training.runtime import rehome_optimizer_state
>>> model.to("cuda:1") # doctest: +SKIP
>>> rehome_optimizer_state(optimizer) # doctest: +SKIP
"""
for group in optimizer.param_groups:
step_follows_param = bool(group.get("capturable") or group.get("fused"))
for param in group["params"]:
state = optimizer.state.get(param)
if not state:
continue
for key, value in state.items():
if (
key == "step"
and not step_follows_param
and isinstance(value, torch.Tensor)
and value.device.type == "cpu"
):
continue
state[key] = _rehome_value(value, param.device)
[docs]
def unwrap_model(model: _ModelT) -> _ModelT:
"""Return the module a parallelism wrapper owns, or the model itself.
Parameters
----------
model : torch.nn.Module
A model as a strategy holds it, wrapped or bare.
Returns
-------
torch.nn.Module
The wrapped module, or ``model`` unchanged when nothing wraps it.
Notes
-----
A wrapper is recognized by the ``module`` attribute it publishes, not by
its class. A hand-rolled or FSDP wrapper therefore unwraps exactly as a
:class:`~torch.nn.parallel.DistributedDataParallel` replica does. The
return type is the argument's type, because a wrapper replaces the model
at run time behind the type the caller declared.
"""
module = getattr(model, "module", None)
return model if module is None else module